Data Toolkit 速查

讀 / 寫

from pyspark.sql import functions as F, Window
from pyspark.sql.types import *

df = (spark.read.option("header", True)
      .option("inferSchema", False)
      .csv("/path/pos/*.csv"))
js = spark.read.json("/path/mobile/")         # JSONL 一行一個
df = df.withColumn("_src", F.col("_metadata.file_path"))

# 明確 schema（production 做法）
schema = StructType([
  StructField("order_id", StringType()),
  StructField("amount",   StringType()),
])

(df.write.format("delta").mode("overwrite")
   .saveAsTable("silver.pos"))
df.write.mode("overwrite").partitionBy("dt").parquet("/out")

Transform

df = (df
  .withColumn("channel", F.initcap(F.trim("channel")))
  .withColumn("amt", F.expr(
      "try_cast(regexp_replace(amount,'[^0-9.]','') AS DOUBLE)"))
  .withColumn("ts", F.coalesce(
      F.expr("try_to_timestamp(ts_raw,'yyyy-MM-dd HH:mm:ss')"),
      F.expr("try_to_timestamp(ts_raw,'dd/MM/yyyy HH:mm')"),
      F.timestamp_seconds(F.expr("try_cast(ts_raw AS BIGINT)"))))
  .withColumn("ts_hk", F.from_utc_timestamp("ts", "Asia/Hong_Kong"))
  .withColumn("tier", F.when(F.col("amt") > 100, "High")
                        .otherwise("Low"))
  .filter(F.col("amt").isNotNull())
  .dropDuplicates(["order_id", "line_no"])
  .withColumnRenamed("sku", "product_id")
  .drop("amount"))

Aggregate / Join

agg = (df.groupBy("store_id", F.to_date("ts_hk").alias("dt"))
  .agg(F.sum("amt").alias("sales"),
       F.countDistinct("order_id").alias("orders"),
       F.avg("amt").alias("avg_line")))

f = fact.join(F.broadcast(dim_store), "store_id", "left")
orphans = fact.join(dim_store, "store_id", "left_anti")

# 唔同名 key
a.join(b, a.pid == b.policy_id, "inner")

# pivot
df.groupBy("store_id").pivot("channel").sum("amt")

Window function

w = Window.partitionBy("member_id").orderBy("ts")

df = (df
  .withColumn("prev_ts", F.lag("ts").over(w))
  .withColumn("next_ts", F.lead("ts").over(w))
  .withColumn("gap_days", F.datediff("ts", "prev_ts"))
  .withColumn("rn", F.row_number().over(w))
  .withColumn("running", F.sum("amt").over(
      w.rowsBetween(Window.unboundedPreceding, 0))))

# 每組 Top 3
wr = Window.partitionBy("region").orderBy(F.desc("sales"))
top3 = agg.withColumn("rk", F.dense_rank().over(wr)).filter("rk <= 3")

# 留最新版本
wl = Window.partitionBy("id").orderBy(F.desc("updated_at"))
latest = df.withColumn("rn", F.row_number().over(wl)).filter("rn = 1")

Nested JSON / array

lines = (js
  .select("order_id", "ts_utc", F.explode("items").alias("it"))
  .select("order_id", "ts_utc",
          F.col("it.sku").alias("sku"),
          F.col("it.qty").alias("qty"),
          F.col("it.price").alias("price")))

# explode_outer：空 array 都保留一行
# higher-order function（唔使 explode）
js = js.withColumn("order_total", F.aggregate(
        "items", F.lit(0.0), lambda acc, x: acc + x["qty"] * x["price"]))
js = js.withColumn("n_custom", F.size(F.filter(
        "items", lambda x: F.size(x["customizations"]) > 0)))

Scaffold / 補 0 / SQL

months = spark.sql("""
  SELECT explode(sequence(DATE'2025-01-01', DATE'2025-12-01',
                          INTERVAL 1 MONTH)) AS m""")
grid = months.crossJoin(stores.select("store_id"))
full = (grid.join(monthly, ["m", "store_id"], "left")
            .fillna(0, subset=["orders"]))

# 隨時轉 SQL
df.createOrReplaceTempView("pos")
spark.sql("""SELECT store_id, SUM(amt) sales
             FROM pos GROUP BY store_id ORDER BY sales DESC""")

df.explain()     # 睇 plan：BroadcastHashJoin？
df.count(); df.printSchema(); df.show(5, truncate=False)

Medallion 分層

Bronze

Raw 照入，加 _source_file、_ingest_ts，全部 string，唔清理

Silver

清理、type、時區統一、去重、DQ check、SCD2 dim

Gold

Business grain：fact_sales_line、日報、KPI view，俾 Tableau

SCD2 MERGE

MERGE INTO silver.dim_member t
USING (
  SELECT c.member_id AS mk, c.* FROM changes c
  UNION ALL
  SELECT NULL AS mk, c.* FROM changes c
  JOIN silver.dim_member d
    ON c.member_id = d.member_id
   AND d.is_current AND c.tier <> d.tier
) s
ON t.member_id = s.mk AND t.is_current
WHEN MATCHED AND t.tier <> s.tier THEN UPDATE
  SET is_current = false, end_ts = s.change_ts
WHEN NOT MATCHED THEN INSERT
  (member_id, tier, start_ts, end_ts, is_current)
  VALUES (s.member_id, s.tier, s.change_ts, NULL, true);

Upsert / idempotent

MERGE INTO silver.orders t
USING updates s ON t.order_id = s.order_id
WHEN MATCHED AND s.updated_at > t.updated_at
  THEN UPDATE SET *
WHEN NOT MATCHED THEN INSERT *;

# 覆寫某段分區，重跑唔重複
(df.write.format("delta").mode("overwrite")
   .option("replaceWhere", "dt >= '2025-10-01' AND dt < '2025-11-01'")
   .saveAsTable("silver.pos"))

Auto Loader（Databricks）

(spark.readStream.format("cloudFiles")
  .option("cloudFiles.format", "json")
  .option("cloudFiles.schemaLocation", CHK + "/schema")
  .load(RAW + "/mobile/")
  .withColumn("_src", F.col("_metadata.file_path"))
  .writeStream
  .option("checkpointLocation", CHK + "/mobile")
  .trigger(availableNow=True)
  .toTable("bronze.mobile"))

Colab / OSS Spark 冇 cloudFiles：改用 spark.readStream.schema(s).json(path)，checkpoint 一樣記住讀過嘅檔。

Table 管理

DESCRIBE HISTORY silver.pos;
SELECT * FROM silver.pos VERSION AS OF 3;
SELECT * FROM silver.pos TIMESTAMP AS OF '2025-10-01';
RESTORE TABLE silver.pos TO VERSION AS OF 3;

ALTER TABLE silver.pos ADD CONSTRAINT amt_ok CHECK (amt >= 0);
ALTER TABLE silver.pos ALTER COLUMN order_id SET NOT NULL;

CREATE TABLE gold.fact_sales_line (...) CLUSTER BY (store_id, dt);
OPTIMIZE gold.fact_sales_line;

CREATE OR REPLACE VIEW gold.v_store_daily AS
SELECT store_id, dt, SUM(amt) sales, COUNT(DISTINCT order_id) orders
FROM gold.fact_sales_line GROUP BY store_id, dt;

Colab setup（練習用）

!pip install -q pyspark==3.5.3 delta-spark==3.2.1
from pyspark.sql import SparkSession
from delta import configure_spark_with_delta_pip
b = (SparkSession.builder
  .config("spark.sql.extensions",
          "io.delta.sql.DeltaSparkSessionExtension")
  .config("spark.sql.catalog.spark_catalog",
          "org.apache.spark.sql.delta.catalog.DeltaCatalog")
  .config("spark.sql.ansi.enabled", "true"))
spark = configure_spark_with_delta_pip(b).getOrCreate()

Databricks serverless 預設 ANSI：parse / cast 失敗會報錯，所以用 try_cast、try_to_timestamp。Serverless 冇 .cache()、RDD。