Serverless 컴퓨트

-- 서버리스 SQL Warehouse에서 쿼리 실행 (일반 SQL과 동일)
SELECT
    region,
    COUNT(*) AS order_count,
    SUM(revenue) AS total_revenue
FROM production.ecommerce.orders
WHERE order_date >= '2025-01-01'
GROUP BY region
ORDER BY total_revenue DESC;

# 노트북 상단에서 %pip로 라이브러리 설치 가능
%pip install scikit-learn==1.4.0 xgboost==2.0.0

# 설치 후 바로 사용
import sklearn
import xgboost

# databricks.yml - 서버리스 Job 설정
resources:
  jobs:
    daily_etl:
      name: "Daily ETL Pipeline"
      tasks:
        - task_key: "extract"
          notebook_task:
            notebook_path: "/Workspace/etl/extract"
          # environment_key를 지정하면 Serverless로 실행됩니다
          environment_key: "default"

        - task_key: "transform"
          depends_on:
            - task_key: "extract"
          notebook_task:
            notebook_path: "/Workspace/etl/transform"
          environment_key: "default"

        - task_key: "load"
          depends_on:
            - task_key: "transform"
          notebook_task:
            notebook_path: "/Workspace/etl/load"
          environment_key: "default"

      environments:
        - environment_key: "default"
          spec:
            client: "1"
            dependencies:
              - "pandas>=2.0"
              - "requests"
              - "great-expectations"

      schedule:
        quartz_cron_expression: "0 0 6 * * ?"  # 매일 오전 6시
        timezone_id: "Asia/Seoul"

# SDP 파이프라인 노트북
import dlt

@dlt.table(comment="원본 주문 데이터")
def bronze_orders():
    return spark.readStream.format("cloudFiles") \
        .option("cloudFiles.format", "json") \
        .load("/Volumes/production/raw/orders/")

@dlt.table(comment="정제된 주문 데이터")
@dlt.expect_or_drop("valid_amount", "total_amount > 0")
def silver_orders():
    return dlt.read_stream("bronze_orders") \
        .filter("status IS NOT NULL")