스케줄 관리, SLA 전략, 트리거 실습

from databricks.sdk import WorkspaceClient

w = WorkspaceClient()

# 스케줄 일시 중지
w.jobs.update(
    job_id=12345,
    new_settings={
        "schedule": {
            "quartz_cron_expression": "0 0 2 * * ?",
            "timezone_id": "Asia/Seoul",
            "pause_status": "PAUSED"
        }
    }
)

# 스케줄 재개
w.jobs.update(
    job_id=12345,
    new_settings={
        "schedule": {
            "quartz_cron_expression": "0 0 2 * * ?",
            "timezone_id": "Asia/Seoul",
            "pause_status": "UNPAUSED"
        }
    }
)

# SLA: 매일 오전 8시까지 Gold 테이블 갱신 완료
schedule:
  quartz_cron_expression: "0 0 2 * * ?"  # 새벽 2시 시작 (6시간 버퍼)
  timezone_id: "Asia/Seoul"

# 3시간 초과 시 경고
health:
  rules:
    - metric: "RUN_DURATION_SECONDS"
      op: "GREATER_THAN"
      value: 10800

# 실패 시 즉시 재시도 (최대 2회)
tasks:
  - task_key: "main_etl"
    max_retries: 2
    min_retry_interval_millis: 120000
    timeout_seconds: 14400    # 4시간 타임아웃

# 알림 설정
email_notifications:
  on_failure:
    - "oncall@company.com"
  on_duration_warning_threshold_exceeded:
    - "data-team@company.com"

webhook_notifications:
  on_failure:
    - id: "${var.pagerduty_webhook_id}"

# 매일 새벽 2시 KST 실행, 동시 실행 방지
resources:
  jobs:
    daily_etl:
      name: "daily-sales-etl"
      schedule:
        quartz_cron_expression: "0 0 2 * * ?"
        timezone_id: "Asia/Seoul"
      max_concurrent_runs: 1
      queue:
        enabled: true
      tasks:
        - task_key: "etl_main"
          notebook_task:
            notebook_path: "/Workspace/etl/daily_sales"

# S3에 파일이 도착하면 5분 간격으로 감지
resources:
  jobs:
    file_ingestion:
      name: "vendor-data-ingestion"
      trigger:
        file_arrival:
          url: "s3://data-lake/incoming/vendor-a/"
          min_time_between_triggers_seconds: 300
          wait_after_last_change_seconds: 120
      max_concurrent_runs: 1
      tasks:
        - task_key: "ingest"
          notebook_task:
            notebook_path: "/Workspace/ingest/vendor_a"

# bronze 테이블이 갱신되면 silver 변환 자동 실행
resources:
  jobs:
    silver_transform:
      name: "silver-orders-transform"
      trigger:
        table:
          condition: "ALL_UPDATED"
          table_names:
            - "catalog.bronze.orders"
            - "catalog.bronze.customers"
          min_time_between_triggers_seconds: 600
          wait_after_last_change_seconds: 180
      tasks:
        - task_key: "transform"
          notebook_task:
            notebook_path: "/Workspace/etl/silver_orders"