DABs 기초: 개념과 리소스 유형

my-project/
├── databricks.yml          # 메인 설정 (필수)
├── resources/
│   ├── jobs.yml            # Job 정의
│   ├── pipelines.yml       # DLT Pipeline 정의
│   └── dashboards.yml      # 대시보드 정의
├── src/
│   ├── notebooks/          # 노트북 소스
│   ├── pipelines/          # DLT 파이프라인 소스
│   └── python/             # Python 모듈
└── tests/                  # 테스트 코드

variables:
  # 1. 기본값 지정
  catalog:
    description: "Unity Catalog 카탈로그 이름"
    default: "dev_catalog"

  # 2. 런타임에 CLI 인자로 전달 (기본값 없음)
  warehouse_id:
    description: "SQL Warehouse ID"

# CLI에서 변수 오버라이드
databricks bundle deploy -t prod --var="catalog=prod_catalog"

# 대화형 템플릿 선택 (권장)
databricks bundle init

# 특정 템플릿 직접 지정
databricks bundle init default-python
databricks bundle init default-sql

# GitHub 저장소의 커스텀 템플릿 사용
databricks bundle init https://github.com/my-org/my-dab-template

Template to use [default-python]:
> default-python
> default-sql
> dbt-sql
> mlops-stacks

Project name: my-etl-pipeline
Include a stub (sample) notebook [yes]: yes

# resources/jobs.yml
resources:
  jobs:
    daily_etl:
      name: "daily-sales-etl-${bundle.target}"
      schedule:
        quartz_cron_expression: "0 0 2 * * ?"  # 매일 새벽 2시 KST
        timezone_id: "Asia/Seoul"
      email_notifications:
        on_failure:
          - "data-team@company.com"

      tasks:
        - task_key: "ingest"
          pipeline_task:
            pipeline_id: "${resources.pipelines.medallion.id}"

        - task_key: "validate"
          depends_on:
            - task_key: "ingest"
          sql_task:
            query:
              query_text: >
                SELECT COUNT(*) AS cnt
                FROM ${var.catalog}.ecommerce.silver_orders
                WHERE order_date = CURRENT_DATE() - INTERVAL 1 DAY
            warehouse_id: "${var.warehouse_id}"

        - task_key: "notify"
          depends_on:
            - task_key: "validate"
          notebook_task:
            notebook_path: "src/notebooks/send_report.py"
          environment_key: "default"

      environments:
        - environment_key: "default"
          spec:
            client: "1"
            dependencies:
              - "requests>=2.28"
              - "pandas>=2.0"

# resources/pipelines.yml
resources:
  pipelines:
    medallion:
      name: "medallion-pipeline-${bundle.target}"
      catalog: "${var.catalog}"
      schema: "ecommerce"
      serverless: true
      continuous: false
      channel: "CURRENT"
      libraries:
        - notebook:
            path: "src/pipelines/bronze_layer.sql"
        - notebook:
            path: "src/pipelines/silver_layer.py"
        - notebook:
            path: "src/pipelines/gold_layer.py"
      notifications:
        - on_failure:
            - "data-team@company.com"
      configuration:
        source_path: "/Volumes/${var.catalog}/raw/landing"

# resources/dashboards.yml
resources:
  dashboards:
    sales_overview:
      display_name: "Sales Overview - ${bundle.target}"
      file_path: "src/dashboards/sales_overview.lvdash.json"
      warehouse_id: "${var.warehouse_id}"
      embed_credentials: false

# resources/model_serving.yml
resources:
  model_serving_endpoints:
    fraud_detector:
      name: "fraud-detector-${bundle.target}"
      config:
        served_models:
          - model_name: "${var.catalog}.ml.fraud_model"
            model_version: "1"
            workload_size: "Small"
            scale_to_zero_enabled: true
      rate_limits:
        - calls: 1000
          renewal_period: "minute"

# resources/quality.yml
resources:
  quality_monitors:
    orders_monitor:
      table_name: "${var.catalog}.ecommerce.silver_orders"
      assets_dir: "/Shared/monitors/orders"
      output_schema_name: "${var.catalog}.monitoring"
      inference_log:
        granularities: ["1 day"]
        model_id_col: "model_version"
        prediction_col: "prediction"
        timestamp_col: "scored_at"

  alerts:
    pipeline_failure_alert:
      display_name: "Pipeline Failure - ${bundle.target}"
      condition:
        op: "GREATER_THAN"
        operand:
          column:
            name: "failed_count"
        threshold:
          value:
            double_value: 0
      query_id: "${var.failure_query_id}"
      seconds_to_retrigger: 3600