SDP 파이프라인 설정 상세

# Asset Bundles로 환경별 파이프라인 자동 분리
# bundle.yml
targets:
  dev:
    default: true
    resources:
      pipelines:
        etl:
          name: "etl-pipeline-DEV"
          catalog: "dev_catalog"
          target: "ecommerce"
          development: true     # ← 개발 모드

  prod:
    resources:
      pipelines:
        etl:
          name: "etl-pipeline-PROD"
          catalog: "production"
          target: "ecommerce"
          development: false    # ← 프로덕션 모드

{
    "name": "my-serverless-pipeline",
    "serverless": true,
    "catalog": "catalog",
    "target": "schema"
}

{
    "name": "my-classic-pipeline",
    "clusters": [
        {
            "label": "default",
            "num_workers": 4,
            "node_type_id": "i3.xlarge",
            "spark_conf": {
                "spark.databricks.delta.optimizeWrite.enabled": "true"
            },
            "aws_attributes": {
                "availability": "SPOT_WITH_FALLBACK",
                "first_on_demand": 1
            }
        },
        {
            "label": "maintenance",
            "num_workers": 2,
            "node_type_id": "m5.xlarge"
        }
    ]
}

{
    "clusters": [
        {
            "label": "default",
            "autoscale": {
                "min_workers": 2,
                "max_workers": 10,
                "mode": "ENHANCED"
            }
        }
    ]
}

[Bronze 수집]     ████████████████████  10 nodes
[Silver 변환]     ████████████          6 nodes
[Gold 집계]       ████                  2 nodes

→ 일반 Autoscaling: 전 구간 10 nodes 유지 → 비용 낭비
→ Enhanced: 단계별 최적화 → 약 40% 절감

{
    "name": "my-pipeline",
    "channel": "CURRENT"
}

{
    "name": "sales-pipeline",
    "catalog": "production",
    "target": "sales_data"
}

{
    "name": "ecommerce-etl-pipeline",
    "catalog": "production",
    "target": "ecommerce",
    "serverless": true,
    "channel": "CURRENT",
    "development": false,
    "continuous": false,
    "photon": true,
    "edition": "ADVANCED",
    "configuration": {
        "source_path": "s3://raw-data/ecommerce/",
        "quality_threshold": "0.95",
        "environment": "production"
    },
    "libraries": [
        {
            "notebook": {
                "path": "/Workspace/pipelines/ecommerce/bronze"
            }
        },
        {
            "notebook": {
                "path": "/Workspace/pipelines/ecommerce/silver"
            }
        },
        {
            "notebook": {
                "path": "/Workspace/pipelines/ecommerce/gold"
            }
        }
    ],
    "notifications": [
        {
            "email_recipients": ["data-team@company.com"],
            "alerts": {
                "on_update_failure": true,
                "on_update_success": false,
                "on_flow_failure": true
            }
        }
    ]
}

resources:
  pipelines:
    ecommerce_etl:
      name: "ecommerce-etl-pipeline"
      catalog: "${var.catalog}"
      target: "${var.schema}"
      serverless: true
      channel: "CURRENT"
      development: ${var.is_dev}
      photon: true
      edition: "ADVANCED"

      configuration:
        source_path: "${var.source_path}"
        quality_threshold: "0.95"
        environment: "${bundle.target}"

      libraries:
        - notebook:
            path: "./pipelines/bronze.sql"
        - notebook:
            path: "./pipelines/silver.py"
        - notebook:
            path: "./pipelines/gold.sql"

      notifications:
        - email_recipients:
            - "data-team@company.com"
          alerts:
            on_update_failure: true
            on_flow_failure: true

# Python에서 파이프라인 설정값 읽기
source_path = spark.conf.get("source_path")
quality_threshold = float(spark.conf.get("quality_threshold"))

-- SQL에서 파이프라인 설정값 읽기
SELECT * FROM STREAM read_files(
    '${source_path}',
    format => 'json'
);

월간 서버리스 SDP 비용 ≈
    (일 데이터 처리량 GB × 처리 단가 × 30일)
  + (유지보수 비중 10~15%)
  + (Continuous 시 대기 비용)

예시: 일 100GB 처리, Triggered 모드
  ≈ (100GB × $0.07/GB × 30일) + 15%
  ≈ $210 + $31.5
  ≈ 약 $242/월

-- 서버리스 SDP의 일별 DBU 사용량 추적
SELECT
    DATE(usage_date) AS date,
    sku_name,
    SUM(usage_quantity) AS total_dbus,
    SUM(usage_quantity * list_price) AS estimated_cost_usd
FROM system.billing.usage
WHERE sku_name LIKE '%SERVERLESS%DLT%'
    AND usage_date > CURRENT_DATE() - INTERVAL 30 DAYS
GROUP BY 1, 2
ORDER BY date DESC;