서버리스 컴퓨팅 보안, 비용, 마이그레이션

-- 서버리스 vs 클래식 비용 비교 쿼리
SELECT
    sku_name,
    billing_origin_product,
    SUM(usage_quantity) AS total_dbus,
    COUNT(DISTINCT usage_date) AS active_days,
    ROUND(SUM(usage_quantity) / COUNT(DISTINCT usage_date), 1) AS avg_daily_dbus
FROM system.billing.usage
WHERE usage_date >= CURRENT_DATE() - INTERVAL 30 DAY
  AND billing_origin_product IN ('JOBS_COMPUTE', 'JOBS_SERVERLESS',
                                  'SQL_COMPUTE', 'SQL_SERVERLESS',
                                  'ALL_PURPOSE_COMPUTE', 'INTERACTIVE_SERVERLESS')
GROUP BY sku_name, billing_origin_product
ORDER BY total_dbus DESC;

-- 현재 클러스터 사용 현황 분석
SELECT
    cluster_name,
    cluster_source,
    COUNT(*) AS job_count,
    SUM(usage_quantity) AS total_dbus
FROM system.billing.usage u
JOIN system.compute.clusters c ON u.usage_metadata.cluster_id = c.cluster_id
WHERE usage_date >= CURRENT_DATE() - INTERVAL 30 DAY
GROUP BY cluster_name, cluster_source
ORDER BY total_dbus DESC;

# Before: Customer-Managed 클러스터 사용
resources:
  jobs:
    my_job:
      tasks:
        - task_key: "etl"
          existing_cluster_id: "0123-456789-abcdef"
          notebook_task:
            notebook_path: "/Workspace/etl/pipeline"

# After: 서버리스로 전환
resources:
  jobs:
    my_job:
      tasks:
        - task_key: "etl"
          # existing_cluster_id 제거
          environment_key: "default"
          notebook_task:
            notebook_path: "/Workspace/etl/pipeline"
      environments:
        - environment_key: "default"
          spec:
            client: "1"
            dependencies:
              - "pandas>=2.0"

-- 마이그레이션 전후 비교
SELECT
    CASE
        WHEN billing_origin_product LIKE '%SERVERLESS%' THEN 'Serverless'
        ELSE 'Classic'
    END AS compute_type,
    SUM(usage_quantity) AS total_dbus,
    COUNT(DISTINCT usage_date) AS active_days
FROM system.billing.usage
WHERE usage_date >= CURRENT_DATE() - INTERVAL 14 DAY
  AND billing_origin_product IN ('JOBS_COMPUTE', 'JOBS_SERVERLESS')
GROUP BY 1;