플랫폼 설계 — 심화

{
  "label": "corporate-office-and-vpn",
  "list_type": "ALLOW",
  "ip_addresses": [
    "203.0.113.0/24",
    "198.51.100.0/24",
    "10.0.0.0/8"
  ]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "AllowDatabricksUsage",
      "Effect": "Allow",
      "Principal": {
        "AWS": "arn:aws:iam::414351767826:root"
      },
      "Action": [
        "kms:Encrypt",
        "kms:Decrypt",
        "kms:ReEncrypt*",
        "kms:GenerateDataKey*",
        "kms:DescribeKey"
      ],
      "Resource": "*"
    },
    {
      "Sid": "AllowDatabricksGrant",
      "Effect": "Allow",
      "Principal": {
        "AWS": "arn:aws:iam::414351767826:root"
      },
      "Action": [
        "kms:CreateGrant",
        "kms:ListGrants",
        "kms:RevokeGrant"
      ],
      "Resource": "*",
      "Condition": {
        "Bool": {
          "kms:GrantIsForAWSResource": "true"
        }
      }
    }
  ]
}

[PrivateLink 아키텍처]

사용자 VPC                           Databricks Control Plane
┌──────────────────┐                ┌─────────────────────┐
│                  │                │                     │
│  사용자 브라우저 ──┼── Front-end ──→│  Workspace UI/API   │
│                  │  PrivateLink   │                     │
│                  │                │                     │
│  Compute Plane VPC ─┼── Back-end ───→│  Control Plane      │
│  (클러스터)       │  PrivateLink   │  Services           │
│                  │                │                     │
│  Compute Plane VPC ─┼── SCC Relay ──→│  SCC Relay          │
│                  │  PrivateLink   │  Endpoint           │
└──────────────────┘                └─────────────────────┘

# Workspace 생성 (AWS)
resource "databricks_mws_workspaces" "prod" {
  account_id     = var.databricks_account_id
  workspace_name = "prod-analytics"
  aws_region     = "ap-northeast-2"

  credentials_id           = databricks_mws_credentials.prod.credentials_id
  storage_configuration_id = databricks_mws_storage_configurations.prod.storage_configuration_id
  network_id               = databricks_mws_networks.prod.network_id

  # CMK 설정
  managed_services_customer_managed_key_id = databricks_mws_customer_managed_keys.managed.customer_managed_key_id
  storage_customer_managed_key_id          = databricks_mws_customer_managed_keys.storage.customer_managed_key_id

  custom_tags = {
    Environment = "production"
    CostCenter  = "platform-team"
    ManagedBy   = "terraform"
  }
}

# Unity Catalog 카탈로그 생성
resource "databricks_catalog" "prod_sales" {
  provider = databricks.workspace
  name     = "prod_sales"
  comment  = "프로덕션 영업 데이터 카탈로그"

  properties = {
    "owner_team" = "data-engineering"
    "environment" = "production"
  }
}

# Compute Policy 설정
resource "databricks_cluster_policy" "dev_policy" {
  provider = databricks.workspace
  name     = "dev-standard-policy"

  definition = jsonencode({
    "autotermination_minutes" : {
      "type" : "range",
      "maxValue" : 30,
      "defaultValue" : 10
    },
    "num_workers" : {
      "type" : "range",
      "maxValue" : 8,
      "defaultValue" : 2
    },
    "node_type_id" : {
      "type" : "allowlist",
      "values" : ["i3.xlarge", "i3.2xlarge", "m5.xlarge"]
    },
    "spark_version" : {
      "type" : "regex",
      "pattern" : "15\\.[0-9]+\\.x-scala.*"
    }
  })
}

# databricks.yml — 멀티 환경 번들 설정
bundle:
  name: sales-pipeline

workspace:
  host: https://prod.cloud.databricks.com

targets:
  dev:
    workspace:
      host: https://dev.cloud.databricks.com
    default: true
    resources:
      jobs:
        daily_etl:
          job_clusters:
            - job_cluster_key: etl_cluster
              new_cluster:
                num_workers: 2
                node_type_id: i3.xlarge
                spark_version: 15.4.x-scala2.12

  staging:
    workspace:
      host: https://staging.cloud.databricks.com
    resources:
      jobs:
        daily_etl:
          job_clusters:
            - job_cluster_key: etl_cluster
              new_cluster:
                num_workers: 4
                node_type_id: i3.2xlarge

  prod:
    workspace:
      host: https://prod.cloud.databricks.com
    run_as:
      service_principal_name: svc-prod-etl
    resources:
      jobs:
        daily_etl:
          job_clusters:
            - job_cluster_key: etl_cluster
              new_cluster:
                num_workers: 8
                node_type_id: i3.4xlarge
                autoscale:
                  min_workers: 4
                  max_workers: 16