실전 템플릿과 모니터링

{
    "spark_version": {
        "type": "regex",
        "pattern": "(15|16)\\.[0-9]+\\.x.*",
        "defaultValue": "15.4.x-scala2.12"
    },
    "node_type_id": {
        "type": "allowlist",
        "values": ["m5.xlarge", "m5.2xlarge", "r5.xlarge"],
        "defaultValue": "m5.xlarge"
    },
    "driver_node_type_id": {
        "type": "fixed",
        "value": "m5.xlarge",
        "hidden": true
    },
    "num_workers": {
        "type": "range",
        "minValue": 0,
        "maxValue": 4,
        "defaultValue": 1
    },
    "autoscale.max_workers": {
        "type": "range",
        "minValue": 1,
        "maxValue": 4,
        "defaultValue": 2
    },
    "autotermination_minutes": {
        "type": "range",
        "minValue": 10,
        "maxValue": 60,
        "defaultValue": 30
    },
    "aws_attributes.availability": {
        "type": "fixed",
        "value": "SPOT_WITH_FALLBACK",
        "hidden": true
    },
    "aws_attributes.first_on_demand": {
        "type": "fixed",
        "value": 1,
        "hidden": true
    },
    "custom_tags.environment": {
        "type": "fixed",
        "value": "development",
        "hidden": true
    },
    "custom_tags.cost_center": {
        "type": "regex",
        "pattern": "CC-[0-9]{4}"
    }
}

{
    "spark_version": {
        "type": "fixed",
        "value": "15.4.x-photon-scala2.12",
        "hidden": true
    },
    "node_type_id": {
        "type": "allowlist",
        "values": ["m5.2xlarge", "r5.2xlarge", "i3.2xlarge"],
        "defaultValue": "m5.2xlarge"
    },
    "autoscale.min_workers": {
        "type": "range",
        "minValue": 2,
        "maxValue": 4,
        "defaultValue": 2
    },
    "autoscale.max_workers": {
        "type": "range",
        "minValue": 4,
        "maxValue": 20,
        "defaultValue": 8
    },
    "autotermination_minutes": {
        "type": "fixed",
        "value": 15,
        "hidden": true
    },
    "aws_attributes.availability": {
        "type": "fixed",
        "value": "SPOT_WITH_FALLBACK",
        "hidden": true
    },
    "aws_attributes.first_on_demand": {
        "type": "fixed",
        "value": 1,
        "hidden": true
    },
    "runtime_engine": {
        "type": "fixed",
        "value": "PHOTON",
        "hidden": true
    },
    "custom_tags.environment": {
        "type": "fixed",
        "value": "production",
        "hidden": true
    },
    "custom_tags.team": {
        "type": "regex",
        "pattern": "[a-z]+-[a-z]+"
    },
    "custom_tags.cost_center": {
        "type": "regex",
        "pattern": "CC-[0-9]{4}"
    },
    "custom_tags.job_name": {
        "type": "unlimited"
    }
}

{
    "spark_version": {
        "type": "regex",
        "pattern": "15\\.[0-9]+\\.x-gpu-ml-scala2\\.12",
        "defaultValue": "15.4.x-gpu-ml-scala2.12"
    },
    "node_type_id": {
        "type": "allowlist",
        "values": ["g5.xlarge", "g5.2xlarge", "g5.4xlarge"],
        "defaultValue": "g5.xlarge"
    },
    "driver_node_type_id": {
        "type": "fixed",
        "value": "m5.xlarge",
        "hidden": true
    },
    "num_workers": {
        "type": "range",
        "minValue": 0,
        "maxValue": 4,
        "defaultValue": 0
    },
    "autotermination_minutes": {
        "type": "range",
        "minValue": 10,
        "maxValue": 120,
        "defaultValue": 60
    },
    "aws_attributes.availability": {
        "type": "fixed",
        "value": "SPOT_WITH_FALLBACK",
        "hidden": true
    },
    "custom_tags.environment": {
        "type": "fixed",
        "value": "ml-training",
        "hidden": true
    },
    "custom_tags.experiment_name": {
        "type": "unlimited"
    }
}

-- 정책 없이 생성된 클러스터 탐지
SELECT
    cluster_id,
    cluster_name,
    creator_user_name,
    cluster_source,
    node_type_id,
    num_workers,
    autotermination_minutes,
    start_time
FROM system.compute.clusters
WHERE policy_id IS NULL
    AND cluster_source != 'JOB'
    AND start_time > CURRENT_DATE() - INTERVAL 7 DAYS
ORDER BY start_time DESC;

-- 시간당 비용이 높은 클러스터 Top 10
SELECT
    cluster_name,
    creator_user_name,
    node_type_id,
    num_workers,
    ROUND(
        TIMESTAMPDIFF(HOUR, start_time, COALESCE(end_time, CURRENT_TIMESTAMP()))
    ) AS running_hours,
    custom_tags:cost_center AS cost_center
FROM system.compute.clusters
WHERE start_time > CURRENT_DATE() - INTERVAL 30 DAYS
    AND node_type_id LIKE '%xlarge%'
ORDER BY running_hours DESC
LIMIT 10;

-- 정책 없이 클러스터를 자주 만드는 사용자
SELECT
    creator_user_name,
    COUNT(*) AS clusters_without_policy
FROM system.compute.clusters
WHERE policy_id IS NULL
    AND cluster_source = 'UI'
    AND start_time > CURRENT_DATE() - INTERVAL 30 DAYS
GROUP BY creator_user_name
HAVING COUNT(*) >= 3
ORDER BY clusters_without_policy DESC;