# =============================================================================
# Prometheus Configuration
# Generated by: dc-overview setup
# =============================================================================

global:
  scrape_interval: 15s
  evaluation_interval: 15s

# Recording rules for efficient queries
rule_files:
  - /etc/prometheus/recording_rules.yml

scrape_configs:
  # Prometheus self-monitoring
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # DC Overview application metrics
  - job_name: 'dc-overview'
    static_configs:
      - targets: ['dc-overview:{{ dc_port | default(5001) }}']
    metrics_path: /metrics

{% if ipmi_enabled %}
  # IPMI Monitor metrics (if installed)
  - job_name: 'ipmi-monitor'
    static_configs:
      - targets: ['ipmi-monitor:5000']
    metrics_path: /metrics
{% endif %}

{% if vast_enabled %}
  # Vast.ai exporter (earnings and reliability)
  - job_name: 'vastai'
    scrape_interval: 60s
    static_configs:
      - targets: ['vastai-exporter:8622']
{% endif %}

{% if runpod_enabled %}
  # RunPod exporter (earnings and reliability)
  - job_name: 'runpod'
    scrape_interval: 60s
    static_configs:
      - targets: ['runpod-exporter:8623']
{% endif %}

  # ==========================================================================
  # GPU Worker Exporters - File-based service discovery
  # Targets are auto-updated by dc-overview when servers are added/removed
  # ==========================================================================
  
  # Node Exporter (CPU, RAM, disk)
  - job_name: 'node-exporter'
    file_sd_configs:
      - files:
          - /etc/prometheus/targets/dc_targets.json
        refresh_interval: 30s
    relabel_configs:
      - source_labels: [job]
        regex: node-exporter
        action: keep

  # DC Exporter (VRAM temp, hotspot temp, AER errors)
  - job_name: 'dc-exporter'
    file_sd_configs:
      - files:
          - /etc/prometheus/targets/dc_targets.json
        refresh_interval: 30s
    relabel_configs:
      - source_labels: [job]
        regex: dc-exporter
        action: keep

  # DCGM Exporter (NVIDIA official metrics)
  - job_name: 'dcgm-exporter'
    file_sd_configs:
      - files:
          - /etc/prometheus/targets/dc_targets.json
        refresh_interval: 30s
    relabel_configs:
      - source_labels: [job]
        regex: dcgm-exporter
        action: keep

{% if static_targets %}
  # ==========================================================================
  # Static targets (legacy - use file_sd instead)
  # ==========================================================================
  - job_name: 'gpu-workers'
    static_configs:
{% for server in static_targets %}
      - targets: ['{{ server.ip }}:9100', '{{ server.ip }}:9835']
        labels:
          instance: '{{ server.name }}'
{% endfor %}
{% endif %}
