# Baselinr Configuration Example
# This is a sample configuration for profiling PostgreSQL tables

environment: development

# Source database connection
source:
  type: postgres
  host: localhost
  port: 5433
  database: baselinr
  username: baselinr
  password: baselinr
  schema: public

# Storage configuration (where to save profiling results)
storage:
  connection:
    type: postgres
    host: localhost
    port: 5433
    database: baselinr
    username: baselinr
    password: baselinr
    schema: public
  results_table: baselinr_results
  runs_table: baselinr_runs
  create_tables: true

# Profiling configuration
profiling:
  # Tables to profile
  tables:
    # Explicit table profiling (highest priority)
    # - table: customers
    #   schema: public
    
    # Explicit table profiling
    # - table: products
    #   schema: public
    
    # Example: Pattern-based selection (matches all tables starting with "user_")
    # - pattern: "user_*"
    #   schema: public
    #   # Uncomment to enable pattern matching
    
    # Example: Schema-based selection (all tables in schema)
     - select_schema: true
       schema: public
    #   exclude_patterns:
    #     - "*_temp"
    #     - "*_backup"
    #   # Uncomment to profile all tables in analytics schema
    
    # Example with partition-aware profiling (commented out)
    # - table: orders
    #   schema: public
      # Uncomment to enable partition-aware profiling:
      # partition:
      #   key: order_date              # Partition column
      #   strategy: latest             # Options: latest | recent_n | sample | all
      #   metadata_fallback: true      # Try to infer partition key if not specified
      
      # Uncomment to enable sampling:
      # sampling:
      #   enabled: true
      #   method: random               # Options: random | stratified | topk
      #   fraction: 0.01               # Sample 1% of rows
      #   max_rows: 500000             # Cap at 500k rows
  
  # Maximum distinct values to compute
  max_distinct_values: 1000
  
  # Whether to compute histograms
  compute_histograms: true
  
  # Number of histogram bins
  histogram_bins: 10
  
  # Metrics to compute
  metrics:
    - count
    - null_count
    - null_percent
    - distinct_count
    - distinct_percent
    - min
    - max
    - mean
    - stddev
    - histogram

# Drift detection configuration
drift_detection:
  # Strategy to use: "absolute_threshold", "standard_deviation", "ml_based", "statistical"
  strategy: absolute_threshold
  
  # Parameters for absolute_threshold strategy
  absolute_threshold:
    low_threshold: 5.0      # 5% change triggers low severity
    medium_threshold: 15.0   # 15% change triggers medium severity
    high_threshold: 30.0     # 30% change triggers high severity
  
  # Baseline auto-selection configuration
  # Let the system pick the correct baseline automatically
  baselines:
    strategy: auto  # auto | last_run | moving_average | prior_period | stable_window
    windows:
      moving_average: 7    # Number of runs for moving average
      prior_period: 7      # Days for prior period (1=day, 7=week, 30=month)
      min_runs: 3          # Minimum runs required for auto-selection
  
  # Parameters for standard_deviation strategy (alternative)
  # standard_deviation:
  #   low_threshold: 1.0     # 1 std dev triggers low severity
  #   medium_threshold: 2.0  # 2 std devs trigger medium severity
  #   high_threshold: 3.0    # 3 std devs trigger high severity
  
  # Parameters for ML-based strategy (placeholder for future)
  # ml_based:
  #   model_type: isolation_forest
  #   sensitivity: 0.8
  
  # Parameters for statistical test strategy (advanced drift detection)
  # statistical:
  #   tests:
  #     - ks_test          # Kolmogorov-Smirnov test for distribution comparison
  #     - psi              # Population Stability Index
  #     - z_score          # Z-score test for mean/variance shifts
  #     - chi_square       # Chi-square test for categorical distributions
  #     - entropy          # Entropy change detection
  #     - top_k            # Top-K category stability
  #   sensitivity: medium  # low, medium, or high
  #   test_params:
  #     ks_test:
  #       alpha: 0.05      # Significance level
  #     psi:
  #       buckets: 10      # Number of buckets for distribution
  #       threshold: 0.2   # PSI threshold for drift detection
  #     z_score:
  #       z_threshold: 2.0 # Z-score threshold (standard deviations)
  #     chi_square:
  #       alpha: 0.05      # Significance level
  #     entropy:
  #       entropy_threshold: 0.1  # Threshold for entropy change
  #     top_k:
  #       k: 10                    # Number of top categories to track
  #       similarity_threshold: 0.7  # Similarity threshold for stability
  
  # Type-specific threshold configuration
  # Adjusts drift detection sensitivity based on column data type to reduce false positives
  enable_type_specific_thresholds: true  # Enable type-specific thresholds (default: true)
  
  type_specific_thresholds:
    # Numeric columns: Accept larger drift in mean, but be sensitive to stddev changes
    numeric:
      mean:
        low: 10.0      # 10% change in mean triggers low severity (more lenient)
        medium: 25.0   # 25% change triggers medium severity
        high: 50.0     # 50% change triggers high severity
      stddev:
        low: 3.0       # 3% change in stddev triggers low severity (more sensitive)
        medium: 8.0    # 8% change triggers medium severity
        high: 15.0     # 15% change triggers high severity
      default:          # Default thresholds for other numeric metrics
        low: 5.0
        medium: 15.0
        high: 30.0
    
    # Categorical columns: Focus on cardinality changes, ignore numeric metrics
    categorical:
      distinct_count:   # Cardinality changes are high signal
        low: 2.0       # 2% change in distinct count triggers low severity
        medium: 5.0    # 5% change triggers medium severity
        high: 10.0     # 10% change triggers high severity
      distinct_percent:
        low: 2.0
        medium: 5.0
        high: 10.0
      default:          # Default thresholds for other categorical metrics
        low: 5.0
        medium: 15.0
        high: 30.0
      # Note: mean, stddev, min, max are automatically ignored for categorical columns
    
    # Timestamp columns: Detect freshness and latency distribution spikes
    timestamp:
      default:
        low: 5.0
        medium: 15.0
        high: 30.0
    
    # Boolean columns: Use lower thresholds for proportion changes (binomial test logic)
    boolean:
      default:
        low: 2.0       # 2% change triggers low severity (more sensitive)
        medium: 5.0    # 5% change triggers medium severity
        high: 10.0     # 10% change triggers high severity
      # Note: mean, stddev, min, max, histogram are automatically ignored for boolean columns

# Monitoring configuration (Prometheus metrics)
monitoring:
  enable_metrics: true   # Set to true to enable Prometheus metrics
  port: 9753             # Port for metrics HTTP server
  keep_alive: false       # Keep server running after profiling (default: true)

# Retry and recovery configuration
retry:
  enabled: true           # Enable retry logic for transient warehouse errors
  retries: 3              # Maximum number of retry attempts (0-10)
  backoff_strategy: exponential  # Options: exponential | fixed
  min_backoff: 0.5        # Minimum backoff delay in seconds
  max_backoff: 8.0        # Maximum backoff delay in seconds

# Execution and parallelism configuration (OPTIONAL)
# Default: max_workers=1 (sequential execution)
# Enable parallelism by setting max_workers > 1
# Note: Dagster users already benefit from asset-level parallelism
execution:
  max_workers: 1          # Default: 1 (sequential). Set > 1 to enable parallelism
  batch_size: 10          # Tables per batch (default: 10)
  queue_size: 100         # Maximum queue size (default: 100)
  
  # Warehouse-specific worker limits (optional, only used if max_workers > 1)
  # warehouse_limits:
  #   snowflake: 20       # Snowflake can handle more concurrency
  #   postgres: 8         # Postgres moderate concurrency
  #   sqlite: 1           # SQLite single writer (parallelism disabled)

# Event hooks configuration
hooks:
  # Master switch - set to false to disable all hooks
  enabled: true
  
  hooks:
    # Example 1: Logging hook (useful for development)
    - type: logging
      enabled: true
      log_level: INFO
    
    # Example 2: SQL event persistence (commented out)
    # - type: sql
    #   enabled: true
    #   table_name: baselinr_events
    #   connection:
    #     type: postgres
    #     host: localhost
    #     port: 5432
    #     database: baselinr
    #     username: baselinr
    #     password: baselinr
    
    # Example 3: Snowflake event persistence (commented out)
    # - type: snowflake
    #   enabled: true
    #   table_name: baselinr_events
    #   connection:
    #     type: snowflake
    #     account: myaccount
    #     database: monitoring
    #     warehouse: compute_wh
    #     username: user
    #     password: pass
    
    # Example 4: Custom hook (commented out)
    # - type: custom
    #   enabled: true
    #   module: my_custom_hooks
    #   class_name: SlackAlertHook
    #   params:
    #     webhook_url: https://hooks.slack.com/services/YOUR/WEBHOOK/URL
    #     channel: "#data-alerts"

