//! Static cost estimation engine.
//!
//! Walks plan steps, looks up table statistics, applies selectivity
//! heuristics, and produces a [`CostEstimate`] with bytes scanned and USD cost.
//!
//! When column-level statistics are available (n_distinct, null_fraction, min/max,
//! MCV lists), uses research-grade selectivity formulas inspired by PostgreSQL's
//! optimizer and the SafeBound/LpBound (SIGMOD 2023/2025) pessimistic bounds.
//! Falls back to fixed heuristics when statistics are absent.

use crate::complexity::cost_profiles::CostProfile;
use crate::complexity::types::*;
use crate::explainer::types::{JoinKind, PlanOperation, PlanStep};
use crate::schema::types::{
    ColumnStatistics, McvEntry, SchemaDefinition, SqlDialect, estimate_column_logical_bytes,
};

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

/// Default row count when table statistics are not available.
const DEFAULT_ROW_COUNT: u64 = 10_000;

/// Default bytes per row when avg_row_bytes is not available.
const DEFAULT_BYTES_PER_ROW: u64 = 200;

/// Fallback byte estimate for unresolved column names in column-level pruning.
const FALLBACK_COLUMN_BYTES: u64 = 50;

/// Tables smaller than this threshold fit in 1-2 micro-partitions and always
/// scan fully, even with partition/clustering filters. 128 MB.
const MICRO_PARTITION_THRESHOLD: u64 = 128 * 1024 * 1024;

/// Tables smaller than this are a single micro-partition; LIMIT cannot reduce
/// their scan size. 16 MB.
const LIMIT_TABLE_THRESHOLD: u64 = 16 * 1024 * 1024;

/// Heuristic rows per micro-partition when partition_count is not available.
const ROWS_PER_MICRO_PARTITION: u64 = 50_000;

/// Minimum micro-partitions scanned in parallel (Snowflake XS warehouse has ~8
/// threads). Even LIMIT 1 reads at least this many micro-partitions.
const MIN_PARALLEL_SCANS: u64 = 8;

/// Floor for join selectivity applied to optimistic byte bounds.
const JOIN_SELECTIVITY_FLOOR: f64 = 0.01;

/// Byte threshold for "large unfiltered scan" recommendation (100 MB).
const LARGE_UNFILTERED_SCAN_THRESHOLD: u64 = 100_000_000;

/// Byte threshold for "consider pre-filtering before window" recommendation (1 GB).
const WINDOW_RECOMMENDATION_THRESHOLD: u64 = 1_000_000_000;

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

/// Check whether a filter string contains a range operator (>, <, >=, <=, BETWEEN).
fn has_range_operator(filter: &str) -> bool {
    let upper = filter.to_uppercase();
    upper.contains(" > ")
        || upper.contains(" < ")
        || upper.contains(" >= ")
        || upper.contains(" <= ")
        || upper.contains("BETWEEN")
}

/// Selectivity heuristic for different predicate types.
#[derive(Debug, Clone, Copy)]
pub enum PredicateSelectivity {
    /// col = value -> 10%
    Equality,
    /// col > value, col < value -> 30%
    Range,
    /// LIKE 'abc%' -> 5%
    PrefixLike,
    /// IN (n values) -> min(n*10%, 100%)
    InList(usize),
    /// IS NULL -> 5%
    IsNull,
    /// No filter -> 100%
    NoFilter,
}

impl PredicateSelectivity {
    /// Get the selectivity factor (0.0 - 1.0).
    pub fn factor(&self) -> f64 {
        match self {
            PredicateSelectivity::Equality => 0.1,
            PredicateSelectivity::Range => 0.3,
            PredicateSelectivity::PrefixLike => 0.05,
            PredicateSelectivity::InList(n) => ((*n as f64) * 0.1).min(1.0),
            PredicateSelectivity::IsNull => 0.05,
            PredicateSelectivity::NoFilter => 1.0,
        }
    }
}

/// Tracks estimation confidence as the cost estimator walks the query plan.
///
/// Starts at High and degrades as problematic patterns are detected.
/// The final confidence is the minimum (worst) across all detected patterns.
struct ConfidenceTracker {
    level: EstimationConfidence,
    factors: Vec<String>,
}

impl ConfidenceTracker {
    fn new() -> Self {
        Self {
            level: EstimationConfidence::High,
            factors: Vec::new(),
        }
    }

    fn degrade(&mut self, level: EstimationConfidence, reason: &str) {
        self.level = self.level.min_confidence(level);
        let reason_str = reason.to_string();
        if !self.factors.contains(&reason_str) {
            self.factors.push(reason_str);
        }
    }

    fn finish(self) -> (EstimationConfidence, Vec<String>) {
        (self.level, self.factors)
    }
}

/// Estimate the cost of a query based on plan steps and schema statistics.
///
/// If table statistics are available, estimates bytes scanned and USD cost.
/// Otherwise returns an estimate with `CostTier::Unknown`.
///
/// When column-level statistics are present, produces confidence intervals
/// (`CostRange`) with optimistic, expected, and pessimistic bounds.
///
/// **Filter association**: DataFusion may place filter predicates in separate
/// `Filter` plan nodes rather than inside `TableScan.filters`. This function
/// collects predicates from both sources and associates `Filter` predicates
/// with their source tables using qualified column references (e.g.
/// `fact_orders.status`).
pub fn estimate_cost(
    steps: &[PlanStep],
    schema: &SchemaDefinition,
    dialect: SqlDialect,
) -> CostEstimate {
    let profile = CostProfile::for_dialect(dialect);
    let mut total_bytes: u64 = 0;
    let mut optimistic_bytes: u64 = 0;
    let mut pessimistic_bytes: u64 = 0;
    let mut table_breakdown = Vec::new();
    let mut warnings = Vec::new();
    let mut any_stats = false;
    let mut any_stats_based_selectivity = false;
    let mut conf_tracker = ConfidenceTracker::new();

    let all = all_steps(steps);

    // Pre-pass: collect standalone Filter predicates and associate them with
    // their source tables. DataFusion often emits Filter nodes separately
    // from TableScan nodes, so we need to merge them.
    let standalone_filters = collect_standalone_filters(&all, schema);

    // Detect metadata-only aggregations (e.g. COUNT(*) without GROUP BY/WHERE).
    // Columnar warehouses answer these from metadata with 0 bytes scanned.
    let is_metadata_agg = is_metadata_only_aggregation(&all);

    // Detect window function presence. Snowflake validation (986 queries) showed:
    // - Ranking functions (ROW_NUMBER, RANK, DENSE_RANK) with PARTITION BY on
    //   some non-partition columns scan ~2x bytes due to sort materialization.
    //   However, ranking with PARTITION BY on storage-partition cols stays 1x.
    //   Since we can't distinguish these cases statically, we default to 1x
    //   (which is accurate for 78/93 window queries, vs 2x which helps 15 but
    //   breaks 14 others — nearly zero net gain).
    // - Aggregate windows (SUM, AVG, COUNT, LAG, LEAD): always 1x scan.
    let _has_window = all
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Window { .. }));

    // Detect LIMIT N for early termination estimation on columnar engines.
    //
    // LIMIT-aware optimization is applied ONLY when ALL of these conditions hold:
    // 1. The table is large enough to span multiple micro-partitions (>= 16 MB)
    // 2. The query has NO filters (selectivity ~1.0) — filtered queries may need
    //    full scans because zone maps cannot always prune effectively
    // 3. The query has NO joins or subqueries — engines must complete joins/semi-joins
    //    before applying LIMIT, so table scans are NOT reduced
    //
    // This is conservative but correct: Snowflake validation (986 queries) showed
    // that LIMIT reduces bytes ONLY for simple unfiltered scans on large tables.
    // Small tables (< 16 MB) fit in 1-2 micro-partitions and always scan fully.
    let query_limit: Option<usize> = all.iter().find_map(|s| {
        if let PlanOperation::Limit { limit, .. } = &s.operation {
            *limit
        } else {
            None
        }
    });

    // Detect operations that prevent LIMIT pushdown to table scans.
    // Sort, Distinct, and Window all require reading ALL rows before producing
    // output, so LIMIT after them doesn't reduce table scan I/O.
    let has_joins = all
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Join { .. }));
    let has_subqueries = all
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Subquery { .. }));
    let has_sort = all
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Sort { .. }));
    let has_distinct = all
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Distinct));
    let has_window_ops = all
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Window { .. }));

    // Walk plan steps and estimate per-table cost
    for step in &all {
        if let PlanOperation::TableScan {
            table,
            filters,
            columns_read,
            ..
        } = &step.operation
        {
            let table_name = table.to_lowercase();
            let table_def = schema.tables.get(&table_name);

            // Merge pushdown filters with standalone Filter predicates
            let mut combined_filters: Vec<String> = filters.clone();
            if let Some(extra) = standalone_filters.get(&table_name) {
                for f in extra {
                    if !combined_filters.iter().any(|cf| cf == f) {
                        combined_filters.push(f.clone());
                    }
                }
            }

            let stats = table_def.and_then(|td| td.statistics.as_ref());

            if let Some(stats) = stats {
                any_stats = true;

                // Metadata-only: COUNT(*) without filters → 0 bytes
                if is_metadata_agg && combined_filters.is_empty() {
                    table_breakdown.push(TableCostBreakdown {
                        table: table_name,
                        bytes_scanned: 0,
                        estimated_rows: 0,
                        selectivity: 0.0,
                        has_statistics: true,
                    });
                    continue;
                }

                let row_count = stats.row_count.unwrap_or(DEFAULT_ROW_COUNT);
                let avg_bytes = stats
                    .avg_row_bytes
                    .map(u64::from)
                    .unwrap_or(DEFAULT_BYTES_PER_ROW);

                // Determine selectivity from filters using column stats when available
                let sel_estimate =
                    estimate_selectivity(&combined_filters, stats, &table_name, schema);
                let selectivity = sel_estimate.expected;

                if sel_estimate.stats_based {
                    any_stats_based_selectivity = true;
                }

                // Confidence rule 1: LIKE filters have very unpredictable selectivity
                // (26% accuracy in Snowflake validation). Zone-map behavior for LIKE
                // is engine-specific and not modelable statically.
                if combined_filters
                    .iter()
                    .any(|f| f.to_uppercase().contains("LIKE"))
                {
                    conf_tracker.degrade(EstimationConfidence::Low, "LIKE filter detected");
                }

                // Confidence rule 7: Range filter without column stats (heuristic only).
                // Without min/max statistics, range selectivity falls back to 30% heuristic
                // which may be far off from reality.
                if !sel_estimate.stats_based
                    && combined_filters.iter().any(|f| has_range_operator(f))
                {
                    conf_tracker.degrade(
                        EstimationConfidence::Medium,
                        "Range filter without column statistics",
                    );
                }

                // Column-level byte estimation: use per-column logical byte sizes
                // for a more accurate column pruning ratio. Columnar engines store
                // data column-by-column, so narrow columns (INT = 4 bytes) cost less
                // to scan than wide columns (VARCHAR(200) = 200 bytes). This replaces
                // the naive count-based ratio (read_cols / total_cols) with a byte-weighted
                // ratio using estimate_column_logical_bytes() for each column's data type.
                let col_ratio = if let Some(td) = table_def {
                    let total_col_bytes: u64 = td
                        .columns
                        .iter()
                        .map(|c| estimate_column_logical_bytes(&c.data_type, c.statistics.as_ref()))
                        .sum();
                    if total_col_bytes > 0 && !columns_read.is_empty() {
                        let read_col_bytes: u64 = columns_read
                            .iter()
                            .map(|cr| {
                                let cr_lower = cr.to_lowercase();
                                td.columns
                                    .iter()
                                    .find(|c| c.name.to_lowercase() == cr_lower)
                                    .map(|c| {
                                        estimate_column_logical_bytes(
                                            &c.data_type,
                                            c.statistics.as_ref(),
                                        )
                                    })
                                    .unwrap_or(FALLBACK_COLUMN_BYTES) // fallback for unresolved column names
                            })
                            .sum();
                        (read_col_bytes as f64 / total_col_bytes as f64).min(1.0)
                    } else {
                        1.0 // reading all columns or no column info
                    }
                } else {
                    // No table definition: fall back to count-based ratio
                    let total_cols = 1usize;
                    let read_cols = if columns_read.is_empty() {
                        total_cols
                    } else {
                        columns_read.len()
                    };
                    read_cols as f64 / total_cols as f64
                };

                let estimated_rows = (row_count as f64 * selectivity) as u64;
                let mut bytes = (estimated_rows as f64 * avg_bytes as f64 * col_ratio) as u64;

                // Columnar storage floor: columnar engines (Snowflake, BigQuery, etc.)
                // scan entire micro-partitions regardless of row-level filter selectivity.
                // Non-partition filters (equality, range, LIKE on non-partition columns)
                // don't reduce bytes_scanned — they only reduce output rows.
                // Partition filters DO reduce bytes by pruning micro-partitions,
                // but only when the table is large enough to span multiple micro-partitions.
                //
                // Column pruning: columnar engines store each column separately,
                // so scanning a subset of columns reads proportionally fewer bytes.
                // referenced_table_bytes = table_total_bytes * col_ratio captures this.
                //
                // LIMIT-aware estimation: When LIMIT N is present, columnar engines
                // can stop scanning after finding enough rows. Snowflake validation
                // showed LIMIT 1000 on a 2M-row table scans ~1 micro-partition (507KB)
                // instead of the full table (38.9MB). We estimate the fraction of
                // the table needed to satisfy the LIMIT and apply a floor of one
                // micro-partition worth of data.
                let table_total_bytes = row_count * avg_bytes;
                let referenced_table_bytes = (table_total_bytes as f64 * col_ratio) as u64;
                if profile.is_columnar {
                    let has_active_partition_filter = stats.partitioned
                        && !stats.partition_columns.is_empty()
                        && stats.partition_columns.iter().any(|pc| {
                            let pc_lower = pc.to_lowercase();
                            combined_filters.iter().any(|f| {
                                f.to_lowercase()
                                    .split(|c: char| !c.is_alphanumeric() && c != '_')
                                    .any(|token| token == pc_lower)
                            })
                        });

                    // Clustering key awareness: filtering on a clustering key column
                    // enables effective zone-map pruning (data is physically sorted),
                    // similar to partition filters. Treat clustering key filters as
                    // equivalent to partition filters for floor bypass decisions.
                    let has_clustering_filter = table_def
                        .map(|td| {
                            !td.clustering_keys.is_empty()
                                && td.clustering_keys.iter().any(|ck| {
                                    let ck_lower = ck.to_lowercase();
                                    combined_filters.iter().any(|f| {
                                        f.to_lowercase()
                                            .split(|c: char| !c.is_alphanumeric() && c != '_')
                                            .any(|token| token == ck_lower)
                                    })
                                })
                        })
                        .unwrap_or(false);

                    let is_small_table = table_total_bytes < MICRO_PARTITION_THRESHOLD;
                    let has_pruning_filter = has_active_partition_filter || has_clustering_filter;

                    // Confidence rule 3: Range filter on non-partition/clustering column
                    // in columnar engine with stats. Zone-map pruning is unpredictable for
                    // arbitrary range predicates (71% accuracy in Snowflake validation).
                    if !has_pruning_filter && sel_estimate.stats_based && !is_small_table {
                        let has_range_no_like = combined_filters
                            .iter()
                            .any(|f| has_range_operator(f) && !f.to_uppercase().contains("LIKE"));
                        if has_range_no_like {
                            conf_tracker.degrade(
                                EstimationConfidence::Low,
                                "Range filter on non-partition/clustering column (columnar)",
                            );
                        }
                    }

                    // Step 1: Apply columnar floor FIRST.
                    // Small tables (< 128 MB) fit in 1-2 micro-partitions and
                    // always scan fully even with partition/clustering filters.
                    // Large tables without pruning filters: use column-pruned floor
                    // (referenced_table_bytes) because columnar engines only scan
                    // referenced columns.
                    if !has_pruning_filter || is_small_table {
                        if is_small_table {
                            // Small tables: always scan full table (all columns)
                            bytes = bytes.max(table_total_bytes);
                        } else {
                            // Large tables: column pruning reduces the floor
                            bytes = bytes.max(referenced_table_bytes);
                        }
                    }

                    // Step 2: Apply LIMIT reduction ON TOP of the columnar floor.
                    // Only for simple unfiltered scans on large tables without
                    // joins or subqueries. Snowflake validation showed:
                    // - Small tables (<16MB) are 1 micro-partition → LIMIT can't reduce
                    // - Filtered queries need full scans (zone maps don't always prune)
                    // - Joins/subqueries prevent LIMIT pushdown to table scan
                    let is_large_table = table_total_bytes >= LIMIT_TABLE_THRESHOLD;
                    let is_unfiltered = combined_filters.is_empty();
                    let is_simple_query = !has_joins
                        && !has_subqueries
                        && !has_sort
                        && !has_distinct
                        && !has_window_ops;

                    if let Some(limit_n) = query_limit
                        && is_large_table
                        && is_unfiltered
                        && is_simple_query
                    {
                        let rows_needed = (limit_n as f64).min(row_count as f64);
                        let scan_fraction = (rows_needed / row_count as f64).min(1.0);

                        // Estimate micro-partition count: prefer real stats if available,
                        // fall back to heuristic of 50K rows per partition.
                        let num_mp = if let Some(pc) = stats.partition_count {
                            pc.max(1)
                        } else {
                            (row_count / ROWS_PER_MICRO_PARTITION).max(1)
                        };
                        let single_mp_bytes = table_total_bytes / num_mp;

                        // Parallelism floor: columnar engines scan MPs in parallel
                        // across worker threads. An XS warehouse has ~8 threads,
                        // so at minimum ~8 MPs are read even for LIMIT 1.
                        // Snowflake validation: fact_pageviews LIMIT 1000 scans
                        // ~12.8 MB ≈ 13 MPs, consistent with 8-16 parallel scans.
                        let parallel_floor =
                            (single_mp_bytes * MIN_PARALLEL_SCANS).min(table_total_bytes);

                        let limit_bytes =
                            ((table_total_bytes as f64 * scan_fraction) as u64).max(parallel_floor);
                        bytes = bytes.min(limit_bytes);
                    }
                }

                // Confidence interval bytes
                let optimistic_rows = (row_count as f64 * sel_estimate.optimistic) as u64;
                let pessimistic_rows = (row_count as f64 * sel_estimate.pessimistic) as u64;
                let mut opt_bytes = (optimistic_rows as f64 * avg_bytes as f64 * col_ratio) as u64;
                let mut pes_bytes = (pessimistic_rows as f64 * avg_bytes as f64 * col_ratio) as u64;
                if profile.is_columnar {
                    // Reuse pruning filter detection from the expected bytes section
                    let ci_has_partition_filter = stats.partitioned
                        && !stats.partition_columns.is_empty()
                        && stats.partition_columns.iter().any(|pc| {
                            let pc_lower = pc.to_lowercase();
                            combined_filters.iter().any(|f| {
                                f.to_lowercase()
                                    .split(|c: char| !c.is_alphanumeric() && c != '_')
                                    .any(|token| token == pc_lower)
                            })
                        });
                    let ci_has_clustering_filter = table_def
                        .map(|td| {
                            !td.clustering_keys.is_empty()
                                && td.clustering_keys.iter().any(|ck| {
                                    let ck_lower = ck.to_lowercase();
                                    combined_filters.iter().any(|f| {
                                        f.to_lowercase()
                                            .split(|c: char| !c.is_alphanumeric() && c != '_')
                                            .any(|token| token == ck_lower)
                                    })
                                })
                        })
                        .unwrap_or(false);
                    let is_small_table = table_total_bytes < MICRO_PARTITION_THRESHOLD;
                    let ci_has_pruning = ci_has_partition_filter || ci_has_clustering_filter;

                    // Step 1: Columnar floor for confidence intervals
                    // Same logic as expected: small tables get full floor,
                    // large tables get column-pruned floor.
                    if !ci_has_pruning || is_small_table {
                        if is_small_table {
                            opt_bytes = opt_bytes.max(table_total_bytes);
                            pes_bytes = pes_bytes.max(table_total_bytes);
                        } else {
                            opt_bytes = opt_bytes.max(referenced_table_bytes);
                            pes_bytes = pes_bytes.max(referenced_table_bytes);
                        }
                    }

                    // Step 2: LIMIT reduction for confidence intervals
                    // Same conditions as expected bytes: large, unfiltered, simple
                    let ci_is_large = table_total_bytes >= LIMIT_TABLE_THRESHOLD;
                    let ci_is_unfiltered = combined_filters.is_empty();
                    let ci_is_simple = !has_joins
                        && !has_subqueries
                        && !has_sort
                        && !has_distinct
                        && !has_window_ops;

                    if let Some(limit_n) = query_limit
                        && ci_is_large
                        && ci_is_unfiltered
                        && ci_is_simple
                    {
                        let num_mp = if let Some(pc) = stats.partition_count {
                            pc.max(1)
                        } else {
                            (row_count / ROWS_PER_MICRO_PARTITION).max(1)
                        };
                        let single_mp_bytes = table_total_bytes / num_mp;

                        let parallel_floor =
                            (single_mp_bytes * MIN_PARALLEL_SCANS).min(table_total_bytes);

                        let opt_rows = (limit_n as f64).min(row_count as f64);
                        let opt_frac = (opt_rows / row_count as f64).min(1.0);
                        let opt_limit =
                            ((table_total_bytes as f64 * opt_frac) as u64).max(parallel_floor);
                        opt_bytes = opt_bytes.min(opt_limit);

                        let pes_rows = (limit_n as f64).min(row_count as f64);
                        let pes_frac = (pes_rows / row_count as f64).min(1.0);
                        let pes_limit =
                            ((table_total_bytes as f64 * pes_frac) as u64).max(parallel_floor);
                        pes_bytes = pes_bytes.min(pes_limit);
                    }
                }

                // Partition filter warning: selectivity already accounts for
                // partition column filters via n_distinct, so we only emit a
                // warning when a partitioned table is scanned without a filter
                // on its partition column.
                if stats.partitioned && !stats.partition_columns.is_empty() {
                    let has_partition_filter = stats.partition_columns.iter().any(|pc| {
                        let pc_lower = pc.to_lowercase();
                        combined_filters.iter().any(|f| {
                            f.to_lowercase()
                                .split(|c: char| !c.is_alphanumeric() && c != '_')
                                .any(|token| token == pc_lower)
                        })
                    });
                    if !has_partition_filter {
                        warnings.push(format!(
                            "Table '{table_name}' is partitioned on [{}] but no partition filter found",
                            stats.partition_columns.join(", ")
                        ));
                    }
                }

                // Window function: no multiplier. Snowflake validation showed
                // the 2x multiplier was incorrect for most window function types.
                // Ranking functions with PARTITION BY on non-partition columns do
                // scan 2x (15/93 queries), but applying 2x broadly also breaks
                // 14 queries that are currently accurate — nearly zero net gain.
                let final_bytes = bytes;
                let final_opt_bytes = opt_bytes;
                let final_pes_bytes = pes_bytes;

                total_bytes += final_bytes;
                optimistic_bytes += final_opt_bytes;
                pessimistic_bytes += final_pes_bytes;

                table_breakdown.push(TableCostBreakdown {
                    table: table_name,
                    bytes_scanned: final_bytes,
                    estimated_rows,
                    selectivity,
                    has_statistics: true,
                });
            } else {
                // No stats -- use fallback heuristic
                let estimated_rows = DEFAULT_ROW_COUNT;
                let bytes = estimated_rows * DEFAULT_BYTES_PER_ROW;

                let selectivity = if combined_filters.is_empty() {
                    1.0
                } else {
                    // Improve range selectivity for typed columns when table
                    // definition is available but table-level stats are not.
                    // DATE/TIMESTAMP columns typically have ~730-3650 distinct
                    // values (2-10 years of data), giving ~0.03-0.14% per value.
                    // Integer ID/key columns are typically high-cardinality.
                    estimate_no_stats_selectivity(&combined_filters, table_def)
                };

                // Confidence degradation for no-stats branch
                if combined_filters
                    .iter()
                    .any(|f| f.to_uppercase().contains("LIKE"))
                {
                    conf_tracker.degrade(EstimationConfidence::Low, "LIKE filter detected");
                }
                if !combined_filters.is_empty()
                    && combined_filters.iter().any(|f| has_range_operator(f))
                {
                    conf_tracker.degrade(
                        EstimationConfidence::Medium,
                        "Range filter without column statistics",
                    );
                }
                let final_bytes = (bytes as f64 * selectivity) as u64;
                total_bytes += final_bytes;
                optimistic_bytes += final_bytes;
                pessimistic_bytes += final_bytes;

                table_breakdown.push(TableCostBreakdown {
                    table: table_name,
                    bytes_scanned: final_bytes,
                    estimated_rows: (estimated_rows as f64 * selectivity) as u64,
                    selectivity,
                    has_statistics: false,
                });
            }
        }
    }

    // Set operations optimization: Snowflake and other columnar engines can merge
    // UNION ALL branches that scan the same table into a single scan. Detect this
    // pattern and deduplicate bytes. Snowflake validation showed queries like
    // `SELECT id FROM fact_orders UNION ALL SELECT id FROM fact_pageviews` where
    // the engine scans each table once, not twice.
    let has_union = all
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Union { .. }));
    if has_union && profile.is_columnar {
        // Count occurrences of each table in the breakdown
        let mut table_counts: std::collections::HashMap<String, usize> =
            std::collections::HashMap::new();
        for tb in &table_breakdown {
            *table_counts.entry(tb.table.clone()).or_insert(0) += 1;
        }

        // For tables that appear multiple times, subtract the duplicate scans
        for (table_name, count) in &table_counts {
            if *count > 1 {
                // Find the largest scan for this table (keep that one, remove extras)
                let max_scan = table_breakdown
                    .iter()
                    .filter(|tb| &tb.table == table_name)
                    .map(|tb| tb.bytes_scanned)
                    .max()
                    .unwrap_or(0);
                let sum_scan: u64 = table_breakdown
                    .iter()
                    .filter(|tb| &tb.table == table_name)
                    .map(|tb| tb.bytes_scanned)
                    .sum();
                // Remove the extra bytes (keep only the max scan)
                let excess = sum_scan.saturating_sub(max_scan);
                total_bytes = total_bytes.saturating_sub(excess);
                optimistic_bytes = optimistic_bytes.saturating_sub(excess);
                pessimistic_bytes = pessimistic_bytes.saturating_sub(excess);
            }
        }
    }

    // For columnar storage engines (BigQuery, Snowflake, etc.), bytes scanned
    // are determined by per-table scans. Join selectivity affects intermediate/output
    // cardinality but does NOT reduce storage I/O. We use join selectivity only
    // to tighten the optimistic cost_range bound.
    for step in &all {
        if let PlanOperation::Join {
            join_type,
            left,
            right,
            condition,
        } = &step.operation
        {
            if *join_type == JoinKind::Cross {
                continue; // Cross joins handled separately below
            }
            let join_sel = estimate_join_selectivity(
                *join_type,
                left,
                right,
                condition,
                &table_breakdown,
                schema,
            );
            // Only tighten the optimistic bound (lower bytes if join is selective)
            if join_sel < 1.0 {
                optimistic_bytes =
                    (optimistic_bytes as f64 * join_sel.max(JOIN_SELECTIVITY_FLOOR)) as u64;
            }
        }
    }

    // Handle cross joins (multiplicative) using row-count product
    let cross_joins: Vec<_> = all
        .iter()
        .filter(|s| {
            matches!(
                &s.operation,
                PlanOperation::Join {
                    join_type: JoinKind::Cross,
                    ..
                }
            )
        })
        .collect();
    if !cross_joins.is_empty() && !is_metadata_agg {
        warnings.push(format!(
            "{} CROSS JOIN(s) detected -- actual bytes may be much higher",
            cross_joins.len()
        ));
        // For each cross join, compute row-count product from the two joined tables
        let max_avg_bytes = table_breakdown
            .iter()
            .map(|tb| {
                if tb.estimated_rows > 0 {
                    tb.bytes_scanned / tb.estimated_rows
                } else {
                    DEFAULT_BYTES_PER_ROW
                }
            })
            .max()
            .unwrap_or(DEFAULT_BYTES_PER_ROW);
        for cj in &cross_joins {
            if let PlanOperation::Join { left, right, .. } = &cj.operation {
                let left_lower = left.to_lowercase();
                let right_lower = right.to_lowercase();
                let left_rows = table_breakdown
                    .iter()
                    .find(|tb| left_lower.contains(&tb.table))
                    .map(|tb| tb.estimated_rows)
                    .unwrap_or(DEFAULT_ROW_COUNT);
                let right_rows = table_breakdown
                    .iter()
                    .find(|tb| right_lower.contains(&tb.table))
                    .map(|tb| tb.estimated_rows)
                    .unwrap_or(DEFAULT_ROW_COUNT);
                let cross_bytes = (left_rows as u128)
                    .saturating_mul(right_rows as u128)
                    .saturating_mul(max_avg_bytes as u128);
                // Cap to u64::MAX
                let cross_bytes = if cross_bytes > u64::MAX as u128 {
                    u64::MAX
                } else {
                    cross_bytes as u64
                };
                total_bytes = total_bytes.saturating_add(cross_bytes);
                optimistic_bytes = optimistic_bytes.saturating_add(cross_bytes);
                pessimistic_bytes = pessimistic_bytes.saturating_add(cross_bytes);
            }
        }
    }

    // Confidence rule 2: Semi/anti joins (EXISTS/NOT EXISTS subqueries) have
    // 0% accuracy in Snowflake validation because the engine may scan or skip
    // the inner table in ways not predictable by static analysis.
    let has_semi_anti = all.iter().any(|s| {
        matches!(
            &s.operation,
            PlanOperation::Join {
                join_type: JoinKind::LeftSemi
                    | JoinKind::LeftAnti
                    | JoinKind::RightSemi
                    | JoinKind::RightAnti,
                ..
            }
        )
    });
    if has_semi_anti {
        conf_tracker.degrade(
            EstimationConfidence::Low,
            "Semi/anti join (EXISTS/NOT EXISTS)",
        );
    }

    // Confidence rule 4: 3+ table joins compound cardinality estimation errors
    let join_count = all
        .iter()
        .filter(|s| matches!(&s.operation, PlanOperation::Join { .. }))
        .count();
    if join_count >= 3 {
        conf_tracker.degrade(
            EstimationConfidence::Medium,
            format!("{join_count} table joins").as_str(),
        );
    }

    // Confidence rule 5: Cross joins have multiplicative cost uncertainty
    if !cross_joins.is_empty() && !is_metadata_agg {
        conf_tracker.degrade(EstimationConfidence::Medium, "Cross join detected");
    }

    let cost_usd = profile.calculate_cost(total_bytes);
    let cost_tier = if total_bytes == 0 && !any_stats {
        CostTier::Unknown
    } else {
        CostTier::from_cost(cost_usd)
    };

    // Confidence rule 6: No table statistics means all selectivity is heuristic
    if cost_tier == CostTier::Unknown {
        conf_tracker.degrade(
            EstimationConfidence::Medium,
            "No table statistics available",
        );
    }

    // Build cost range when stats-based selectivity was used
    let cost_range = if any_stats_based_selectivity {
        let low_cost = profile.calculate_cost(optimistic_bytes);
        let high_cost = profile.calculate_cost(pessimistic_bytes);
        Some(CostRange {
            low: low_cost,
            expected: cost_usd,
            high: high_cost,
        })
    } else {
        None
    };

    // Generate actionable cost reduction recommendations
    let recommendations =
        generate_recommendations(&all, &table_breakdown, &warnings, schema, total_bytes);

    let (confidence, confidence_factors) = conf_tracker.finish();

    CostEstimate {
        bytes_scanned: total_bytes,
        cost_usd,
        cost_tier,
        table_breakdown,
        warnings,
        disclaimer: profile.disclaimer(),
        cost_range,
        recommendations,
        confidence,
        confidence_factors,
        estimation_method: None,
        time_based_cost: None,
    }
}

// ---------------------------------------------------------------------------
// Cost reduction recommendations
// ---------------------------------------------------------------------------

/// Generate actionable recommendations for reducing query cost.
///
/// Analyzes the query plan to identify optimization opportunities such as
/// missing partition filters, unnecessary full table scans, and SELECT *.
fn generate_recommendations(
    steps: &[&PlanStep],
    table_breakdown: &[TableCostBreakdown],
    warnings: &[String],
    schema: &SchemaDefinition,
    total_bytes: u64,
) -> Vec<CostRecommendation> {
    let mut recs = Vec::new();

    // 1. Missing partition filter (highest impact)
    for warning in warnings {
        if warning.contains("partitioned on") && warning.contains("no partition filter") {
            // Extract table name and partition columns from warning
            if let Some(table_name) = warning
                .strip_prefix("Table '")
                .and_then(|s| s.split('\'').next())
            {
                let partition_cols = schema
                    .tables
                    .get(table_name)
                    .and_then(|td| td.statistics.as_ref())
                    .map(|s| s.partition_columns.join(", "))
                    .unwrap_or_default();

                let n_distinct = schema.tables.get(table_name).and_then(|td| {
                    let stats = td.statistics.as_ref()?;
                    let pc = stats.partition_columns.first()?;
                    td.column_stats(&pc.to_lowercase())
                        .and_then(|cs| cs.n_distinct)
                });

                let savings = if let Some(nd) = n_distinct {
                    let nd = if nd < 0.0 { nd.abs() * 1000.0 } else { nd };
                    if nd > 1.0 { 1.0 - (1.0 / nd) } else { 0.9 }
                } else {
                    0.9
                };

                let detail = if let Some(nd) = n_distinct {
                    let nd = if nd < 0.0 { nd.abs() * 1000.0 } else { nd } as u64;
                    format!(
                        "Add WHERE {partition_cols} = <value> to enable partition pruning. \
                         The partition column has ~{nd} distinct values, so filtering \
                         can reduce scan by ~{nd}x."
                    )
                } else {
                    format!(
                        "Add WHERE {partition_cols} = <value> to enable partition pruning \
                         and reduce bytes scanned by ~90%."
                    )
                };

                recs.push(CostRecommendation {
                    action: "Add partition filter".to_string(),
                    detail,
                    estimated_savings_pct: savings,
                    priority: RecommendationPriority::High,
                });
            }
        }
    }

    // 2. SELECT * detection (medium impact)
    for step in steps {
        if let PlanOperation::TableScan {
            table,
            columns_read,
            ..
        } = &step.operation
        {
            let table_lower = table.to_lowercase();
            let table_def = schema.tables.get(&table_lower);
            if let Some(td) = table_def {
                let total_cols = td.columns.len();
                if !columns_read.is_empty() && columns_read.len() == total_cols && total_cols > 3 {
                    let savings = 1.0 - (3.0 / total_cols as f64).min(0.8);
                    recs.push(CostRecommendation {
                        action: "Select specific columns".to_string(),
                        detail: format!(
                            "Table '{table_lower}' has {total_cols} columns but the query reads all of them. \
                             Select only needed columns to reduce data transfer in columnar stores."
                        ),
                        estimated_savings_pct: savings,
                        priority: RecommendationPriority::Medium,
                    });
                }
            }
        }
    }

    // 3. Cross join warning (high impact)
    let cross_join_count = steps
        .iter()
        .filter(|s| {
            matches!(
                &s.operation,
                PlanOperation::Join {
                    join_type: crate::explainer::types::JoinKind::Cross,
                    ..
                }
            )
        })
        .count();
    if cross_join_count > 0 {
        recs.push(CostRecommendation {
            action: "Eliminate CROSS JOIN".to_string(),
            detail: format!(
                "{cross_join_count} CROSS JOIN(s) detected. Cross joins produce \
                 a cartesian product (N x M rows), which can cause exponential \
                 cost growth. Replace with an equi-join or add a WHERE clause."
            ),
            estimated_savings_pct: 0.99,
            priority: RecommendationPriority::High,
        });
    }

    // 4. Large unfiltered table scans (medium impact)
    for tb in table_breakdown {
        if tb.selectivity >= 1.0 && tb.bytes_scanned > LARGE_UNFILTERED_SCAN_THRESHOLD {
            let table_def = schema.tables.get(&tb.table);
            let has_partition = table_def
                .and_then(|td| td.statistics.as_ref())
                .map(|s| s.partitioned)
                .unwrap_or(false);

            // Skip if already warned about partition filter
            if has_partition {
                continue;
            }

            recs.push(CostRecommendation {
                action: "Add WHERE clause".to_string(),
                detail: format!(
                    "Table '{}' is scanned without filters ({} estimated). \
                     Adding a WHERE clause with selective predicates can significantly \
                     reduce bytes scanned.",
                    tb.table,
                    format_bytes_human(tb.bytes_scanned)
                ),
                estimated_savings_pct: 0.5,
                priority: RecommendationPriority::Medium,
            });
        }
    }

    // 5. Window function materialization hint
    let has_window = steps
        .iter()
        .any(|s| matches!(&s.operation, PlanOperation::Window { .. }));
    if has_window && total_bytes > WINDOW_RECOMMENDATION_THRESHOLD {
        recs.push(CostRecommendation {
            action: "Consider pre-filtering before window".to_string(),
            detail: "Window functions require scanning the full partition. \
                     Pre-filter rows with a CTE or subquery before applying \
                     the window function to reduce the data volume."
                .to_string(),
            estimated_savings_pct: 0.3,
            priority: RecommendationPriority::Low,
        });
    }

    // Sort by priority (High first) then by savings descending
    recs.sort_by(|a, b| {
        let pri_ord = priority_order(a.priority).cmp(&priority_order(b.priority));
        if pri_ord == std::cmp::Ordering::Equal {
            b.estimated_savings_pct
                .partial_cmp(&a.estimated_savings_pct)
                .unwrap_or(std::cmp::Ordering::Equal)
        } else {
            pri_ord
        }
    });

    recs
}

fn priority_order(p: RecommendationPriority) -> u8 {
    match p {
        RecommendationPriority::High => 0,
        RecommendationPriority::Medium => 1,
        RecommendationPriority::Low => 2,
    }
}

fn format_bytes_human(bytes: u64) -> String {
    if bytes < 1024 {
        format!("{bytes} B")
    } else if bytes < 1024 * 1024 {
        format!("{:.1} KB", bytes as f64 / 1024.0)
    } else if bytes < 1024 * 1024 * 1024 {
        format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0))
    } else {
        format!("{:.2} GB", bytes as f64 / (1024.0 * 1024.0 * 1024.0))
    }
}

// ---------------------------------------------------------------------------
// Filter collection from standalone Filter nodes
// ---------------------------------------------------------------------------

/// Collect filter predicates from standalone `PlanOperation::Filter` nodes and
/// associate them with their source tables.
///
/// DataFusion often emits filter predicates as separate `Filter` plan nodes
/// rather than pushing them into `TableScan.filters`. This function examines
/// each `Filter` predicate and associates it with a table by looking for
/// qualified column references (e.g., `fact_orders.status = Utf8("completed")`)
/// or, when only one table is in the plan, assigns all filters to that table.
fn collect_standalone_filters(
    steps: &[&PlanStep],
    schema: &SchemaDefinition,
) -> std::collections::HashMap<String, Vec<String>> {
    use std::collections::HashMap;

    let mut result: HashMap<String, Vec<String>> = HashMap::new();

    // Collect all table names from TableScan steps
    let table_names: Vec<String> = steps
        .iter()
        .filter_map(|s| {
            if let PlanOperation::TableScan { table, .. } = &s.operation {
                Some(table.to_lowercase())
            } else {
                None
            }
        })
        .collect();

    for step in steps {
        if let PlanOperation::Filter { predicate } = &step.operation {
            // Try to find the table from qualified column references like "table.column"
            let target_table = find_table_in_predicate(predicate, &table_names, schema);

            if let Some(table) = target_table {
                // Clean the predicate: strip table prefix for column-based matching
                let cleaned = clean_predicate(predicate, &table);
                result.entry(table).or_default().push(cleaned);
            } else if table_names.len() == 1 {
                // Single-table query: assign filter to the only table
                let cleaned = clean_predicate(predicate, &table_names[0]);
                result
                    .entry(table_names[0].clone())
                    .or_default()
                    .push(cleaned);
            }
            // For multi-table queries where we can't determine the table, skip
        }
    }

    result
}

/// Find which table a predicate belongs to by looking for qualified column references.
///
/// Examines the predicate for patterns like `table_name.column_name` and matches
/// against known table names. Also checks column names against table definitions.
fn find_table_in_predicate(
    predicate: &str,
    table_names: &[String],
    schema: &SchemaDefinition,
) -> Option<String> {
    let predicate_lower = predicate.to_lowercase();

    // First: check for qualified references like "fact_orders.status"
    for table in table_names {
        let prefix = format!("{table}.");
        if predicate_lower.contains(&prefix) {
            return Some(table.clone());
        }
    }

    // Second: extract unqualified column name and match against table definitions
    let col_name = extract_column_from_predicate(&predicate_lower)?;
    for table in table_names {
        if let Some(td) = schema.tables.get(table)
            && td.columns.iter().any(|c| c.name.to_lowercase() == col_name)
        {
            return Some(table.clone());
        }
    }

    None
}

/// Extract column name from a predicate, stripping table qualifiers and DataFusion formatting.
fn extract_column_from_predicate(predicate: &str) -> Option<String> {
    let trimmed = predicate.trim();
    // Take the first token (before whitespace or operator)
    let first_token = trimmed.split(|c: char| c.is_whitespace()).next()?;

    // Strip table alias: "t.col" -> "col"
    let col = if let Some(pos) = first_token.rfind('.') {
        &first_token[pos + 1..]
    } else {
        first_token
    };

    if col.is_empty() || !col.chars().all(|c| c.is_alphanumeric() || c == '_') {
        return None;
    }

    Some(col.to_string())
}

/// Clean a DataFusion predicate for consumption by the selectivity estimator.
///
/// - Strips table qualifiers: `fact_orders.status = Utf8("completed")` → `status = 'completed'`
/// - Converts DataFusion type wrappers: `Int64(42)` → `42`, `Utf8("completed")` → `'completed'`
fn clean_predicate(predicate: &str, table_name: &str) -> String {
    let mut cleaned = predicate.to_string();

    // Strip table qualifier: "fact_orders.status" → "status"
    let prefix = format!("{table_name}.");
    cleaned = cleaned.replace(&prefix, "");
    // Also handle case-insensitive replacement
    let prefix_lower = prefix.to_lowercase();
    let cleaned_lower = cleaned.to_lowercase();
    if cleaned_lower.contains(&prefix_lower) {
        // Rebuild without the prefix
        let mut result = String::new();
        let mut remaining = cleaned.as_str();
        while let Some(pos) = remaining.to_lowercase().find(&prefix_lower) {
            result.push_str(&remaining[..pos]);
            remaining = &remaining[pos + prefix_lower.len()..];
        }
        result.push_str(remaining);
        cleaned = result;
    }

    // Convert DataFusion type wrappers
    // Utf8("value") or Utf8("value") → 'value'
    let re_utf8 = regex::Regex::new(r#"Utf8\("([^"]*)"\)"#).ok();
    if let Some(re) = re_utf8 {
        cleaned = re.replace_all(&cleaned, "'$1'").to_string();
    }

    // Int64(42) → 42, Int32(42) → 42, Float64(3.14) → 3.14
    let re_num = regex::Regex::new(
        r"(?:Int8|Int16|Int32|Int64|UInt8|UInt16|UInt32|UInt64|Float32|Float64)\(([^)]*)\)",
    )
    .ok();
    if let Some(re) = re_num {
        cleaned = re.replace_all(&cleaned, "$1").to_string();
    }

    // Date32("2024-01-15") → '2024-01-15'
    let re_date = regex::Regex::new(r#"Date32\("([^"]*)"\)"#).ok();
    if let Some(re) = re_date {
        cleaned = re.replace_all(&cleaned, "'$1'").to_string();
    }

    cleaned
}

/// Detect metadata-only aggregations: queries that columnar warehouses can
/// answer from metadata alone (0 bytes scanned).
///
/// Pattern: single table scan → aggregate with COUNT(*) and no GROUP BY.
fn is_metadata_only_aggregation(steps: &[&PlanStep]) -> bool {
    let table_scans: Vec<_> = steps
        .iter()
        .filter(|s| matches!(&s.operation, PlanOperation::TableScan { .. }))
        .collect();
    let aggregates: Vec<_> = steps
        .iter()
        .filter(|s| matches!(&s.operation, PlanOperation::Aggregate { .. }))
        .collect();
    let filters: Vec<_> = steps
        .iter()
        .filter(|s| matches!(&s.operation, PlanOperation::Filter { .. }))
        .collect();

    // Must have at least 1 table scan, exactly 1 aggregate, and 0 standalone filters.
    // Multiple table scans are allowed (e.g. cross join self-join) as long as all
    // scans are unfiltered — the warehouse can still answer COUNT(*) from metadata.
    if table_scans.is_empty() || aggregates.len() != 1 || !filters.is_empty() {
        return false;
    }

    // Check that ALL TableScans have no pushed-down filters
    for ts in &table_scans {
        if let PlanOperation::TableScan { filters, .. } = &ts.operation
            && !filters.is_empty()
        {
            return false;
        }
    }

    // Check aggregate is COUNT-only with no GROUP BY
    if let PlanOperation::Aggregate {
        group_by,
        functions,
    } = &aggregates[0].operation
    {
        if !group_by.is_empty() {
            return false;
        }
        // Functions that can be answered from metadata alone:
        // - COUNT(*), COUNT(col) — but NOT COUNT(DISTINCT) which requires full scan
        // - MIN(col), MAX(col) — answered from zone maps on columnar engines
        // Snowflake validation showed COUNT(DISTINCT) scans full data (8.8MB+).
        return functions.iter().all(|f| {
            let upper = f.to_uppercase();
            let is_count = upper.starts_with("COUNT") && !upper.contains("DISTINCT");
            let is_min_max = upper.starts_with("MIN") || upper.starts_with("MAX");
            is_count || is_min_max
        });
    }

    false
}

// ---------------------------------------------------------------------------
// Selectivity estimation
// ---------------------------------------------------------------------------

/// Estimate selectivity from filter predicates, using column statistics when available.
///
/// Returns a `SelectivityEstimate` with expected, pessimistic, and optimistic bounds.
/// When column stats are present, uses PostgreSQL-style formulas (1/n_distinct for
/// equality, range proportion for comparisons, null_fraction for IS NULL).
/// Falls back to fixed heuristics when no column statistics exist.
fn estimate_selectivity(
    filters: &[String],
    stats: &crate::schema::types::TableStatistics,
    table_name: &str,
    schema: &SchemaDefinition,
) -> SelectivityEstimate {
    if filters.is_empty() {
        return SelectivityEstimate {
            expected: 1.0,
            pessimistic: 1.0,
            optimistic: 1.0,
            stats_based: false,
        };
    }

    let table_def = schema.tables.get(table_name);
    let row_count = stats.row_count.unwrap_or(DEFAULT_ROW_COUNT);

    let mut combined_expected = 1.0f64;
    let mut combined_pessimistic = 1.0f64;
    let mut combined_optimistic = 1.0f64;
    let mut any_stats_used = false;
    let mut is_first = true;

    // Track which correlation group indices we have already seen a filter for
    let mut seen_corr_groups: Vec<usize> = Vec::new();

    for filter in filters {
        let col_name = extract_column_name(filter);

        // Look up column stats if we can identify the column
        let col_stats = col_name
            .as_ref()
            .and_then(|cn| table_def.and_then(|td| td.column_stats(cn)));

        let (sel_exp, sel_pes, sel_opt, used_stats) = if let Some(cs) = col_stats {
            estimate_filter_with_stats(filter, cs, row_count)
        } else {
            estimate_filter_heuristic(filter)
        };

        if used_stats {
            any_stats_used = true;
        }

        // Check if this column is in a correlation group we already have a filter for
        let in_seen_group = col_name.as_ref().is_some_and(|cn| {
            find_correlation_group(cn, &stats.correlated_columns)
                .is_some_and(|gi| seen_corr_groups.contains(&gi))
        });

        if in_seen_group {
            // Correlated column: use max selectivity instead of multiplying.
            // This prevents unrealistically low combined selectivity.
            combined_expected = combined_expected.max(sel_exp);
            combined_pessimistic = combined_pessimistic.max(sel_pes);
            combined_optimistic = combined_optimistic.max(sel_opt);
        } else if is_first {
            combined_expected *= sel_exp;
            combined_pessimistic *= sel_pes;
            combined_optimistic *= sel_opt;
            is_first = false;
        } else {
            // sqrt dampening for subsequent independent filters
            combined_expected *= sel_exp.sqrt();
            combined_pessimistic *= sel_pes.sqrt().max(sel_pes * 0.5);
            combined_optimistic *= sel_opt.sqrt().sqrt();
        }

        // Record the correlation group for this column
        if let Some(cn) = &col_name
            && let Some(gi) = find_correlation_group(cn, &stats.correlated_columns)
            && !seen_corr_groups.contains(&gi)
        {
            seen_corr_groups.push(gi);
        }
    }

    let expected = combined_expected.max(0.001);
    let pessimistic = combined_pessimistic.clamp(0.01, 1.0);
    let optimistic = combined_optimistic.max(0.0001);

    // Enforce ordering invariant: optimistic <= expected <= pessimistic
    let pessimistic = pessimistic.max(expected);
    let optimistic = optimistic.min(expected);

    SelectivityEstimate {
        expected,
        pessimistic,
        optimistic,
        stats_based: any_stats_used,
    }
}

/// Extract the column name from a simple predicate string.
///
/// Handles patterns like "col = value", "col > 100", "col IS NULL",
/// "col IN (1,2,3)", "col LIKE 'abc%'", "col BETWEEN 1 AND 10".
/// Returns the left-hand-side token (column name), stripped of any table alias.
fn extract_column_name(filter: &str) -> Option<String> {
    let trimmed = filter.trim();
    if trimmed.is_empty() {
        return None;
    }

    // Split on whitespace, take the first token as the potential column reference
    let first_token = trimmed.split(|c: char| c.is_whitespace()).next()?;

    // Strip table alias: "t.col" -> "col"
    let col = if let Some(pos) = first_token.rfind('.') {
        &first_token[pos + 1..]
    } else {
        first_token
    };

    // Validate it looks like an identifier (alphanumeric + underscores)
    if col.is_empty() || !col.chars().all(|c| c.is_alphanumeric() || c == '_') {
        return None;
    }

    Some(col.to_lowercase())
}

/// Estimate filter selectivity using column statistics (PostgreSQL-style formulas).
///
/// Returns (expected, pessimistic, optimistic, stats_used).
fn estimate_filter_with_stats(
    filter: &str,
    cs: &ColumnStatistics,
    row_count: u64,
) -> (f64, f64, f64, bool) {
    let upper = filter.to_uppercase();
    let null_factor = 1.0 - cs.null_fraction.unwrap_or(0.0);

    // IS NULL / IS NOT NULL (check before equality since "IS" contains no "=")
    if upper.contains("IS NOT NULL") {
        let nf = 1.0 - cs.null_fraction.unwrap_or(0.05);
        return (nf, 1.0, nf * 0.9, true);
    }
    if upper.contains("IS NULL") {
        let nf = cs.null_fraction.unwrap_or(0.05);
        // When null_fraction is exactly 0.0 (no NULLs), IS NULL returns 0 rows.
        // Use a tiny epsilon so the estimate isn't literally zero (avoids 0.00x ratio).
        if nf == 0.0 {
            return (0.0001, 0.001, 0.0001, true);
        }
        return (nf, (nf * 2.0).min(1.0), (nf * 0.5).max(0.0001), true);
    }

    // Equality: col = value or col == value
    if upper.contains(" = ") || upper.contains(" == ") {
        // Check MCV list first for exact frequency
        if let Some(freq) = find_mcv_frequency(filter, &cs.most_common_values) {
            let sel = freq * null_factor;
            return (sel, (sel * 3.0).min(1.0), (sel * 0.3).max(0.0001), true);
        }
        // Use n_distinct for uniform assumption
        if let Some(nd) = cs.n_distinct {
            let distinct = if nd < 0.0 {
                nd.abs() * row_count as f64
            } else {
                nd
            };
            let sel = (1.0 / distinct.max(1.0)) * null_factor;
            return (sel, (sel * 3.0).min(1.0), (sel * 0.3).max(0.0001), true);
        }
    }

    // NOT IN: complement of IN selectivity
    if upper.contains(" NOT IN (") || upper.contains(" NOT IN(") {
        let count = (filter.matches(',').count() + 1) as f64;
        if let Some(nd) = cs.n_distinct {
            let distinct = if nd < 0.0 {
                nd.abs() * row_count as f64
            } else {
                nd
            };
            let in_sel = (count / distinct.max(1.0)).min(1.0);
            let sel = (1.0 - in_sel) * null_factor;
            return (sel, 1.0, (sel * 0.5).max(0.0001), true);
        }
    }

    // IN list: col IN (v1, v2, v3)
    if upper.contains(" IN (") || upper.contains(" IN(") {
        let count = (filter.matches(',').count() + 1) as f64;
        if let Some(nd) = cs.n_distinct {
            let distinct = if nd < 0.0 {
                nd.abs() * row_count as f64
            } else {
                nd
            };
            let sel = (count / distinct.max(1.0)).min(1.0) * null_factor;
            return (sel, (sel * 2.0).min(1.0), (sel * 0.5).max(0.0001), true);
        }
    }

    // Range: col > value, col < value, col >= value, col <= value, col BETWEEN
    if has_range_operator(filter)
        && cs.min_value.is_some()
        && cs.max_value.is_some()
        && let Some(range_sel) = estimate_range_selectivity(filter, cs)
    {
        let sel = range_sel * null_factor;
        return (sel, (sel * 2.0).min(1.0), (sel * 0.5).max(0.0001), true);
    }

    // LIKE: use n_distinct and pattern type for better estimation
    if upper.contains(" NOT LIKE ") {
        // NOT LIKE: complement of LIKE selectivity
        let like_sel = if let Some(nd) = cs.n_distinct {
            let distinct = if nd < 0.0 {
                nd.abs() * row_count as f64
            } else {
                nd
            };
            // Prefix LIKE matches ~1/sqrt(n_distinct) of rows
            (1.0 / distinct.sqrt()).min(0.5)
        } else {
            0.05
        };
        let sel = (1.0 - like_sel) * null_factor;
        return (sel, 1.0, (sel * 0.8).max(0.5), true);
    }
    if upper.contains(" LIKE ") {
        // Distinguish prefix LIKE ('abc%') from infix LIKE ('%abc%')
        if let Some(rhs) = extract_rhs_value(filter) {
            let is_prefix = rhs.starts_with('%') || rhs.starts_with("'%");
            let is_infix = rhs.contains('%')
                && !rhs.ends_with('%')
                && rhs.chars().filter(|&c| c == '%').count() == 1;

            if let Some(nd) = cs.n_distinct {
                let distinct = if nd < 0.0 {
                    nd.abs() * row_count as f64
                } else {
                    nd
                };
                let sel = if is_prefix || is_infix {
                    // Suffix/infix LIKE: broader match, ~30% heuristic or 1/sqrt(sqrt(nd))
                    (1.0 / distinct.sqrt().sqrt()).clamp(0.05, 0.5) * null_factor
                } else {
                    // Prefix LIKE ('abc%'): narrower match, ~1/sqrt(nd)
                    (1.0 / distinct.sqrt()).clamp(0.001, 0.5) * null_factor
                };
                return (sel, (sel * 3.0).min(1.0), (sel * 0.3).max(0.0001), true);
            }
        }
        // Fallback: no stats
        let sel = PredicateSelectivity::PrefixLike.factor();
        return (sel, sel * 3.0, sel * 0.3, false);
    }

    // No stats matched -- fall back to heuristics
    estimate_filter_heuristic(filter)
}

/// Heuristic-based filter selectivity (no column stats).
///
/// Returns (expected, pessimistic, optimistic, stats_used=false).
fn estimate_filter_heuristic(filter: &str) -> (f64, f64, f64, bool) {
    let upper = filter.to_uppercase();

    // Handle NOT variants first (complement of positive selectivity)
    if upper.contains(" NOT LIKE ") {
        let f = 1.0 - PredicateSelectivity::PrefixLike.factor(); // 0.95
        return (f, 1.0, (f * 0.8).max(0.5), false);
    }
    if upper.contains(" NOT IN (") || upper.contains(" NOT IN(") {
        let count = filter.matches(',').count() + 1;
        let in_f = PredicateSelectivity::InList(count).factor();
        let f = 1.0 - in_f;
        return (f, 1.0, (f * 0.5).max(0.3), false);
    }
    if upper.contains(" != ") || upper.contains(" <> ") {
        let f = 1.0 - PredicateSelectivity::Equality.factor(); // 0.9
        return (f, 1.0, (f * 0.8).max(0.5), false);
    }

    let sel = if upper.contains(" = ") || upper.contains(" == ") {
        PredicateSelectivity::Equality
    } else if has_range_operator(filter) {
        PredicateSelectivity::Range
    } else if upper.contains(" LIKE ") {
        PredicateSelectivity::PrefixLike
    } else if upper.contains(" IN (") || upper.contains(" IN(") {
        let count = filter.matches(',').count() + 1;
        PredicateSelectivity::InList(count)
    } else if upper.contains("IS NULL") || upper.contains("IS NOT NULL") {
        PredicateSelectivity::IsNull
    } else {
        PredicateSelectivity::Range // default heuristic
    };

    let f = sel.factor();
    // For heuristics: pessimistic = 3x, optimistic = 0.3x
    (f, (f * 3.0).min(1.0), (f * 0.3).max(0.0001), false)
}

/// Estimate selectivity for the no-stats path using column type information.
///
/// When a table has column definitions but no table-level statistics,
/// we can make smarter defaults based on column data types:
/// - DATE/TIMESTAMP columns: assume ~730 distinct values (2 years of daily data),
///   so a range filter covering ~6 months is ~0.25 selectivity.
/// - Integer columns named like "id" or "_key": assume high cardinality,
///   so equality is ~0.001 and range is ~0.1.
/// - Other columns: fall back to the standard 0.3 heuristic.
fn estimate_no_stats_selectivity(
    filters: &[String],
    table_def: Option<&crate::schema::types::TableDef>,
) -> f64 {
    if filters.is_empty() {
        return 1.0;
    }

    let td = match table_def {
        Some(td) => td,
        None => return 0.3, // no column info at all
    };

    // Use the first filter for type-aware estimation
    let mut combined_sel = 1.0;
    for filter in filters {
        if !has_range_operator(filter) {
            // Non-range filters: use standard heuristic
            let (expected, _, _, _) = estimate_filter_heuristic(filter);
            combined_sel *= expected;
            continue;
        }

        // Try to extract column name from the filter (left of operator)
        let col_name = extract_filter_column_name(filter);
        let col_type = col_name.as_deref().and_then(|cn| {
            let cn_lower = cn.to_lowercase();
            td.columns
                .iter()
                .find(|c| c.name.to_lowercase() == cn_lower)
                .map(|c| c.data_type.to_uppercase())
        });

        let sel = match col_type.as_deref() {
            Some(t) if is_temporal_type(t) => {
                // DATE/TIMESTAMP: assume ~730 distinct values (2 years daily).
                // A typical range filter covers ~25% of the date range.
                0.25
            }
            Some(_) => {
                // Known column but not temporal: standard range heuristic
                PredicateSelectivity::Range.factor()
            }
            None => PredicateSelectivity::Range.factor(),
        };
        combined_sel *= sel;
    }

    combined_sel.clamp(0.001, 1.0)
}

/// Check if a SQL type string represents a temporal (date/time) type.
fn is_temporal_type(type_str: &str) -> bool {
    type_str.starts_with("DATE")
        || type_str.starts_with("TIMESTAMP")
        || type_str.starts_with("DATETIME")
        || type_str == "TIME"
}

/// Extract the column name from the left side of a filter predicate.
///
/// "order_date >= '2024-01-01'" -> Some("order_date")
/// "t.order_date > 100"        -> Some("order_date")
fn extract_filter_column_name(filter: &str) -> Option<String> {
    let ops = [" >= ", " <= ", " > ", " < ", " BETWEEN "];
    let mut op_start = None;
    for op in &ops {
        if let Some(pos) = filter.to_uppercase().find(op) {
            op_start = Some(pos);
            break;
        }
    }
    let pos = op_start?;
    let lhs = filter[..pos].trim();
    // Handle qualified names like "t.col" or "schema.table.col"
    let col = lhs.rsplit('.').next().unwrap_or(lhs);
    if col.is_empty() {
        None
    } else {
        Some(col.to_string())
    }
}

/// Check if a filter value matches an entry in the MCV (Most Common Values) list.
///
/// Extracts the RHS value from equality predicates and compares against MCV entries.
fn find_mcv_frequency(filter: &str, mcvs: &[McvEntry]) -> Option<f64> {
    if mcvs.is_empty() {
        return None;
    }

    // Extract the value from "col = 'value'" or "col = value"
    let value = extract_rhs_value(filter)?;
    let value_lower = value.to_lowercase();

    mcvs.iter()
        .find(|e| e.value.to_lowercase() == value_lower)
        .map(|e| e.frequency)
}

/// Extract the right-hand-side value from a simple predicate.
///
/// "col = 'completed'" -> "completed"
/// "col = 42"          -> "42"
/// "col > 100"         -> "100"
fn extract_rhs_value(filter: &str) -> Option<String> {
    // Find the operator position
    let ops = [" == ", " = ", " >= ", " <= ", " > ", " < "];
    let mut op_end = None;
    for op in &ops {
        if let Some(pos) = filter.find(op) {
            op_end = Some(pos + op.len());
            break;
        }
    }
    let start = op_end?;
    let rhs = filter[start..].trim();

    // Strip surrounding quotes
    let value = if (rhs.starts_with('\'') && rhs.ends_with('\''))
        || (rhs.starts_with('"') && rhs.ends_with('"'))
    {
        &rhs[1..rhs.len() - 1]
    } else {
        rhs
    };

    if value.is_empty() {
        None
    } else {
        Some(value.to_string())
    }
}

/// Estimate range selectivity using min/max bounds.
///
/// For "col > value": selectivity = (max - value) / (max - min)
/// For "col < value": selectivity = (value - min) / (max - min)
/// For "col >= value": same as > (close enough for estimation)
/// For "col BETWEEN a AND b": selectivity = (b - a) / (max - min)
fn estimate_range_selectivity(filter: &str, cs: &ColumnStatistics) -> Option<f64> {
    let min_str = cs.min_value.as_deref()?;
    let max_str = cs.max_value.as_deref()?;

    let min_val = parse_numeric(min_str)?;
    let max_val = parse_numeric(max_str)?;
    let range = max_val - min_val;
    if range <= 0.0 {
        return None;
    }

    let upper = filter.to_uppercase();

    if upper.contains(" BETWEEN ") {
        // Extract "col BETWEEN a AND b"
        let between_pos = upper.find(" BETWEEN ")?;
        let rest = &filter[between_pos + 9..];
        let and_pos = rest.to_uppercase().find(" AND ")?;
        let a_str = rest[..and_pos].trim();
        let b_str = rest[and_pos + 5..].trim();
        let a = parse_numeric(strip_quotes(a_str))?;
        let b = parse_numeric(strip_quotes(b_str))?;
        let sel = ((b - a) / range).clamp(0.001, 1.0);
        return Some(sel);
    }

    // Extract RHS value
    let value_str = extract_rhs_value(filter)?;
    let value = parse_numeric(&value_str)?;

    if upper.contains(" > ") || upper.contains(" >= ") {
        let sel = ((max_val - value) / range).clamp(0.001, 1.0);
        Some(sel)
    } else if upper.contains(" < ") || upper.contains(" <= ") {
        let sel = ((value - min_val) / range).clamp(0.001, 1.0);
        Some(sel)
    } else {
        None
    }
}

/// Parse a string as a numeric value (f64).
///
/// Handles integers, floats, date strings (YYYY-MM-DD), and timestamp strings
/// (YYYY-MM-DD HH:MM:SS) as days since epoch for ordering.
fn parse_numeric(s: &str) -> Option<f64> {
    let stripped = strip_quotes(s);
    // Try direct f64 parse first
    if let Ok(v) = stripped.parse::<f64>() {
        return Some(v);
    }
    // Try date/timestamp parse: extract YYYY-MM-DD prefix
    // Handles both "2024-01-15" and "2024-01-15 00:00:00" formats
    let date_part = if stripped.len() >= 10 && stripped.chars().nth(4) == Some('-') {
        Some(&stripped[..10])
    } else {
        None
    };
    if let Some(date_str) = date_part {
        let parts: Vec<&str> = date_str.split('-').collect();
        if parts.len() == 3
            && let (Ok(y), Ok(m), Ok(d)) = (
                parts[0].parse::<f64>(),
                parts[1].parse::<f64>(),
                parts[2].parse::<f64>(),
            )
        {
            return Some(y * 365.25 + m * 30.44 + d);
        }
    }
    None
}

/// Strip surrounding single or double quotes from a string.
fn strip_quotes(s: &str) -> &str {
    let s = s.trim();
    if (s.starts_with('\'') && s.ends_with('\'')) || (s.starts_with('"') && s.ends_with('"')) {
        &s[1..s.len() - 1]
    } else {
        s
    }
}

/// Find which correlation group (by index) a column belongs to.
fn find_correlation_group(col_name: &str, groups: &[Vec<String>]) -> Option<usize> {
    let lower = col_name.to_lowercase();
    groups
        .iter()
        .position(|group| group.iter().any(|c| c.to_lowercase() == lower))
}

// ---------------------------------------------------------------------------
// Join cardinality estimation
// ---------------------------------------------------------------------------

/// Estimate join selectivity for non-cross joins.
///
/// Uses foreign key information and n_distinct statistics when available:
/// - FK join: selectivity ~ 1.0 (no fan-out)
/// - Both sides have n_distinct: 1.0 / max(n_distinct_left, n_distinct_right)
/// - Default: 0.1 for equi-joins
fn estimate_join_selectivity(
    _join_type: JoinKind,
    left: &str,
    right: &str,
    condition: &str,
    _table_breakdown: &[TableCostBreakdown],
    schema: &SchemaDefinition,
) -> f64 {
    let (left_col, right_col) = parse_join_condition(condition);

    let left_lower = left.to_lowercase();
    let right_lower = right.to_lowercase();

    // Check for FK relationship
    if is_fk_join(&left_lower, &left_col, &right_lower, &right_col, schema) {
        return 1.0;
    }
    // Check reverse direction too
    if is_fk_join(&right_lower, &right_col, &left_lower, &left_col, schema) {
        return 1.0;
    }

    // Use n_distinct for join selectivity
    let left_distinct = get_column_n_distinct(&left_lower, &left_col, schema);
    let right_distinct = get_column_n_distinct(&right_lower, &right_col, schema);

    if let (Some(ld), Some(rd)) = (left_distinct, right_distinct) {
        return 1.0 / ld.max(rd).max(1.0);
    }

    // Default: 10% selectivity for equi-joins
    0.1
}

/// Parse a join condition to extract left and right column names.
///
/// Handles patterns like:
/// - "o.user_id = c.id"
/// - "users.id = orders.user_id"
/// - "left_col = right_col"
fn parse_join_condition(condition: &str) -> (String, String) {
    let trimmed = condition.trim();
    if trimmed.is_empty() {
        return (String::new(), String::new());
    }

    // Split on " = "
    let parts: Vec<&str> = trimmed.splitn(2, " = ").collect();
    if parts.len() != 2 {
        return (String::new(), String::new());
    }

    let left_part = parts[0].trim();
    let right_part = parts[1].trim();

    // Extract column name (strip table alias)
    let left_col = left_part
        .rfind('.')
        .map(|p| &left_part[p + 1..])
        .unwrap_or(left_part)
        .to_lowercase();
    let right_col = right_part
        .rfind('.')
        .map(|p| &right_part[p + 1..])
        .unwrap_or(right_part)
        .to_lowercase();

    (left_col, right_col)
}

/// Check if a join is a foreign key join.
///
/// Looks at `left_table.foreign_keys` to see if `left_col` references `right_table.right_col`.
fn is_fk_join(
    left_table: &str,
    left_col: &str,
    right_table: &str,
    right_col: &str,
    schema: &SchemaDefinition,
) -> bool {
    if left_col.is_empty() || right_col.is_empty() {
        return false;
    }
    let table_def = match schema.tables.get(left_table) {
        Some(td) => td,
        None => return false,
    };
    table_def.foreign_keys.iter().any(|fk| {
        let fk_cols_match = fk.columns.iter().any(|c| c.to_lowercase() == left_col);
        let ref_table_match = fk.references.table.to_lowercase() == right_table;
        let ref_cols_match = fk
            .references
            .columns
            .iter()
            .any(|c| c.to_lowercase() == right_col);
        fk_cols_match && ref_table_match && ref_cols_match
    })
}

/// Look up n_distinct for a column in a table.
fn get_column_n_distinct(
    table_name: &str,
    col_name: &str,
    schema: &SchemaDefinition,
) -> Option<f64> {
    if col_name.is_empty() {
        return None;
    }
    let table_def = schema.tables.get(table_name)?;
    let row_count = table_def
        .statistics
        .as_ref()
        .and_then(|s| s.row_count)
        .unwrap_or(DEFAULT_ROW_COUNT);
    let col_def = table_def
        .columns
        .iter()
        .find(|c| c.name.to_lowercase() == col_name)?;
    col_def.n_distinct(row_count)
}

/// Flatten all steps including those nested in subqueries.
fn all_steps(steps: &[PlanStep]) -> Vec<&PlanStep> {
    let mut result = Vec::new();
    for step in steps {
        result.push(step);
        if let PlanOperation::Subquery { inner_steps, .. } = &step.operation {
            result.extend(all_steps(inner_steps));
        }
    }
    result
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::schema::types::*;
    use std::collections::HashMap;

    fn schema_with_stats() -> SchemaDefinition {
        let mut tables = HashMap::new();
        tables.insert(
            "users".to_string(),
            TableDef {
                description: None,
                database: None,
                schema: None,
                columns: vec![
                    ColumnDef {
                        name: "id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],

                        statistics: None,
                    },
                    ColumnDef {
                        name: "email".to_string(),
                        data_type: "VARCHAR".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],

                        statistics: None,
                    },
                ],
                primary_key: Some(vec!["id".to_string()]),
                foreign_keys: vec![],
                statistics: Some(TableStatistics {
                    row_count: Some(1_000_000),
                    avg_row_bytes: Some(256),
                    partitioned: false,
                    partition_columns: vec![],
                    correlated_columns: vec![],
                    ..Default::default()
                }),

                clustering_keys: vec![],
            },
        );
        tables.insert(
            "events".to_string(),
            TableDef {
                description: None,
                database: None,
                schema: None,
                columns: vec![
                    ColumnDef {
                        name: "id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],

                        statistics: None,
                    },
                    ColumnDef {
                        name: "event_date".to_string(),
                        data_type: "DATE".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],

                        statistics: None,
                    },
                    ColumnDef {
                        name: "user_id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],

                        statistics: None,
                    },
                ],
                primary_key: Some(vec!["id".to_string()]),
                foreign_keys: vec![],
                statistics: Some(TableStatistics {
                    row_count: Some(100_000_000),
                    avg_row_bytes: Some(128),
                    partitioned: true,
                    partition_columns: vec!["event_date".to_string()],
                    correlated_columns: vec![],
                    ..Default::default()
                }),

                clustering_keys: vec![],
            },
        );
        SchemaDefinition {
            version: "1".to_string(),
            dialect: SqlDialect::BigQuery,
            tables,
            database: None,
            schema_name: None,
            glossary: None,
        }
    }

    #[test]
    fn test_estimate_simple_scan_with_stats() {
        let schema = schema_with_stats();
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "users".to_string(),
                columns_read: vec!["id".to_string(), "email".to_string()],
                filters: vec![],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert!(result.bytes_scanned > 0);
        assert_eq!(result.cost_tier, CostTier::Cheap);
        assert_eq!(result.table_breakdown.len(), 1);
        assert!(result.table_breakdown[0].has_statistics);
    }

    #[test]
    fn test_estimate_filtered_scan() {
        let schema = schema_with_stats();
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "users".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec!["id = 42".to_string()],
            },
            notes: None,
        }];
        // Use Generic dialect (non-columnar) to test filter selectivity reduction.
        // Columnar dialects apply a floor that masks selectivity effects.
        let result = estimate_cost(&steps, &schema, SqlDialect::Generic);
        // Filtered scan should be much less than full scan
        let full_scan = estimate_cost(
            &[PlanStep {
                step: 1,
                operation: PlanOperation::TableScan {
                    table: "users".to_string(),
                    columns_read: vec!["id".to_string()],
                    filters: vec![],
                },
                notes: None,
            }],
            &schema,
            SqlDialect::Generic,
        );
        assert!(result.bytes_scanned < full_scan.bytes_scanned);
    }

    #[test]
    fn test_estimate_partition_filter_warning() {
        let schema = schema_with_stats();
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "events".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec!["user_id = 1".to_string()], // Not partition column!
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert!(!result.warnings.is_empty());
        assert!(result.warnings[0].contains("partitioned"));
    }

    #[test]
    fn test_estimate_partition_filter_applied() {
        let schema = schema_with_stats();
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "events".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec!["event_date = '2024-01-01'".to_string()],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        // Should have reduced bytes due to partition pruning
        assert!(result.warnings.is_empty());
    }

    #[test]
    fn test_estimate_no_stats_returns_unknown() {
        let schema = SchemaDefinition {
            version: "1".to_string(),
            dialect: SqlDialect::Generic,
            tables: {
                let mut t = HashMap::new();
                t.insert(
                    "foo".to_string(),
                    TableDef {
                        description: None,
                        database: None,
                        schema: None,
                        columns: vec![ColumnDef {
                            name: "id".to_string(),
                            data_type: "INT".to_string(),
                            nullable: false,
                            description: None,
                            tags: vec![],
                            synonyms: vec![],

                            statistics: None,
                        }],
                        primary_key: None,
                        foreign_keys: vec![],
                        statistics: None,

                        clustering_keys: vec![],
                    },
                );
                t
            },
            database: None,
            schema_name: None,
            glossary: None,
        };
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "foo".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec![],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::Generic);
        // Even without statistics, fallback heuristic produces a cost estimate
        // so the tier should be classified (not Unknown)
        assert_ne!(result.cost_tier, CostTier::Unknown);
        assert!(result.bytes_scanned > 0);
    }

    #[test]
    fn test_selectivity_equality() {
        assert!((PredicateSelectivity::Equality.factor() - 0.1).abs() < f64::EPSILON);
    }

    #[test]
    fn test_selectivity_range() {
        assert!((PredicateSelectivity::Range.factor() - 0.3).abs() < f64::EPSILON);
    }

    #[test]
    fn test_selectivity_in_list() {
        assert!((PredicateSelectivity::InList(5).factor() - 0.5).abs() < f64::EPSILON);
        assert!((PredicateSelectivity::InList(20).factor() - 1.0).abs() < f64::EPSILON);
    }

    #[test]
    fn test_cost_tier_classification() {
        assert_eq!(CostTier::from_cost(0.001), CostTier::Cheap);
        assert_eq!(CostTier::from_cost(0.5), CostTier::Moderate);
        assert_eq!(CostTier::from_cost(50.0), CostTier::Expensive);
        assert_eq!(CostTier::from_cost(500.0), CostTier::Dangerous);
    }

    // --- Bug fix tests ---

    #[test]
    fn test_empty_plan_steps() {
        let schema = schema_with_stats();
        let steps: Vec<PlanStep> = vec![];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert_eq!(result.bytes_scanned, 0);
        assert_eq!(result.cost_tier, CostTier::Unknown);
        assert!(result.table_breakdown.is_empty());
        assert!(result.warnings.is_empty());
    }

    #[test]
    fn test_zero_byte_table() {
        let mut schema = schema_with_stats();
        schema.tables.get_mut("users").unwrap().statistics = Some(TableStatistics {
            row_count: Some(1_000),
            avg_row_bytes: Some(0),
            partitioned: false,
            partition_columns: vec![],
            correlated_columns: vec![],
            ..Default::default()
        });
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "users".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec![],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert_eq!(result.bytes_scanned, 0);
    }

    #[test]
    fn test_zero_row_table() {
        let mut schema = schema_with_stats();
        schema.tables.get_mut("users").unwrap().statistics = Some(TableStatistics {
            row_count: Some(0),
            avg_row_bytes: Some(256),
            partitioned: false,
            partition_columns: vec![],
            correlated_columns: vec![],
            ..Default::default()
        });
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "users".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec![],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert_eq!(result.bytes_scanned, 0);
        assert_eq!(result.table_breakdown[0].estimated_rows, 0);
    }

    #[test]
    fn test_cost_tier_nan() {
        assert_eq!(CostTier::from_cost(f64::NAN), CostTier::Unknown);
    }

    #[test]
    fn test_cost_tier_positive_infinity() {
        assert_eq!(CostTier::from_cost(f64::INFINITY), CostTier::Unknown);
    }

    #[test]
    fn test_cost_tier_negative_infinity() {
        assert_eq!(CostTier::from_cost(f64::NEG_INFINITY), CostTier::Unknown);
    }

    #[test]
    fn test_cost_tier_negative_value() {
        assert_eq!(CostTier::from_cost(-1.0), CostTier::Unknown);
    }

    #[test]
    fn test_partition_filter_substring_no_false_positive() {
        // Bug 1: filter on "id" should NOT match partition column "order_id"
        let mut schema = schema_with_stats();
        schema.tables.insert(
            "orders".to_string(),
            TableDef {
                description: None,
                database: None,
                schema: None,
                columns: vec![
                    ColumnDef {
                        name: "id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],

                        statistics: None,
                    },
                    ColumnDef {
                        name: "order_id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],

                        statistics: None,
                    },
                ],
                primary_key: None,
                foreign_keys: vec![],
                statistics: Some(TableStatistics {
                    row_count: Some(1_000_000),
                    avg_row_bytes: Some(100),
                    partitioned: true,
                    partition_columns: vec!["order_id".to_string()],
                    correlated_columns: vec![],
                    ..Default::default()
                }),

                clustering_keys: vec![],
            },
        );
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "orders".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec!["id = 5".to_string()],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        // Should warn about missing partition filter (id != order_id)
        assert!(
            !result.warnings.is_empty(),
            "Filter on 'id' should NOT match partition column 'order_id'"
        );
        assert!(result.warnings[0].contains("partitioned"));
    }

    #[test]
    fn test_partition_filter_exact_match() {
        // Exact token match should work
        let mut schema = schema_with_stats();
        schema.tables.insert(
            "orders".to_string(),
            TableDef {
                description: None,
                database: None,
                schema: None,
                columns: vec![ColumnDef {
                    name: "order_id".to_string(),
                    data_type: "INT".to_string(),
                    nullable: false,
                    description: None,
                    tags: vec![],
                    synonyms: vec![],

                    statistics: None,
                }],
                primary_key: None,
                foreign_keys: vec![],
                statistics: Some(TableStatistics {
                    row_count: Some(1_000_000),
                    avg_row_bytes: Some(100),
                    partitioned: true,
                    partition_columns: vec!["order_id".to_string()],
                    correlated_columns: vec![],
                    ..Default::default()
                }),

                clustering_keys: vec![],
            },
        );
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "orders".to_string(),
                columns_read: vec!["order_id".to_string()],
                filters: vec!["order_id = 42".to_string()],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert!(
            result.warnings.is_empty(),
            "Exact token match on 'order_id' should match partition column 'order_id'"
        );
    }

    #[test]
    fn test_selectivity_sqrt_dampening_multiple_filters() {
        // Bug 4: Two equality filters should NOT give 0.01 (independent multiplication)
        let stats = TableStatistics {
            row_count: Some(100_000),
            avg_row_bytes: Some(200),
            partitioned: false,
            partition_columns: vec![],
            correlated_columns: vec![],
            ..Default::default()
        };
        let schema = SchemaDefinition {
            version: "1".to_string(),
            dialect: SqlDialect::Generic,
            tables: HashMap::new(),
            database: None,
            schema_name: None,
            glossary: None,
        };
        let filters = vec!["a = 1".to_string(), "b = 2".to_string()];
        let sel = estimate_selectivity(&filters, &stats, "test", &schema);
        // First filter: 0.1, second with sqrt: 0.1 * sqrt(0.1) ~= 0.0316
        // Without dampening it would be 0.01
        assert!(
            sel.expected > 0.01,
            "Two equality filters with sqrt dampening should be > 0.01, got {}",
            sel.expected
        );
        let expected = 0.1 * (0.1_f64).sqrt();
        assert!(
            (sel.expected - expected).abs() < 1e-9,
            "Expected ~{expected}, got {}",
            sel.expected
        );
    }

    #[test]
    fn test_selectivity_single_filter_unchanged() {
        // A single filter should still use the exact factor (no dampening)
        let stats = TableStatistics {
            row_count: Some(100_000),
            avg_row_bytes: Some(200),
            partitioned: false,
            partition_columns: vec![],
            correlated_columns: vec![],
            ..Default::default()
        };
        let schema = SchemaDefinition {
            version: "1".to_string(),
            dialect: SqlDialect::Generic,
            tables: HashMap::new(),
            database: None,
            schema_name: None,
            glossary: None,
        };
        let filters = vec!["x = 1".to_string()];
        let sel = estimate_selectivity(&filters, &stats, "test", &schema);
        assert!(
            (sel.expected - 0.1).abs() < f64::EPSILON,
            "Single filter should be exactly 0.1, got {}",
            sel.expected
        );
    }

    #[test]
    fn test_cross_join_row_count_based() {
        // Bug 2: Cross join should use row-count product, not naive 10x multiplier
        let schema = schema_with_stats();
        let steps = vec![
            PlanStep {
                step: 1,
                operation: PlanOperation::TableScan {
                    table: "users".to_string(),
                    columns_read: vec!["id".to_string()],
                    filters: vec![],
                },
                notes: None,
            },
            PlanStep {
                step: 2,
                operation: PlanOperation::TableScan {
                    table: "events".to_string(),
                    columns_read: vec!["id".to_string()],
                    filters: vec![],
                },
                notes: None,
            },
            PlanStep {
                step: 3,
                operation: PlanOperation::Join {
                    join_type: JoinKind::Cross,
                    left: "users".to_string(),
                    right: "events".to_string(),
                    condition: String::new(),
                },
                notes: None,
            },
        ];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        // Cross join bytes should be based on row counts (1M * 100M * max_avg_bytes)
        assert!(result.bytes_scanned > 0);
        assert!(!result.warnings.is_empty());
        assert!(
            result.warnings.iter().any(|w| w.contains("CROSS JOIN")),
            "Expected a CROSS JOIN warning, got: {:?}",
            result.warnings
        );
    }

    #[test]
    fn test_disclaimer_field_defaults_to_none() {
        let schema = schema_with_stats();
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "users".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec![],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert!(result.disclaimer.is_none());
    }

    #[test]
    fn test_disclaimer_skipped_in_serialization() {
        let cost = CostEstimate {
            bytes_scanned: 100,
            cost_usd: 0.001,
            cost_tier: CostTier::Cheap,
            table_breakdown: vec![],
            warnings: vec![],
            disclaimer: None,
            cost_range: None,
            recommendations: vec![],
            confidence: EstimationConfidence::High,
            confidence_factors: vec![],
            estimation_method: None,
            time_based_cost: None,
        };
        let json = serde_json::to_string(&cost).unwrap();
        assert!(
            !json.contains("disclaimer"),
            "disclaimer=None should be skipped in serialization"
        );

        let cost_with_disclaimer = CostEstimate {
            disclaimer: Some("Estimated using on-demand pricing".to_string()),
            ..cost
        };
        let json2 = serde_json::to_string(&cost_with_disclaimer).unwrap();
        assert!(
            json2.contains("disclaimer"),
            "disclaimer=Some should be present in serialization"
        );
    }

    // -----------------------------------------------------------------------
    // New tests: column-statistics-driven selectivity
    // -----------------------------------------------------------------------

    /// Build a schema with column-level statistics for testing.
    fn schema_with_column_stats() -> SchemaDefinition {
        let mut tables = HashMap::new();
        tables.insert(
            "orders".to_string(),
            TableDef {
                description: None,
                database: None,
                schema: None,
                columns: vec![
                    ColumnDef {
                        name: "id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: Some(ColumnStatistics {
                            n_distinct: Some(-1.0), // all unique
                            null_fraction: Some(0.0),
                            min_value: Some("1".to_string()),
                            max_value: Some("1000000".to_string()),
                            most_common_values: vec![],

                            avg_bytes: None,
                            compression_ratio: None,
                        }),
                    },
                    ColumnDef {
                        name: "status".to_string(),
                        data_type: "VARCHAR".to_string(),
                        nullable: true,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: Some(ColumnStatistics {
                            n_distinct: Some(5.0),
                            null_fraction: Some(0.01),
                            min_value: None,
                            max_value: None,
                            most_common_values: vec![
                                McvEntry {
                                    value: "completed".to_string(),
                                    frequency: 0.60,
                                },
                                McvEntry {
                                    value: "pending".to_string(),
                                    frequency: 0.20,
                                },
                                McvEntry {
                                    value: "cancelled".to_string(),
                                    frequency: 0.10,
                                },
                            ],

                            avg_bytes: None,
                            compression_ratio: None,
                        }),
                    },
                    ColumnDef {
                        name: "amount".to_string(),
                        data_type: "DECIMAL(10,2)".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: Some(ColumnStatistics {
                            n_distinct: Some(-0.8),
                            null_fraction: Some(0.0),
                            min_value: Some("0.01".to_string()),
                            max_value: Some("99999.99".to_string()),
                            most_common_values: vec![],

                            avg_bytes: None,
                            compression_ratio: None,
                        }),
                    },
                    ColumnDef {
                        name: "region".to_string(),
                        data_type: "VARCHAR".to_string(),
                        nullable: true,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: Some(ColumnStatistics {
                            n_distinct: Some(50.0),
                            null_fraction: Some(0.02),
                            min_value: None,
                            max_value: None,
                            most_common_values: vec![McvEntry {
                                value: "US-East".to_string(),
                                frequency: 0.25,
                            }],

                            avg_bytes: None,
                            compression_ratio: None,
                        }),
                    },
                    ColumnDef {
                        name: "notes".to_string(),
                        data_type: "VARCHAR".to_string(),
                        nullable: true,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: Some(ColumnStatistics {
                            n_distinct: None,
                            null_fraction: Some(0.40),
                            min_value: None,
                            max_value: None,
                            most_common_values: vec![],

                            avg_bytes: None,
                            compression_ratio: None,
                        }),
                    },
                    ColumnDef {
                        name: "user_id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: Some(ColumnStatistics {
                            n_distinct: Some(50000.0),
                            null_fraction: Some(0.0),
                            min_value: None,
                            max_value: None,
                            most_common_values: vec![],

                            avg_bytes: None,
                            compression_ratio: None,
                        }),
                    },
                ],
                primary_key: Some(vec!["id".to_string()]),
                foreign_keys: vec![ForeignKeyDef {
                    columns: vec!["user_id".to_string()],
                    references: ForeignKeyRef {
                        table: "customers".to_string(),
                        columns: vec!["id".to_string()],
                    },
                }],
                statistics: Some(TableStatistics {
                    row_count: Some(1_000_000),
                    avg_row_bytes: Some(256),
                    partitioned: false,
                    partition_columns: vec![],
                    correlated_columns: vec![vec!["status".to_string(), "region".to_string()]],
                    ..Default::default()
                }),

                clustering_keys: vec![],
            },
        );
        tables.insert(
            "customers".to_string(),
            TableDef {
                description: None,
                database: None,
                schema: None,
                columns: vec![
                    ColumnDef {
                        name: "id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: Some(ColumnStatistics {
                            n_distinct: Some(-1.0),
                            null_fraction: Some(0.0),
                            ..Default::default()
                        }),
                    },
                    ColumnDef {
                        name: "name".to_string(),
                        data_type: "VARCHAR".to_string(),
                        nullable: true,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: None,
                    },
                ],
                primary_key: Some(vec!["id".to_string()]),
                foreign_keys: vec![],
                statistics: Some(TableStatistics {
                    row_count: Some(50_000),
                    avg_row_bytes: Some(128),
                    partitioned: false,
                    partition_columns: vec![],
                    correlated_columns: vec![],
                    ..Default::default()
                }),

                clustering_keys: vec![],
            },
        );
        SchemaDefinition {
            version: "1".to_string(),
            dialect: SqlDialect::BigQuery,
            tables,
            database: None,
            schema_name: None,
            glossary: None,
        }
    }

    // --- extract_column_name tests ---

    #[test]
    fn test_extract_column_name_simple() {
        assert_eq!(
            extract_column_name("status = 'completed'"),
            Some("status".to_string())
        );
    }

    #[test]
    fn test_extract_column_name_with_alias() {
        assert_eq!(
            extract_column_name("o.status = 'completed'"),
            Some("status".to_string())
        );
    }

    #[test]
    fn test_extract_column_name_range() {
        assert_eq!(
            extract_column_name("amount > 100"),
            Some("amount".to_string())
        );
    }

    #[test]
    fn test_extract_column_name_is_null() {
        assert_eq!(
            extract_column_name("notes IS NULL"),
            Some("notes".to_string())
        );
    }

    #[test]
    fn test_extract_column_name_empty() {
        assert_eq!(extract_column_name(""), None);
    }

    // --- Stats-driven equality selectivity ---

    #[test]
    fn test_stats_equality_n_distinct() {
        // status has n_distinct=5, null_fraction=0.01
        // expected selectivity = (1/5) * (1.0 - 0.01) = 0.198
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["status = 'unknown_value'".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based, "Should use column statistics");
        // 1/5 * 0.99 = 0.198
        assert!(
            (sel.expected - 0.198).abs() < 0.01,
            "Expected ~0.198, got {}",
            sel.expected
        );
    }

    #[test]
    fn test_stats_equality_mcv_hit() {
        // "completed" has MCV frequency 0.60, null_fraction=0.01
        // expected = 0.60 * 0.99 = 0.594
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["status = 'completed'".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        assert!(
            (sel.expected - 0.594).abs() < 0.01,
            "Expected ~0.594 (MCV hit), got {}",
            sel.expected
        );
    }

    #[test]
    fn test_stats_equality_mcv_pending() {
        // "pending" has MCV frequency 0.20
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["status = 'pending'".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        // 0.20 * 0.99 = 0.198
        assert!(
            (sel.expected - 0.198).abs() < 0.01,
            "Expected ~0.198 for 'pending', got {}",
            sel.expected
        );
    }

    // --- IS NULL / IS NOT NULL ---

    #[test]
    fn test_stats_is_null() {
        // notes has null_fraction=0.40
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["notes IS NULL".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        assert!(
            (sel.expected - 0.40).abs() < 0.01,
            "Expected ~0.40 for IS NULL, got {}",
            sel.expected
        );
    }

    #[test]
    fn test_stats_is_not_null() {
        // notes has null_fraction=0.40, so IS NOT NULL = 0.60
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["notes IS NOT NULL".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        assert!(
            (sel.expected - 0.60).abs() < 0.01,
            "Expected ~0.60 for IS NOT NULL, got {}",
            sel.expected
        );
    }

    // --- Range selectivity ---

    #[test]
    fn test_stats_range_greater_than() {
        // amount: min=0.01, max=99999.99, range=99999.98
        // "amount > 50000" -> (99999.99 - 50000) / 99999.98 ~= 0.5
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["amount > 50000".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        assert!(
            (sel.expected - 0.5).abs() < 0.05,
            "Expected ~0.5 for amount > 50000, got {}",
            sel.expected
        );
    }

    #[test]
    fn test_stats_range_less_than() {
        // "amount < 10000" -> (10000 - 0.01) / 99999.98 ~= 0.1
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["amount < 10000".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        assert!(
            (sel.expected - 0.1).abs() < 0.02,
            "Expected ~0.1 for amount < 10000, got {}",
            sel.expected
        );
    }

    #[test]
    fn test_stats_range_between() {
        // "amount BETWEEN 20000 AND 40000" -> (40000-20000)/99999.98 ~= 0.2
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["amount BETWEEN 20000 AND 40000".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        assert!(
            (sel.expected - 0.2).abs() < 0.03,
            "Expected ~0.2 for BETWEEN 20000 AND 40000, got {}",
            sel.expected
        );
    }

    // --- IN list selectivity ---

    #[test]
    fn test_stats_in_list() {
        // status n_distinct=5, IN ('completed', 'pending') -> 2/5 * 0.99 = 0.396
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["status IN ('completed', 'pending')".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(sel.stats_based);
        assert!(
            (sel.expected - 0.396).abs() < 0.05,
            "Expected ~0.396 for IN list with 2 values, got {}",
            sel.expected
        );
    }

    // --- Confidence intervals ---

    #[test]
    fn test_confidence_pessimistic_ge_expected() {
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec!["status = 'completed'".to_string()];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);
        assert!(
            sel.pessimistic >= sel.expected,
            "Pessimistic ({}) should be >= expected ({})",
            sel.pessimistic,
            sel.expected
        );
        assert!(
            sel.optimistic <= sel.expected,
            "Optimistic ({}) should be <= expected ({})",
            sel.optimistic,
            sel.expected
        );
    }

    #[test]
    fn test_confidence_interval_ordering() {
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        for filter_str in &[
            "amount > 50000",
            "status = 'pending'",
            "notes IS NULL",
            "id = 42",
        ] {
            let filters = vec![filter_str.to_string()];
            let sel = estimate_selectivity(&filters, stats, "orders", &schema);
            assert!(
                sel.optimistic <= sel.expected,
                "Filter '{filter_str}': optimistic ({}) should be <= expected ({})",
                sel.optimistic,
                sel.expected
            );
            assert!(
                sel.expected <= sel.pessimistic,
                "Filter '{filter_str}': expected ({}) should be <= pessimistic ({})",
                sel.expected,
                sel.pessimistic
            );
        }
    }

    // --- Correlation groups ---

    #[test]
    fn test_correlated_columns_no_over_multiply() {
        // status and region are correlated in orders
        // Without correlation handling: 0.198 * sqrt(0.0196) ~= 0.028
        // With correlation: max(0.198, 0.0196) = 0.198 (the larger selectivity)
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec![
            "status = 'unknown'".to_string(),
            "region = 'US-East'".to_string(),
        ];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);

        // The second filter is correlated with the first, so we use max()
        // status = 'unknown': 1/5 * 0.99 = 0.198
        // region = 'US-East': MCV freq 0.25 * 0.98 = 0.245
        // Combined (correlated): max(0.198, 0.245) = 0.245
        assert!(
            sel.expected > 0.15,
            "Correlated columns should not over-multiply, got {}",
            sel.expected
        );
    }

    #[test]
    fn test_uncorrelated_columns_do_multiply() {
        // status and amount are NOT correlated
        let schema = schema_with_column_stats();
        let stats = schema.tables["orders"].statistics.as_ref().unwrap();
        let filters = vec![
            "status = 'unknown'".to_string(),
            "amount > 50000".to_string(),
        ];
        let sel = estimate_selectivity(&filters, stats, "orders", &schema);

        // status: 0.198, amount > 50000: ~0.5
        // Combined: 0.198 * sqrt(0.5) ~= 0.14
        assert!(
            sel.expected < 0.198,
            "Uncorrelated columns should multiply down, got {}",
            sel.expected
        );
    }

    // --- Join cardinality estimation ---

    #[test]
    fn test_fk_join_selectivity() {
        // orders.user_id -> customers.id is a FK join
        let schema = schema_with_column_stats();
        let sel = estimate_join_selectivity(
            JoinKind::Inner,
            "orders",
            "customers",
            "orders.user_id = customers.id",
            &[],
            &schema,
        );
        assert!(
            (sel - 1.0).abs() < f64::EPSILON,
            "FK join should have selectivity ~1.0, got {sel}"
        );
    }

    #[test]
    fn test_n_distinct_join_selectivity() {
        // If no FK, use n_distinct. orders.status (5) vs customers.id (50000)
        // Join selectivity = 1/max(5, 50000) = 1/50000 = 0.00002
        let schema = schema_with_column_stats();
        let sel = estimate_join_selectivity(
            JoinKind::Inner,
            "orders",
            "customers",
            "orders.status = customers.id",
            &[],
            &schema,
        );
        assert!(
            sel < 0.001,
            "n_distinct join should have low selectivity, got {sel}"
        );
    }

    #[test]
    fn test_default_join_selectivity() {
        // Unknown columns -> default 0.1
        let schema = schema_with_column_stats();
        let sel = estimate_join_selectivity(
            JoinKind::Inner,
            "orders",
            "customers",
            "unknown_col = other_col",
            &[],
            &schema,
        );
        assert!(
            (sel - 0.1).abs() < f64::EPSILON,
            "Default join selectivity should be 0.1, got {sel}"
        );
    }

    // --- parse_join_condition ---

    #[test]
    fn test_parse_join_condition_aliased() {
        let (left, right) = parse_join_condition("o.user_id = c.id");
        assert_eq!(left, "user_id");
        assert_eq!(right, "id");
    }

    #[test]
    fn test_parse_join_condition_simple() {
        let (left, right) = parse_join_condition("user_id = id");
        assert_eq!(left, "user_id");
        assert_eq!(right, "id");
    }

    #[test]
    fn test_parse_join_condition_empty() {
        let (left, right) = parse_join_condition("");
        assert_eq!(left, "");
        assert_eq!(right, "");
    }

    // --- CostRange wiring ---

    #[test]
    fn test_cost_range_present_with_column_stats() {
        let schema = schema_with_column_stats();
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "orders".to_string(),
                columns_read: vec!["id".to_string(), "status".to_string()],
                filters: vec!["status = 'completed'".to_string()],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert!(
            result.cost_range.is_some(),
            "CostRange should be present when column stats drive selectivity"
        );
        let range = result.cost_range.unwrap();
        assert!(
            range.low <= range.expected,
            "low ({}) should be <= expected ({})",
            range.low,
            range.expected
        );
        assert!(
            range.expected <= range.high,
            "expected ({}) should be <= high ({})",
            range.expected,
            range.high
        );
    }

    #[test]
    fn test_cost_range_absent_without_column_stats() {
        // schema_with_stats() has no column-level statistics
        let schema = schema_with_stats();
        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "users".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec!["id = 42".to_string()],
            },
            notes: None,
        }];
        let result = estimate_cost(&steps, &schema, SqlDialect::BigQuery);
        assert!(
            result.cost_range.is_none(),
            "CostRange should be absent when no column stats used"
        );
    }

    // --- Backwards compatibility: no column stats produces same heuristic ---

    #[test]
    fn test_backwards_compat_no_column_stats() {
        // With no column stats, the selectivity should still be heuristic-based
        let schema = schema_with_stats();
        let stats = schema.tables["users"].statistics.as_ref().unwrap();
        let filters = vec!["id = 42".to_string()];
        let sel = estimate_selectivity(&filters, stats, "users", &schema);
        // Should use heuristic: equality = 0.1
        assert!(
            (sel.expected - 0.1).abs() < f64::EPSILON,
            "Without column stats, equality should still be 0.1, got {}",
            sel.expected
        );
        assert!(!sel.stats_based);
    }

    #[test]
    fn test_backwards_compat_empty_filters() {
        let schema = schema_with_stats();
        let stats = schema.tables["users"].statistics.as_ref().unwrap();
        let filters: Vec<String> = vec![];
        let sel = estimate_selectivity(&filters, stats, "users", &schema);
        assert!(
            (sel.expected - 1.0).abs() < f64::EPSILON,
            "Empty filters should give selectivity 1.0"
        );
        assert!(!sel.stats_based);
    }

    // --- extract_rhs_value ---

    #[test]
    fn test_extract_rhs_value_quoted() {
        assert_eq!(
            extract_rhs_value("status = 'completed'"),
            Some("completed".to_string())
        );
    }

    #[test]
    fn test_extract_rhs_value_numeric() {
        assert_eq!(extract_rhs_value("amount > 100"), Some("100".to_string()));
    }

    #[test]
    fn test_extract_rhs_value_no_operator() {
        assert_eq!(extract_rhs_value("justtext"), None);
    }

    // --- parse_numeric ---

    #[test]
    fn test_parse_numeric_integer() {
        assert_eq!(parse_numeric("42"), Some(42.0));
    }

    #[test]
    fn test_parse_numeric_float() {
        assert!((parse_numeric("3.14").unwrap() - 3.14).abs() < f64::EPSILON);
    }

    #[test]
    fn test_parse_numeric_date() {
        // Should parse as a numeric representation for ordering
        assert!(parse_numeric("2024-01-15").is_some());
    }

    #[test]
    fn test_parse_numeric_non_numeric() {
        assert_eq!(parse_numeric("hello"), None);
    }

    // --- Full integration: stats-driven query cost ---

    #[test]
    fn test_full_query_stats_vs_heuristic() {
        // With column stats: "status = 'completed'" -> 60% selectivity (MCV)
        // Without column stats: equality = 10%
        // Stats-driven should estimate more bytes (higher selectivity)
        // Use Generic dialect (non-columnar) so the columnar floor doesn't
        // mask the selectivity difference.
        let schema_stats = schema_with_column_stats();
        let schema_no_stats = schema_with_stats();

        let steps = vec![PlanStep {
            step: 1,
            operation: PlanOperation::TableScan {
                table: "orders".to_string(),
                columns_read: vec!["id".to_string()],
                filters: vec!["status = 'completed'".to_string()],
            },
            notes: None,
        }];

        let with_stats = estimate_cost(&steps, &schema_stats, SqlDialect::Generic);
        // For schema_no_stats, we need an "orders" table with table-level stats
        let mut schema_heuristic = schema_no_stats.clone();
        schema_heuristic.tables.insert(
            "orders".to_string(),
            TableDef {
                description: None,
                database: None,
                schema: None,
                columns: vec![
                    ColumnDef {
                        name: "id".to_string(),
                        data_type: "INT".to_string(),
                        nullable: false,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: None, // no column stats
                    },
                    ColumnDef {
                        name: "status".to_string(),
                        data_type: "VARCHAR".to_string(),
                        nullable: true,
                        description: None,
                        tags: vec![],
                        synonyms: vec![],
                        statistics: None, // no column stats
                    },
                ],
                primary_key: None,
                foreign_keys: vec![],
                statistics: Some(TableStatistics {
                    row_count: Some(1_000_000),
                    avg_row_bytes: Some(256),
                    partitioned: false,
                    partition_columns: vec![],
                    correlated_columns: vec![],
                    ..Default::default()
                }),

                clustering_keys: vec![],
            },
        );
        let without_stats = estimate_cost(&steps, &schema_heuristic, SqlDialect::Generic);

        // MCV frequency 0.60 > heuristic 0.10, so stats-driven should scan MORE
        assert!(
            with_stats.bytes_scanned > without_stats.bytes_scanned,
            "Stats-driven ({}) should scan more than heuristic ({}) for common value",
            with_stats.bytes_scanned,
            without_stats.bytes_scanned
        );
    }

    // --- find_mcv_frequency ---

    #[test]
    fn test_find_mcv_hit() {
        let mcvs = vec![
            McvEntry {
                value: "completed".to_string(),
                frequency: 0.60,
            },
            McvEntry {
                value: "pending".to_string(),
                frequency: 0.20,
            },
        ];
        assert!(
            (find_mcv_frequency("status = 'completed'", &mcvs).unwrap() - 0.60).abs()
                < f64::EPSILON
        );
    }

    #[test]
    fn test_find_mcv_miss() {
        let mcvs = vec![McvEntry {
            value: "completed".to_string(),
            frequency: 0.60,
        }];
        assert!(find_mcv_frequency("status = 'rare_value'", &mcvs).is_none());
    }

    #[test]
    fn test_find_mcv_empty_list() {
        assert!(find_mcv_frequency("status = 'x'", &[]).is_none());
    }

    // --- find_correlation_group ---

    #[test]
    fn test_find_correlation_group_found() {
        let groups = vec![vec!["status".to_string(), "region".to_string()]];
        assert_eq!(find_correlation_group("status", &groups), Some(0));
        assert_eq!(find_correlation_group("region", &groups), Some(0));
    }

    #[test]
    fn test_find_correlation_group_not_found() {
        let groups = vec![vec!["status".to_string(), "region".to_string()]];
        assert_eq!(find_correlation_group("amount", &groups), None);
    }

    #[test]
    fn test_find_correlation_group_multiple_groups() {
        let groups = vec![
            vec!["status".to_string(), "region".to_string()],
            vec!["channel".to_string(), "payment".to_string()],
        ];
        assert_eq!(find_correlation_group("channel", &groups), Some(1));
    }

    // --- Join estimation with FK schema ---

    #[test]
    fn test_join_reduces_total_bytes() {
        // An inner join with FK relationship should reduce bytes
        let schema = schema_with_column_stats();
        let steps_no_join = vec![
            PlanStep {
                step: 1,
                operation: PlanOperation::TableScan {
                    table: "orders".to_string(),
                    columns_read: vec!["id".to_string(), "user_id".to_string()],
                    filters: vec![],
                },
                notes: None,
            },
            PlanStep {
                step: 2,
                operation: PlanOperation::TableScan {
                    table: "customers".to_string(),
                    columns_read: vec!["id".to_string(), "name".to_string()],
                    filters: vec![],
                },
                notes: None,
            },
        ];
        let steps_with_join = vec![
            PlanStep {
                step: 1,
                operation: PlanOperation::TableScan {
                    table: "orders".to_string(),
                    columns_read: vec!["id".to_string(), "user_id".to_string()],
                    filters: vec![],
                },
                notes: None,
            },
            PlanStep {
                step: 2,
                operation: PlanOperation::TableScan {
                    table: "customers".to_string(),
                    columns_read: vec!["id".to_string(), "name".to_string()],
                    filters: vec![],
                },
                notes: None,
            },
            PlanStep {
                step: 3,
                operation: PlanOperation::Join {
                    join_type: JoinKind::Inner,
                    left: "orders".to_string(),
                    right: "customers".to_string(),
                    condition: "orders.user_id = customers.id".to_string(),
                },
                notes: None,
            },
        ];
        let result_no_join = estimate_cost(&steps_no_join, &schema, SqlDialect::BigQuery);
        let result_with_join = estimate_cost(&steps_with_join, &schema, SqlDialect::BigQuery);

        // FK join selectivity is 1.0, so bytes should be the same or close
        // (FK join means no fan-out reduction)
        assert!(
            result_with_join.bytes_scanned <= result_no_join.bytes_scanned,
            "FK join should not increase bytes: {} vs {}",
            result_with_join.bytes_scanned,
            result_no_join.bytes_scanned
        );
    }
}
