//! Analyze failure patterns from the Snowflake ground truth benchmark.
//!
//! For every non-exact-match case, classifies each per-column mismatch into
//! one of six root cause categories, then aggregates to show what code changes
//! would fix the most cases.
//!
//! ## WRONG_SOURCES sub-categories (output cols match, sources differ)
//!
//! | Tag                  | Meaning                                             |
//! |----------------------|-----------------------------------------------------|
//! | phantom_column       | We reference a column not in the schema at all       |
//! | wrong_table          | We trace to a different table than expected           |
//! | extra_agg_input      | Extra cols from aggregate/window function arguments   |
//! | case_when_branch     | Difference in CASE WHEN branch tracing               |
//! | subquery_passthrough | Source from subquery/CTE we don't trace through       |
//! | other                | Uncategorized                                        |
//!
//! ## MIXED pattern analysis
//!
//! For cases with multiple simultaneous failure modes (extra cols + missing
//! sources, etc.), shows the most common co-occurring patterns.
//!
//! Run:  cargo run --example analyze_patterns --release

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;

// ---------------------------------------------------------------------------
// Ground truth case
// ---------------------------------------------------------------------------

#[derive(Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
    #[allow(dead_code)]
    source_query_type: Option<String>,
}

// ---------------------------------------------------------------------------
// WRONG_SOURCES sub-tags
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
enum WrongSourceTag {
    PhantomColumn,
    WrongTable,
    ExtraAggInput,
    CaseWhenBranch,
    SubqueryPassthrough,
    Other,
}

impl WrongSourceTag {
    fn label(&self) -> &'static str {
        match self {
            Self::PhantomColumn => "phantom_column",
            Self::WrongTable => "wrong_table",
            Self::ExtraAggInput => "extra_agg_input",
            Self::CaseWhenBranch => "case_when_branch",
            Self::SubqueryPassthrough => "subquery_passthrough",
            Self::Other => "other",
        }
    }

    fn description(&self) -> &'static str {
        match self {
            Self::PhantomColumn => "We reference a column that doesn't exist in any schema table",
            Self::WrongTable => "We trace to a different table than expected",
            Self::ExtraAggInput => {
                "Extra cols from aggregate/window function arguments (COUNT, SUM, etc.)"
            }
            Self::CaseWhenBranch => "Difference in CASE WHEN / IFF branch tracing",
            Self::SubqueryPassthrough => "Source comes from a subquery/CTE we don't trace through",
            Self::Other => "Uncategorized wrong-source difference",
        }
    }
}

// ---------------------------------------------------------------------------
// Top-level failure category
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
enum FailureKind {
    ExactMatch,
    ExtraColsOnly,
    MissingColsOnly,
    WrongSourcesOnly,
    Mixed,
    EmptyActual,
    ParseError,
}

impl FailureKind {
    fn label(self) -> &'static str {
        match self {
            Self::ExactMatch => "exact_match",
            Self::ExtraColsOnly => "extra_cols_only",
            Self::MissingColsOnly => "missing_cols_only",
            Self::WrongSourcesOnly => "wrong_sources_only",
            Self::Mixed => "mixed",
            Self::EmptyActual => "empty_actual",
            Self::ParseError => "parse_error",
        }
    }
}

// ---------------------------------------------------------------------------
// MIXED combination key
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
struct MixedPattern {
    has_extra_cols: bool,
    has_missing_cols: bool,
    has_extra_sources: bool,
    has_missing_sources: bool,
}

impl MixedPattern {
    fn label(&self) -> String {
        let mut parts = Vec::new();
        if self.has_extra_cols {
            parts.push("extra_cols");
        }
        if self.has_missing_cols {
            parts.push("missing_cols");
        }
        if self.has_extra_sources {
            parts.push("extra_srcs");
        }
        if self.has_missing_sources {
            parts.push("missing_srcs");
        }
        if parts.is_empty() {
            "none".to_string()
        } else {
            parts.join(" + ")
        }
    }
}

// ---------------------------------------------------------------------------
// Sample holder
// ---------------------------------------------------------------------------

struct TagSample {
    name: String,
    sql_preview: String,
    col: String,
    expected_sources: Vec<String>,
    actual_sources: Vec<String>,
}

struct MixedSample {
    name: String,
    sql_preview: String,
    num_extra_cols: usize,
    num_missing_cols: usize,
    num_wrong_source_cols: usize,
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

/// Collect all column names from the schema (uppercased), keyed by table.
fn schema_columns(schema: &HashMap<String, serde_json::Value>) -> HashMap<String, HashSet<String>> {
    let mut result: HashMap<String, HashSet<String>> = HashMap::new();
    for (table_key, val) in schema {
        if let serde_json::Value::Object(cols) = val {
            // Table name: last segment of potentially dot-separated key
            let table_name = table_key
                .split('.')
                .last()
                .unwrap_or(table_key)
                .trim_matches('"')
                .to_uppercase();
            let entry = result.entry(table_name).or_default();
            for col_name in cols.keys() {
                entry.insert(col_name.to_uppercase());
            }
        }
    }
    result
}

/// All schema column names (flat set, uppercased).
fn all_schema_columns_flat(schema: &HashMap<String, serde_json::Value>) -> HashSet<String> {
    schema_columns(schema)
        .into_values()
        .flat_map(|cols| cols.into_iter())
        .collect()
}

/// Normalize column_dict: sort + dedup.
fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, BTreeSet<String>> {
    map.iter()
        .map(|(k, v)| (k.clone(), v.iter().cloned().collect()))
        .collect()
}

/// Normalize actual output to use short table names (last 2 segments only).
/// Our engine may produce "schema"."table"."col" while ground truth uses "table"."col".
fn normalize_to_short_names(
    map: &HashMap<String, Vec<String>>,
) -> HashMap<String, BTreeSet<String>> {
    map.iter()
        .map(|(k, v)| {
            let normalized: BTreeSet<String> = v
                .iter()
                .map(|ref_str| {
                    let parts: Vec<&str> =
                        ref_str.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        let table = parts[parts.len() - 2];
                        let col = parts[parts.len() - 1];
                        format!("\"{table}\".\"{col}\"")
                    } else {
                        ref_str.clone()
                    }
                })
                .collect();
            (k.clone(), normalized)
        })
        .collect()
}

/// Extract table name from `"TABLE"."COLUMN"` reference.
fn table_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    if parts.len() >= 2 {
        parts[parts.len() - 2].to_uppercase()
    } else {
        String::new()
    }
}

/// Extract column name from `"TABLE"."COLUMN"` reference.
fn col_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    parts.last().unwrap_or(&"").to_uppercase()
}

fn sql_preview(sql: &str) -> String {
    let preview = if sql.len() > 200 {
        format!("{}...", &sql[..200])
    } else {
        sql.to_string()
    };
    preview.replace('\n', " ").replace("  ", " ")
}

// ---------------------------------------------------------------------------
// Aggregate / window function keywords
// ---------------------------------------------------------------------------

const AGG_FUNCS: &[&str] = &[
    "COUNT(",
    "SUM(",
    "AVG(",
    "MIN(",
    "MAX(",
    "LISTAGG(",
    "ARRAY_AGG(",
    "GROUP_CONCAT(",
    "OBJECT_AGG(",
    "MEDIAN(",
    "PERCENTILE_CONT(",
    "PERCENTILE_DISC(",
    "STDDEV(",
    "VARIANCE(",
    "APPROX_COUNT_DISTINCT(",
    "HLL(",
    "COLLECT_LIST(",
    "COLLECT_SET(",
];

const WINDOW_FUNCS: &[&str] = &[
    "ROW_NUMBER(",
    "RANK(",
    "DENSE_RANK(",
    "NTILE(",
    "LAG(",
    "LEAD(",
    "FIRST_VALUE(",
    "LAST_VALUE(",
    "NTH_VALUE(",
];

fn sql_has_agg_or_window(sql_up: &str) -> bool {
    AGG_FUNCS
        .iter()
        .chain(WINDOW_FUNCS.iter())
        .any(|pat| sql_up.contains(pat))
}

fn sql_has_case(sql_up: &str) -> bool {
    sql_up.contains("CASE ") || sql_up.contains("IFF(")
}

fn sql_has_subquery(sql_up: &str) -> bool {
    // Count opening parens followed by SELECT — proxy for subqueries
    sql_up.contains("(SELECT ") || sql_up.contains("( SELECT ")
}

fn sql_has_cte(sql_up: &str) -> bool {
    sql_up.starts_with("WITH ") || sql_up.contains("\nWITH ") || sql_up.contains(" WITH ")
}

// ---------------------------------------------------------------------------
// Classify a single column's source difference
// ---------------------------------------------------------------------------

fn classify_source_diff(
    col_name: &str,
    expected_sources: &BTreeSet<String>,
    actual_sources: &BTreeSet<String>,
    sql_up: &str,
    schema_cols_by_table: &HashMap<String, HashSet<String>>,
    all_schema_cols: &HashSet<String>,
) -> WrongSourceTag {
    let extra: Vec<&String> = actual_sources.difference(expected_sources).collect();
    let missing: Vec<&String> = expected_sources.difference(actual_sources).collect();

    // --- 1. PHANTOM COLUMN ---
    // We reference columns that don't exist in any schema table.
    if !extra.is_empty() {
        let phantom_count = extra
            .iter()
            .filter(|src| {
                let c = col_of(src);
                !c.is_empty() && !all_schema_cols.contains(&c)
            })
            .count();
        if phantom_count > 0 && phantom_count == extra.len() && missing.is_empty() {
            return WrongSourceTag::PhantomColumn;
        }
    }
    if !missing.is_empty() {
        let phantom_missing = missing
            .iter()
            .filter(|src| {
                let c = col_of(src);
                !c.is_empty() && !all_schema_cols.contains(&c)
            })
            .count();
        if phantom_missing > 0 && phantom_missing == missing.len() && extra.is_empty() {
            return WrongSourceTag::PhantomColumn;
        }
    }

    // --- 2. WRONG TABLE ---
    // All expected/actual columns exist in the schema, but from different tables.
    if !extra.is_empty() && !missing.is_empty() {
        let extra_tables: BTreeSet<String> = extra.iter().map(|s| table_of(s)).collect();
        let missing_tables: BTreeSet<String> = missing.iter().map(|s| table_of(s)).collect();
        // Same column names but different tables
        let extra_col_names: BTreeSet<String> = extra.iter().map(|s| col_of(s)).collect();
        let missing_col_names: BTreeSet<String> = missing.iter().map(|s| col_of(s)).collect();
        if extra_col_names == missing_col_names && extra_tables != missing_tables {
            return WrongSourceTag::WrongTable;
        }
        // Also classify as wrong_table if tables are completely disjoint
        if extra_tables.is_disjoint(&missing_tables) {
            // Check if the missing tables look like base tables and extra tables look like
            // aliases or CTEs (they wouldn't be in schema)
            let missing_in_schema = missing_tables
                .iter()
                .any(|t| schema_cols_by_table.contains_key(t));
            let extra_not_in_schema = extra_tables
                .iter()
                .all(|t| !schema_cols_by_table.contains_key(t));
            if missing_in_schema && extra_not_in_schema {
                return WrongSourceTag::SubqueryPassthrough;
            }
            return WrongSourceTag::WrongTable;
        }
    }

    // --- 3. EXTRA AGG INPUT ---
    // We include extra columns from aggregate or window function arguments.
    if !extra.is_empty() && missing.is_empty() && sql_has_agg_or_window(sql_up) {
        // Extra sources from same table as expected — likely agg input columns
        let expected_tables: BTreeSet<String> =
            expected_sources.iter().map(|s| table_of(s)).collect();
        let extra_from_same_table = extra
            .iter()
            .filter(|s| expected_tables.contains(&table_of(s)))
            .count();
        if extra_from_same_table > 0 {
            return WrongSourceTag::ExtraAggInput;
        }
    }

    // --- 4. CASE WHEN BRANCH ---
    // Difference related to CASE WHEN / IFF conditional tracing.
    if sql_has_case(sql_up) {
        let col_up = col_name.to_uppercase();
        // Heuristic: check if extra/missing source columns appear in WHEN conditions
        let relevant = extra.iter().chain(missing.iter()).any(|src| {
            let src_col = col_of(src);
            if src_col.is_empty() {
                return false;
            }
            // Check if this column appears between WHEN and THEN
            let mut found = false;
            let mut pos = 0;
            while let Some(when_pos) = sql_up[pos..].find("WHEN ") {
                let abs_when = pos + when_pos;
                if let Some(then_off) = sql_up[abs_when..].find(" THEN ") {
                    let span = &sql_up[abs_when..abs_when + then_off];
                    if span.contains(&src_col) {
                        found = true;
                        break;
                    }
                    pos = abs_when + then_off + 6;
                } else {
                    break;
                }
            }
            // Also check IFF conditions: IFF(cond, val, else)
            if !found {
                let iff_pat = "IFF(";
                let mut iff_pos = 0;
                while let Some(idx) = sql_up[iff_pos..].find(iff_pat) {
                    let abs_idx = iff_pos + idx + iff_pat.len();
                    // Rough: take text until first comma as condition
                    if let Some(comma) = sql_up[abs_idx..].find(',') {
                        let cond = &sql_up[abs_idx..abs_idx + comma];
                        if cond.contains(&src_col) {
                            found = true;
                            break;
                        }
                    }
                    iff_pos = abs_idx;
                }
            }
            let _ = col_up; // suppress unused warning
            found
        });
        if relevant {
            return WrongSourceTag::CaseWhenBranch;
        }
    }

    // --- 5. SUBQUERY PASSTHROUGH ---
    // Source comes from a subquery or CTE that we don't trace through correctly.
    if sql_has_subquery(sql_up) || sql_has_cte(sql_up) {
        // Check if missing sources reference tables that are in the schema but
        // actual references a CTE/alias name instead
        if !missing.is_empty() {
            let missing_tables: BTreeSet<String> = missing.iter().map(|s| table_of(s)).collect();
            let actual_tables: BTreeSet<String> =
                actual_sources.iter().map(|s| table_of(s)).collect();
            // If any missing table is a real schema table and actual references
            // something else (CTE alias), it's subquery passthrough
            let any_missing_is_base_table = missing_tables
                .iter()
                .any(|t| schema_cols_by_table.contains_key(t));
            let any_actual_is_not_base = actual_tables
                .iter()
                .any(|t| !schema_cols_by_table.contains_key(t));
            if any_missing_is_base_table && any_actual_is_not_base {
                return WrongSourceTag::SubqueryPassthrough;
            }
        }
        // Superset case: extra sources from a CTE/alias we expanded incorrectly
        if !extra.is_empty() && missing.is_empty() {
            let extra_tables: BTreeSet<String> = extra.iter().map(|s| table_of(s)).collect();
            let any_extra_not_in_schema = extra_tables
                .iter()
                .any(|t| !schema_cols_by_table.contains_key(t));
            if any_extra_not_in_schema {
                return WrongSourceTag::SubqueryPassthrough;
            }
        }
    }

    // --- 6. Fallback: classify extra-from-different-table as subquery if CTE/subquery ---
    if !extra.is_empty() && missing.is_empty() {
        let expected_tables: BTreeSet<String> =
            expected_sources.iter().map(|s| table_of(s)).collect();
        let all_extra_from_diff_table = extra
            .iter()
            .all(|s| !expected_tables.contains(&table_of(s)));
        if all_extra_from_diff_table && (sql_has_cte(sql_up) || sql_has_subquery(sql_up)) {
            return WrongSourceTag::SubqueryPassthrough;
        }
    }

    WrongSourceTag::Other
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    if !fixture_path.exists() {
        eprintln!("Ground-truth fixture not found: {}", fixture_path.display());
        eprintln!("Expected at tests/fixtures/lineage/snowflake_ground_truth.jsonl");
        std::process::exit(1);
    }

    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader = std::io::BufReader::with_capacity(2 * 1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    let start = std::time::Instant::now();

    // --- Counters ---
    let mut total = 0u64;
    let mut kind_counts: BTreeMap<FailureKind, u64> = BTreeMap::new();

    // --- WRONG_SOURCES tag tracking ---
    // Per-column tag counts
    let mut ws_tag_col_counts: BTreeMap<WrongSourceTag, u64> = BTreeMap::new();
    // Per-query: tag → count of queries whose DOMINANT tag is this
    let mut ws_tag_query_counts: BTreeMap<WrongSourceTag, u64> = BTreeMap::new();
    // Samples per tag (capped)
    let mut ws_tag_samples: BTreeMap<WrongSourceTag, Vec<TagSample>> = BTreeMap::new();
    let ws_sample_limit = 10usize;

    // --- MIXED pattern tracking ---
    let mut mixed_pattern_counts: BTreeMap<MixedPattern, u64> = BTreeMap::new();
    let mut mixed_samples: BTreeMap<MixedPattern, Vec<MixedSample>> = BTreeMap::new();
    let mixed_sample_limit = 5usize;

    // --- Extra source reference counts (for wrong_sources superset) ---
    let mut extra_source_refs: BTreeMap<String, u64> = BTreeMap::new();
    // --- Missing source reference counts (for wrong_sources subset) ---
    let mut missing_source_refs: BTreeMap<String, u64> = BTreeMap::new();

    // --- Extra output column name counts ---
    let mut extra_col_name_counts: BTreeMap<String, u64> = BTreeMap::new();
    // --- Missing output column name counts ---
    let mut missing_col_name_counts: BTreeMap<String, u64> = BTreeMap::new();

    // --- Per wrong_sources query: track the combo of tags across its columns ---
    let mut ws_tag_combos: BTreeMap<BTreeSet<WrongSourceTag>, u64> = BTreeMap::new();

    // --- "Queries fixable if we fixed tag X" ---
    // A query is "fixable by X" if ALL its wrong-source columns have tag X.
    let mut queries_fixable_by: BTreeMap<WrongSourceTag, u64> = BTreeMap::new();

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let schema = schema_from_json(&case.schema);
        let schema_cols_by_table = schema_columns(&schema);
        let all_schema_cols = all_schema_columns_flat(&schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => {
                *kind_counts.entry(FailureKind::ParseError).or_default() += 1;
                continue;
            }
        };

        let actual = normalize_to_short_names(&result.column_dict);
        let expected = normalize(&case.expected);

        // Exact match?
        if actual == expected {
            *kind_counts.entry(FailureKind::ExactMatch).or_default() += 1;
            continue;
        }

        // Empty actual?
        if actual.is_empty() && !expected.is_empty() {
            *kind_counts.entry(FailureKind::EmptyActual).or_default() += 1;
            continue;
        }

        // Classify top-level
        let has_extra_cols = actual.keys().any(|k| !expected.contains_key(k));
        let has_missing_cols = expected.keys().any(|k| !actual.contains_key(k));
        let has_wrong_sources = expected.keys().any(|k| {
            actual
                .get(k)
                .map_or(false, |act| act != expected.get(k).unwrap())
        });

        let kind = match (has_extra_cols, has_missing_cols, has_wrong_sources) {
            (true, false, false) => FailureKind::ExtraColsOnly,
            (false, true, false) => FailureKind::MissingColsOnly,
            (false, false, true) => FailureKind::WrongSourcesOnly,
            _ => FailureKind::Mixed,
        };
        *kind_counts.entry(kind).or_default() += 1;

        let sql_up = case.sql.to_uppercase();

        // --- Track extra/missing column names ---
        for col in actual.keys() {
            if !expected.contains_key(col) {
                *extra_col_name_counts.entry(col.clone()).or_default() += 1;
            }
        }
        for col in expected.keys() {
            if !actual.contains_key(col) {
                *missing_col_name_counts.entry(col.clone()).or_default() += 1;
            }
        }

        // --- WRONG_SOURCES analysis (for pure wrong_sources and mixed) ---
        if has_wrong_sources {
            let mut query_tags: BTreeSet<WrongSourceTag> = BTreeSet::new();

            for (col, exp_sources) in &expected {
                let act_sources = match actual.get(col) {
                    Some(s) if s != exp_sources => s,
                    _ => continue,
                };

                // Track extra/missing source refs
                for extra in act_sources.difference(exp_sources) {
                    *extra_source_refs.entry(extra.clone()).or_default() += 1;
                }
                for missing in exp_sources.difference(act_sources) {
                    *missing_source_refs.entry(missing.clone()).or_default() += 1;
                }

                // Classify this column's source diff
                let tag = classify_source_diff(
                    col,
                    exp_sources,
                    act_sources,
                    &sql_up,
                    &schema_cols_by_table,
                    &all_schema_cols,
                );

                *ws_tag_col_counts.entry(tag.clone()).or_default() += 1;
                query_tags.insert(tag.clone());

                // Collect sample
                let samples = ws_tag_samples.entry(tag).or_default();
                if samples.len() < ws_sample_limit {
                    samples.push(TagSample {
                        name: case.name.clone(),
                        sql_preview: sql_preview(&case.sql),
                        col: col.clone(),
                        expected_sources: exp_sources.iter().cloned().collect(),
                        actual_sources: act_sources.iter().cloned().collect(),
                    });
                }
            }

            // Dominant tag for this query (most frequent, or first alphabetically)
            if !query_tags.is_empty() {
                // If all columns have the same tag, that's the dominant one
                if query_tags.len() == 1 {
                    let tag = query_tags.iter().next().unwrap().clone();
                    *ws_tag_query_counts.entry(tag.clone()).or_default() += 1;
                    *queries_fixable_by.entry(tag).or_default() += 1;
                } else {
                    // Multiple tags: use "Other" as dominant, but track the combo
                    *ws_tag_query_counts
                        .entry(WrongSourceTag::Other)
                        .or_default() += 1;
                }
                *ws_tag_combos.entry(query_tags).or_default() += 1;
            }
        }

        // --- MIXED pattern analysis ---
        if kind == FailureKind::Mixed {
            let has_extra_srcs = expected.keys().any(|k| {
                actual.get(k).map_or(false, |act| {
                    let exp = expected.get(k).unwrap();
                    act.difference(exp).next().is_some()
                })
            });
            let has_missing_srcs = expected.keys().any(|k| {
                actual.get(k).map_or(false, |act| {
                    let exp = expected.get(k).unwrap();
                    exp.difference(act).next().is_some()
                })
            });

            let pattern = MixedPattern {
                has_extra_cols,
                has_missing_cols,
                has_extra_sources: has_extra_srcs,
                has_missing_sources: has_missing_srcs,
            };
            *mixed_pattern_counts.entry(pattern.clone()).or_default() += 1;

            let samples = mixed_samples.entry(pattern).or_default();
            if samples.len() < mixed_sample_limit {
                let num_extra = actual.keys().filter(|k| !expected.contains_key(*k)).count();
                let num_missing = expected.keys().filter(|k| !actual.contains_key(*k)).count();
                let num_wrong = expected
                    .keys()
                    .filter(|k| {
                        actual
                            .get(*k)
                            .map_or(false, |a| a != expected.get(*k).unwrap())
                    })
                    .count();
                samples.push(MixedSample {
                    name: case.name.clone(),
                    sql_preview: sql_preview(&case.sql),
                    num_extra_cols: num_extra,
                    num_missing_cols: num_missing,
                    num_wrong_source_cols: num_wrong,
                });
            }
        }

        if total % 2000 == 0 {
            eprint!("\r  processed {} / ~19875 ...", total);
        }
    }

    let elapsed = start.elapsed();
    eprintln!(
        "\r  done. {total} queries in {:.1}s                    ",
        elapsed.as_secs_f64()
    );

    // =========================================================================
    // OUTPUT
    // =========================================================================

    let pct = |n: u64| -> f64 {
        if total == 0 {
            0.0
        } else {
            n as f64 / total as f64 * 100.0
        }
    };

    // --- 1. Top-level category table ---
    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  FAILURE ANALYSIS — Snowflake Ground Truth ({total} queries)");
    eprintln!("=====================================================================");
    eprintln!();
    eprintln!("  {:<22} {:>7} {:>8}", "Category", "Count", "%");
    eprintln!("  {:-<22} {:-<7} {:-<8}", "", "", "");

    let display_order = [
        FailureKind::ExactMatch,
        FailureKind::ExtraColsOnly,
        FailureKind::MissingColsOnly,
        FailureKind::WrongSourcesOnly,
        FailureKind::Mixed,
        FailureKind::EmptyActual,
        FailureKind::ParseError,
    ];
    for kind in &display_order {
        let n = *kind_counts.get(kind).unwrap_or(&0);
        eprintln!("  {:<22} {:>7} {:>7.1}%", kind.label(), n, pct(n));
    }

    let failures = total
        - kind_counts
            .get(&FailureKind::ExactMatch)
            .copied()
            .unwrap_or(0);
    eprintln!();
    eprintln!(
        "  Failures total:        {:>7} ({:.1}%)",
        failures,
        pct(failures)
    );

    // Shared tag display order
    let tag_order = [
        WrongSourceTag::PhantomColumn,
        WrongSourceTag::WrongTable,
        WrongSourceTag::ExtraAggInput,
        WrongSourceTag::CaseWhenBranch,
        WrongSourceTag::SubqueryPassthrough,
        WrongSourceTag::Other,
    ];

    // --- 2. WRONG_SOURCES tag analysis (per-column) ---
    let ws_col_total: u64 = ws_tag_col_counts.values().sum();
    if ws_col_total > 0 {
        eprintln!();
        eprintln!("=====================================================================");
        eprintln!("  WRONG_SOURCES — Per-Column Classification ({ws_col_total} columns)");
        eprintln!("=====================================================================");
        eprintln!();
        eprintln!(
            "  {:<24} {:>7} {:>8}  {}",
            "Tag", "Cols", "%", "Description"
        );
        eprintln!("  {:-<24} {:-<7} {:-<8}  {:-<50}", "", "", "", "");

        let ws_pct = |n: u64| -> f64 {
            if ws_col_total == 0 {
                0.0
            } else {
                n as f64 / ws_col_total as f64 * 100.0
            }
        };

        for tag in &tag_order {
            let n = *ws_tag_col_counts.get(tag).unwrap_or(&0);
            eprintln!(
                "  {:<24} {:>7} {:>7.1}%  {}",
                tag.label(),
                n,
                ws_pct(n),
                tag.description()
            );
        }
    }

    // --- 3. WRONG_SOURCES "fixable by" analysis ---
    let ws_query_total: u64 = ws_tag_query_counts.values().sum();
    if ws_query_total > 0 {
        eprintln!();
        eprintln!("---------------------------------------------------------------------");
        eprintln!(
            "  WRONG_SOURCES — Queries Fixable by Single Tag ({ws_query_total} queries with wrong sources)"
        );
        eprintln!("---------------------------------------------------------------------");
        eprintln!();
        eprintln!("  If ALL wrong-source columns in a query have the SAME tag,");
        eprintln!("  fixing that tag would make the query exact-match (modulo other issues).");
        eprintln!();
        eprintln!("  {:<24} {:>7}  {}", "Tag", "Queries", "Impact");
        eprintln!("  {:-<24} {:-<7}  {:-<30}", "", "", "");

        for tag in &tag_order {
            let n = *queries_fixable_by.get(tag).unwrap_or(&0);
            if n > 0 {
                eprintln!(
                    "  {:<24} {:>7}  Fix {tag_label} → {n} queries become exact",
                    tag.label(),
                    n,
                    tag_label = tag.label(),
                );
            }
        }
    }

    // --- 4. Tag combo analysis for wrong_sources ---
    if !ws_tag_combos.is_empty() {
        eprintln!();
        eprintln!("---------------------------------------------------------------------");
        eprintln!("  WRONG_SOURCES — Tag Combinations per Query");
        eprintln!("---------------------------------------------------------------------");
        eprintln!();

        let mut combo_sorted: Vec<_> = ws_tag_combos.iter().collect();
        combo_sorted.sort_by(|a, b| b.1.cmp(a.1));
        for (combo, count) in combo_sorted.iter().take(15) {
            let labels: Vec<&str> = combo.iter().map(|t| t.label()).collect();
            eprintln!("  {:>5}  {}", count, labels.join(" + "));
        }
    }

    // --- 5. Top extra/missing source references ---
    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  TOP EXTRA SOURCE REFERENCES (we produce but shouldn't)");
    eprintln!("=====================================================================");
    eprintln!();

    let mut extra_sorted: Vec<_> = extra_source_refs.iter().collect();
    extra_sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (ref_str, count) in extra_sorted.iter().take(25) {
        eprintln!("  {:>5}  {}", count, ref_str);
    }

    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  TOP MISSING SOURCE REFERENCES (expected but we don't produce)");
    eprintln!("=====================================================================");
    eprintln!();

    let mut missing_sorted: Vec<_> = missing_source_refs.iter().collect();
    missing_sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (ref_str, count) in missing_sorted.iter().take(25) {
        eprintln!("  {:>5}  {}", count, ref_str);
    }

    // --- 6. Top extra/missing output column names ---
    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  TOP EXTRA OUTPUT COLUMNS (we emit but shouldn't)");
    eprintln!("=====================================================================");
    eprintln!();

    let mut extra_cols_sorted: Vec<_> = extra_col_name_counts.iter().collect();
    extra_cols_sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (name, count) in extra_cols_sorted.iter().take(20) {
        eprintln!("  {:>5}  {}", count, name);
    }

    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  TOP MISSING OUTPUT COLUMNS (expected but we don't emit)");
    eprintln!("=====================================================================");
    eprintln!();

    let mut missing_cols_sorted: Vec<_> = missing_col_name_counts.iter().collect();
    missing_cols_sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (name, count) in missing_cols_sorted.iter().take(20) {
        eprintln!("  {:>5}  {}", count, name);
    }

    // --- 7. MIXED pattern analysis ---
    if !mixed_pattern_counts.is_empty() {
        eprintln!();
        eprintln!("=====================================================================");
        eprintln!("  MIXED — Co-occurring Failure Patterns");
        eprintln!("=====================================================================");
        eprintln!();

        let mut mixed_sorted: Vec<_> = mixed_pattern_counts.iter().collect();
        mixed_sorted.sort_by(|a, b| b.1.cmp(a.1));

        eprintln!("  {:>7}  {}", "Count", "Pattern");
        eprintln!("  {:-<7}  {:-<50}", "", "");

        for (pattern, count) in mixed_sorted.iter().take(10) {
            eprintln!("  {:>7}  {}", count, pattern.label());
        }

        // Show samples for top mixed patterns
        eprintln!();
        for (pattern, count) in mixed_sorted.iter().take(3) {
            eprintln!("  ---- {} ({} queries) ----", pattern.label(), count);
            if let Some(samples) = mixed_samples.get(*pattern) {
                for (i, s) in samples.iter().take(3).enumerate() {
                    eprintln!();
                    eprintln!("    {}. {}", i + 1, s.name);
                    eprintln!(
                        "       extra_cols={} missing_cols={} wrong_sources={}",
                        s.num_extra_cols, s.num_missing_cols, s.num_wrong_source_cols
                    );
                    eprintln!("       sql: {}", s.sql_preview);
                }
            }
            eprintln!();
        }
    }

    // --- 8. WRONG_SOURCES samples with full diffs ---
    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  WRONG_SOURCES — Top 10 Samples per Tag");
    eprintln!("=====================================================================");

    for tag in &tag_order {
        let n = *ws_tag_col_counts.get(tag).unwrap_or(&0);
        if n == 0 {
            continue;
        }

        eprintln!();
        eprintln!("  ---- {} ({} columns) ----", tag.label(), n);

        if let Some(samples) = ws_tag_samples.get(tag) {
            for (i, s) in samples.iter().enumerate() {
                eprintln!();
                eprintln!("    {}. {} (col: {})", i + 1, s.name, s.col);
                let extra: Vec<&String> = s
                    .actual_sources
                    .iter()
                    .filter(|a| !s.expected_sources.contains(a))
                    .collect();
                let missing: Vec<&String> = s
                    .expected_sources
                    .iter()
                    .filter(|e| !s.actual_sources.contains(e))
                    .collect();
                if !extra.is_empty() {
                    eprintln!("       +extra:   {:?}", extra);
                }
                if !missing.is_empty() {
                    eprintln!("       -missing: {:?}", missing);
                }
                eprintln!("       sql: {}", s.sql_preview);
            }
        }
    }

    // --- 9. Actionable summary ---
    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  ACTIONABLE SUMMARY — What to Fix First");
    eprintln!("=====================================================================");
    eprintln!();

    // Build a ranked list of fixes by potential query impact
    struct FixCandidate {
        label: String,
        queries: u64,
        columns: u64,
        action: String,
    }

    let mut candidates: Vec<FixCandidate> = Vec::new();

    // Extra cols only → lenient match (easy win)
    let extra_only = *kind_counts.get(&FailureKind::ExtraColsOnly).unwrap_or(&0);
    if extra_only > 0 {
        candidates.push(FixCandidate {
            label: "Suppress extra output columns".into(),
            queries: extra_only,
            columns: 0,
            action: "Filter out ROW_NUMBER/RANK partition cols, FLATTEN pseudo-cols, \
                     and Hightouch metadata from output"
                .into(),
        });
    }

    // Missing cols only
    let missing_only = *kind_counts.get(&FailureKind::MissingColsOnly).unwrap_or(&0);
    if missing_only > 0 {
        candidates.push(FixCandidate {
            label: "Fix missing output columns".into(),
            queries: missing_only,
            columns: 0,
            action: "Improve star expansion, expression alias tracing, INSERT positional \
                     column mapping"
                .into(),
        });
    }

    // Wrong sources by tag
    for tag in &tag_order {
        let q = *queries_fixable_by.get(tag).unwrap_or(&0);
        let c = *ws_tag_col_counts.get(tag).unwrap_or(&0);
        if q > 0 || c > 0 {
            candidates.push(FixCandidate {
                label: format!("Fix {}", tag.label()),
                queries: q,
                columns: c,
                action: tag.description().into(),
            });
        }
    }

    // Empty actual
    let empty = *kind_counts.get(&FailureKind::EmptyActual).unwrap_or(&0);
    if empty > 0 {
        candidates.push(FixCandidate {
            label: "Fix empty output".into(),
            queries: empty,
            columns: 0,
            action: "Deep CTE chains, complex subqueries returning zero edges".into(),
        });
    }

    candidates.sort_by(|a, b| b.queries.cmp(&a.queries));

    eprintln!(
        "  {:>5}  {:>5}  {:<35}  {}",
        "Qrys", "Cols", "Fix", "Action"
    );
    eprintln!("  {:-<5}  {:-<5}  {:-<35}  {:-<50}", "", "", "", "");

    for c in &candidates {
        eprintln!(
            "  {:>5}  {:>5}  {:<35}  {}",
            c.queries, c.columns, c.label, c.action
        );
    }

    eprintln!();
    eprintln!("=====================================================================");
    eprintln!("  Done in {:.1}s", elapsed.as_secs_f64());
    eprintln!("=====================================================================");
}
