//! Deep dive on "extra output columns" failures.
//! These are queries where ALL expected columns match perfectly,
//! but we produce additional output columns not in the expected set.
//!
//! Run: cargo run --example extra_cols_deep_dive --release 2>&1 | head -500

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::HashMap;
use std::io::BufRead;
use std::path::Path;

#[derive(Debug, Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

fn schema_map(v: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match v {
        serde_json::Value::Object(m) => m.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut s = v.clone();
            s.sort();
            s.dedup();
            (k.clone(), s)
        })
        .collect()
}

fn short_names(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let n: Vec<String> = v
                .iter()
                .map(|r| {
                    let parts: Vec<&str> = r.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        format!(
                            "\"{}\".\"{}\"\n",
                            parts[parts.len() - 2],
                            parts[parts.len() - 1]
                        )
                        .trim()
                        .to_string()
                    } else {
                        r.clone()
                    }
                })
                .collect();
            let mut s = n;
            s.sort();
            s.dedup();
            (k.clone(), s)
        })
        .collect()
}

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");
    let file = std::fs::File::open(&fixture_path).expect("open");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    let mut only_extra_count = 0u64;
    let mut extra_col_patterns: HashMap<String, u64> = HashMap::new();
    let mut extra_col_source_counts: HashMap<String, u64> = HashMap::new(); // "has_sources" vs "empty_sources"
    let mut printed = 0u32;

    for line in reader.lines() {
        let line = line.expect("read");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }
        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };

        let schema = schema_map(&case.schema);
        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => continue,
        };

        let actual = short_names(&result.column_dict);
        let expected = normalize(&case.expected);

        if actual == expected {
            continue;
        }

        // Check: all expected cols match, but we have extras?
        let mut all_expected_match = true;
        for (col, exp_sources) in &expected {
            match actual.get(col) {
                Some(act_sources) if act_sources == exp_sources => {}
                _ => {
                    all_expected_match = false;
                    break;
                }
            }
        }
        if !all_expected_match {
            continue;
        }

        // This IS an only-extra case
        only_extra_count += 1;

        let extra_cols: Vec<&String> = actual
            .keys()
            .filter(|k| !expected.contains_key(*k))
            .collect();

        for col in &extra_cols {
            // Categorize the extra column name
            let pattern = if col.starts_with("_HT_") {
                "_HT_*".to_string()
            } else if col.starts_with("_col_") || col.starts_with("_COL_") {
                "_col_N".to_string()
            } else if col.starts_with("_") {
                format!("_{}", &col[1..col.len().min(10)])
            } else {
                col.to_string()
            };
            *extra_col_patterns.entry(pattern).or_default() += 1;

            // Check if the extra column has sources
            let sources = actual.get(*col).map(|s| s.len()).unwrap_or(0);
            let key = if sources == 0 {
                "empty_sources"
            } else {
                "has_sources"
            };
            *extra_col_source_counts.entry(key.to_string()).or_default() += 1;
        }

        // Print first 10 samples
        if printed < 10 {
            printed += 1;
            eprintln!("\n=== ONLY-EXTRA #{} [{}] ===", printed, case.name);
            eprintln!("  Expected cols: {}", expected.len());
            eprintln!("  Actual cols: {}", actual.len());
            eprintln!("  Extra cols ({}):", extra_cols.len());
            for col in extra_cols.iter().take(5) {
                let sources = actual.get(*col).cloned().unwrap_or_default();
                eprintln!("    {} → {:?}", col, sources);
            }
            if extra_cols.len() > 5 {
                eprintln!("    ... and {} more", extra_cols.len() - 5);
            }
            // Show first 200 chars of SQL
            eprintln!("  SQL: {}", &case.sql[..case.sql.len().min(300)]);
        }
    }

    eprintln!("\n============================================================");
    eprintln!("ONLY-EXTRA COLUMNS ANALYSIS");
    eprintln!("============================================================");
    eprintln!("Total only-extra queries: {only_extra_count}");

    eprintln!("\nExtra column source status:");
    let mut sc: Vec<_> = extra_col_source_counts.iter().collect();
    sc.sort_by(|a, b| b.1.cmp(a.1));
    for (k, v) in &sc {
        eprintln!("  {v:5} {k}");
    }

    eprintln!("\nExtra column name patterns (top 30):");
    let mut sorted: Vec<_> = extra_col_patterns.iter().collect();
    sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (pat, count) in sorted.iter().take(30) {
        eprintln!("  {count:5} {pat}");
    }
}
