//! Diagnostic: categorize ground truth failures and detect view resolution cases.
//!
//! Key question: Does the ground truth reference tables NOT mentioned in the SQL?
//! If yes, Snowflake resolved through views — we can't match without view definitions.
//!
//! Run: cargo run --example failure_diagnosis --release 2>&1 | tee /tmp/failure_diagnosis.txt
#![allow(unused)]

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{BTreeSet, HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;

#[derive(Debug, Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
    #[allow(dead_code)]
    query_hash: Option<String>,
    #[allow(dead_code)]
    source_query_type: Option<String>,
}

fn schema_map(v: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match v {
        serde_json::Value::Object(m) => m.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut s = v.clone();
            s.sort();
            s.dedup();
            (k.clone(), s)
        })
        .collect()
}

fn short_names(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let n: Vec<String> = v
                .iter()
                .map(|r| {
                    let parts: Vec<&str> = r.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        let t = parts[parts.len() - 2];
                        let c = parts[parts.len() - 1];
                        format!("\"{t}\".\"{c}\"")
                    } else {
                        r.clone()
                    }
                })
                .collect();
            let mut s = n;
            s.sort();
            s.dedup();
            (k.clone(), s)
        })
        .collect()
}

/// Extract table names referenced in SQL (rough heuristic: after FROM, JOIN, etc.)
fn extract_sql_tables(sql: &str) -> HashSet<String> {
    let upper = sql.to_uppercase();
    let mut tables = HashSet::new();
    // Simple regex-like extraction: tokens after FROM, JOIN, INTO
    let tokens: Vec<&str> = upper.split_whitespace().collect();
    for i in 0..tokens.len().saturating_sub(1) {
        let kw = tokens[i].trim_end_matches(',');
        if matches!(kw, "FROM" | "JOIN" | "INTO" | "TABLE" | "UPDATE") {
            let next =
                tokens[i + 1].trim_matches(|c: char| c == '(' || c == ')' || c == ',' || c == '"');
            if !next.is_empty()
                && !matches!(
                    next,
                    "(" | "SELECT" | "VALUES" | "SET" | "WHERE" | "LATERAL" | "UNNEST" | "TABLE"
                )
            {
                // Take just the last part (table name) if it has dots
                let table_name = next.split('.').last().unwrap_or(next).trim_matches('"');
                if !table_name.is_empty() {
                    tables.insert(table_name.to_string());
                }
            }
        }
    }
    tables
}

/// Extract table names from ground truth expected output
fn extract_expected_tables(expected: &HashMap<String, Vec<String>>) -> HashSet<String> {
    let mut tables = HashSet::new();
    for sources in expected.values() {
        for src in sources {
            let parts: Vec<&str> = src.split('.').map(|p| p.trim_matches('"')).collect();
            if parts.len() >= 2 {
                tables.insert(parts[parts.len() - 2].to_string());
            }
        }
    }
    tables
}

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    if !fixture_path.exists() {
        eprintln!("Fixture not found: {}", fixture_path.display());
        return;
    }

    let file = std::fs::File::open(&fixture_path).expect("open");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    let mut total = 0u64;
    let mut exact = 0u64;
    let mut failures = 0u64;

    // View resolution detection
    let mut view_resolution = 0u64; // expected refs tables NOT in SQL
    let mut no_view_resolution = 0u64; // all expected tables are in SQL
    let mut view_samples: Vec<(String, HashSet<String>, HashSet<String>, String)> = Vec::new();

    // Failure categorization (for non-view cases)
    let mut extra_cols_only = 0u64;
    let mut missing_cols_only = 0u64;
    let mut pure_superset = 0u64; // all wrong cols have EXTRA sources
    let mut pure_subset = 0u64; // all wrong cols have MISSING sources
    let mut mixed_wrong = 0u64;
    let mut empty_actual = 0u64;

    // Sub-categorize superset
    let mut superset_with_case = 0u64;
    let mut superset_no_case = 0u64;
    let mut superset_samples: Vec<(String, String, Vec<String>, Vec<String>)> = Vec::new(); // name, col, expected, actual

    // Sub-categorize subset
    let mut subset_samples: Vec<(String, String, Vec<String>, Vec<String>)> = Vec::new();

    // Extra col analysis
    let mut extra_col_names: HashMap<String, u64> = HashMap::new();
    let mut extra_cols_samples: Vec<(String, Vec<String>, String)> = Vec::new();

    // Missing col analysis
    let mut missing_col_samples: Vec<(String, Vec<String>, String)> = Vec::new();

    for line in reader.lines() {
        let line = line.expect("read");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let schema = schema_map(&case.schema);
        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => {
                failures += 1;
                empty_actual += 1;
                continue;
            }
        };

        let actual = short_names(&result.column_dict);
        let expected = normalize(&case.expected);

        if actual == expected {
            exact += 1;
            continue;
        }
        failures += 1;

        // CHECK: Do expected source references mention tables NOT in the SQL?
        let sql_tables = extract_sql_tables(&case.sql);
        let expected_tables = extract_expected_tables(&expected);
        let phantom_tables: HashSet<_> = expected_tables.difference(&sql_tables).cloned().collect();

        if !phantom_tables.is_empty() {
            view_resolution += 1;
            if view_samples.len() < 20 {
                view_samples.push((
                    case.name.clone(),
                    phantom_tables.clone(),
                    sql_tables.clone(),
                    case.sql[..case.sql.len().min(200)].to_string(),
                ));
            }
            continue; // Don't count these in other categories — they're unfixable without view defs
        }
        no_view_resolution += 1;

        // Classify the failure
        let mut n_extra_cols = 0u64;
        let mut n_missing_cols = 0u64;
        let mut n_correct_cols = 0u64;
        let mut n_superset = 0u64;
        let mut n_subset = 0u64;
        let mut n_other_wrong = 0u64;

        let mut first_superset_col: Option<(String, Vec<String>, Vec<String>)> = None;
        let mut first_subset_col: Option<(String, Vec<String>, Vec<String>)> = None;

        for (col, exp_srcs) in &expected {
            match actual.get(col) {
                Some(act_srcs) if act_srcs == exp_srcs => n_correct_cols += 1,
                Some(act_srcs) => {
                    let es: BTreeSet<_> = exp_srcs.iter().collect();
                    let as_: BTreeSet<_> = act_srcs.iter().collect();
                    if as_.is_superset(&es) {
                        n_superset += 1;
                        if first_superset_col.is_none() {
                            first_superset_col =
                                Some((col.clone(), exp_srcs.clone(), act_srcs.clone()));
                        }
                    } else if as_.is_subset(&es) {
                        n_subset += 1;
                        if first_subset_col.is_none() {
                            first_subset_col =
                                Some((col.clone(), exp_srcs.clone(), act_srcs.clone()));
                        }
                    } else {
                        n_other_wrong += 1;
                    }
                }
                None => n_missing_cols += 1,
            }
        }
        let extra_cols: Vec<String> = actual
            .keys()
            .filter(|k| !expected.contains_key(*k))
            .cloned()
            .collect();
        n_extra_cols = extra_cols.len() as u64;

        if actual.is_empty() && !expected.is_empty() {
            empty_actual += 1;
        } else if n_superset == 0
            && n_subset == 0
            && n_other_wrong == 0
            && n_missing_cols == 0
            && n_extra_cols > 0
        {
            extra_cols_only += 1;
            for c in &extra_cols {
                *extra_col_names.entry(c.clone()).or_default() += 1;
            }
            if extra_cols_samples.len() < 10 {
                extra_cols_samples.push((
                    case.name.clone(),
                    extra_cols,
                    case.sql[..case.sql.len().min(200)].to_string(),
                ));
            }
        } else if n_superset == 0
            && n_subset == 0
            && n_other_wrong == 0
            && n_extra_cols == 0
            && n_missing_cols > 0
        {
            missing_cols_only += 1;
            let missing: Vec<_> = expected
                .keys()
                .filter(|k| !actual.contains_key(*k))
                .cloned()
                .collect();
            if missing_col_samples.len() < 10 {
                missing_col_samples.push((
                    case.name.clone(),
                    missing,
                    case.sql[..case.sql.len().min(200)].to_string(),
                ));
            }
        } else if n_superset > 0 && n_subset == 0 && n_other_wrong == 0 && n_missing_cols == 0 {
            pure_superset += 1;
            if case.sql.to_uppercase().contains("CASE") {
                superset_with_case += 1;
            } else {
                superset_no_case += 1;
            }
            if let Some((col, exp, act)) = first_superset_col {
                if superset_samples.len() < 20 {
                    superset_samples.push((case.name.clone(), col, exp, act));
                }
            }
        } else if n_subset > 0 && n_superset == 0 && n_other_wrong == 0 && n_extra_cols == 0 {
            pure_subset += 1;
            if let Some((col, exp, act)) = first_subset_col {
                if subset_samples.len() < 10 {
                    subset_samples.push((case.name.clone(), col, exp, act));
                }
            }
        } else {
            mixed_wrong += 1;
        }
    }

    eprintln!("\n============================================================");
    eprintln!(
        "FAILURE DIAGNOSIS: {} failed / {} total ({:.1}% exact)",
        failures,
        total,
        exact as f64 / total as f64 * 100.0
    );
    eprintln!("============================================================\n");

    eprintln!("=== VIEW RESOLUTION (unfixable without view definitions) ===");
    eprintln!(
        "View resolution failures: {} ({:.1}% of all failures)",
        view_resolution,
        view_resolution as f64 / failures.max(1) as f64 * 100.0
    );
    for (name, phantom, sql_t, sql) in view_samples.iter().take(10) {
        eprintln!("  [{}]", name);
        eprintln!(
            "    Phantom tables (in expected, NOT in SQL): {:?}",
            phantom
        );
        eprintln!("    SQL tables: {:?}", sql_t);
        eprintln!("    SQL: {}", sql);
    }

    let fixable = failures - view_resolution;
    eprintln!(
        "\n=== FIXABLE FAILURES: {} ({:.1}% of total) ===\n",
        fixable,
        fixable as f64 / total as f64 * 100.0
    );

    eprintln!("--- EXTRA OUTPUT COLS ONLY: {} ---", extra_cols_only);
    for (name, cols, sql) in extra_cols_samples.iter().take(5) {
        eprintln!("  [{}] extra={:?}", name, cols);
        eprintln!("    SQL: {}", sql);
    }
    eprintln!("  Top extra column names:");
    let mut ecn: Vec<_> = extra_col_names.iter().collect();
    ecn.sort_by(|a, b| b.1.cmp(a.1));
    for (name, count) in ecn.iter().take(10) {
        eprintln!("    {count:5} {name}");
    }

    eprintln!("\n--- MISSING OUTPUT COLS ONLY: {} ---", missing_cols_only);
    for (name, cols, sql) in missing_col_samples.iter().take(5) {
        eprintln!("  [{}] missing={:?}", name, cols);
        eprintln!("    SQL: {}", sql);
    }

    eprintln!(
        "\n--- PURE SUPERSET (extra sources, no missing): {} ---",
        pure_superset
    );
    eprintln!(
        "  With CASE WHEN: {}  Without: {}",
        superset_with_case, superset_no_case
    );
    for (name, col, exp, act) in superset_samples.iter().take(10) {
        let exp_set: BTreeSet<_> = exp.iter().collect();
        let act_set: BTreeSet<_> = act.iter().collect();
        let extra: Vec<_> = act_set.difference(&exp_set).collect();
        eprintln!("  [{}] col={} extra_sources={:?}", name, col, extra);
    }

    eprintln!("\n--- PURE SUBSET (missing sources): {} ---", pure_subset);
    for (name, col, exp, act) in subset_samples.iter().take(5) {
        let exp_set: BTreeSet<_> = exp.iter().collect();
        let act_set: BTreeSet<_> = act.iter().collect();
        let missing: Vec<_> = exp_set.difference(&act_set).collect();
        eprintln!("  [{}] col={} missing_sources={:?}", name, col, missing);
    }

    eprintln!("\n--- EMPTY ACTUAL: {} ---", empty_actual);
    eprintln!("--- MIXED WRONG: {} ---\n", mixed_wrong);

    eprintln!("=== SUMMARY ===");
    eprintln!(
        "Exact match:     {:5} ({:.1}%)",
        exact,
        exact as f64 / total as f64 * 100.0
    );
    eprintln!("View resolution: {:5} (unfixable)", view_resolution);
    eprintln!(
        "Extra cols only: {:5} (fixable - filter output)",
        extra_cols_only
    );
    eprintln!(
        "Missing cols:    {:5} (fixable - expand SELECT *)",
        missing_cols_only
    );
    eprintln!(
        "Pure superset:   {:5} (fixable - remove extra sources)",
        pure_superset
    );
    eprintln!(
        "Pure subset:     {:5} (fixable - add missing sources)",
        pure_subset
    );
    eprintln!("Mixed/other:     {:5}", mixed_wrong);
    eprintln!("Empty actual:    {:5}", empty_actual);
}
