//! Detailed column-level mismatch analysis for Snowflake ground truth lineage.
//!
//! Categorizes every output column into one of six buckets:
//!   exact | subset | superset | overlap | missing_in_actual | extra_in_actual
//!
//! Then drills into subset mismatches (same-table vs cross-table missing sources)
//! and extra_in_actual columns (top 20 column names), with INSERT/MERGE awareness.
//!
//! Run: cargo run --example detailed_analysis --release

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;

// ---------------------------------------------------------------------------
// Fixture type
// ---------------------------------------------------------------------------

#[derive(Debug, Deserialize)]
struct Case {
    #[allow(dead_code)]
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

/// Convert the nested schema JSON to the flat HashMap that the lineage API expects.
fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

/// Sort + dedup a column-lineage map (no name transformation).
fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

/// Normalize actual output to short `"TABLE"."COL"` form (trim to last 2 dot-separated parts).
fn normalize_to_short(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let normalized: Vec<String> = v
                .iter()
                .map(|ref_str| {
                    let parts: Vec<&str> =
                        ref_str.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        let table = parts[parts.len() - 2];
                        let col = parts[parts.len() - 1];
                        format!("\"{table}\".\"{col}\"")
                    } else {
                        ref_str.clone()
                    }
                })
                .collect();
            let mut sorted = normalized;
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

/// Extract the table portion from a `"TABLE"."COL"` reference (uppercased).
fn table_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    if parts.len() >= 2 {
        parts[parts.len() - 2].to_uppercase()
    } else {
        String::new()
    }
}

/// Returns true if the SQL (trimmed, uppercased) starts with INSERT or MERGE.
fn is_insert_or_merge(sql: &str) -> bool {
    let trimmed = sql.trim_start().to_uppercase();
    trimmed.starts_with("INSERT") || trimmed.starts_with("MERGE")
}

// ---------------------------------------------------------------------------
// Column-level category
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ColumnCategory {
    /// actual sources == expected sources
    Exact,
    /// actual sources are a strict subset of expected (missing some)
    Subset,
    /// actual sources are a strict superset of expected (has extras)
    Superset,
    /// partial overlap: some match, some missing, some extra
    Overlap,
    /// column exists in expected but not in actual
    MissingInActual,
    /// column exists in actual but not in expected
    ExtraInActual,
}

/// Classify a single output column given its actual and expected source sets.
fn classify_column(
    actual_sources: Option<&Vec<String>>,
    expected_sources: Option<&Vec<String>>,
) -> ColumnCategory {
    match (actual_sources, expected_sources) {
        (None, Some(_)) => ColumnCategory::MissingInActual,
        (Some(_), None) => ColumnCategory::ExtraInActual,
        (Some(act), Some(exp)) => {
            if act == exp {
                return ColumnCategory::Exact;
            }
            let act_set: HashSet<&String> = act.iter().collect();
            let exp_set: HashSet<&String> = exp.iter().collect();
            let all_exp_in_act = exp_set.is_subset(&act_set);
            let all_act_in_exp = act_set.is_subset(&exp_set);
            match (all_exp_in_act, all_act_in_exp) {
                (true, false) => ColumnCategory::Superset,
                (false, true) => ColumnCategory::Subset,
                _ => ColumnCategory::Overlap,
            }
        }
        (None, None) => ColumnCategory::Exact, // degenerate, shouldn't happen
    }
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "Ground-truth fixture not found: {}. Skipping.",
            fixture_path.display()
        );
        return;
    }

    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture file");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);

    let dialect = SqlDialect::Snowflake;

    // -----------------------------------------------------------------------
    // Counters
    // -----------------------------------------------------------------------

    let mut total_queries: u64 = 0;
    let mut parse_fail_queries: u64 = 0;

    // Per-column counters
    let mut col_exact: u64 = 0;
    let mut col_subset: u64 = 0;
    let mut col_superset: u64 = 0;
    let mut col_overlap: u64 = 0;
    let mut col_missing_in_actual: u64 = 0;
    let mut col_extra_in_actual: u64 = 0;

    // Subset drill-down: for each missing source ref, was the missing source
    // from the SAME table as at least one actual source, or a DIFFERENT table?
    let mut subset_missing_same_table: u64 = 0;
    let mut subset_missing_diff_table: u64 = 0;
    let mut subset_total_missing_refs: u64 = 0;

    // Extra-in-actual: frequency of output column names
    let mut extra_col_names: HashMap<String, u64> = HashMap::new();

    // Missing-in-actual: how many come from INSERT/MERGE queries
    let mut missing_from_insert_merge: u64 = 0;
    let mut missing_from_other: u64 = 0;

    // -----------------------------------------------------------------------
    // Stream through each fixture line
    // -----------------------------------------------------------------------

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total_queries += 1;

        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => {
                parse_fail_queries += 1;
                continue;
            }
        };

        let actual = normalize_to_short(&result.column_dict);
        let expected = normalize(&case.expected);

        let is_dml = is_insert_or_merge(&case.sql);

        // Collect all column names from both sides
        let mut all_columns: HashSet<&String> = HashSet::new();
        for k in expected.keys() {
            all_columns.insert(k);
        }
        for k in actual.keys() {
            all_columns.insert(k);
        }

        for col in &all_columns {
            let act = actual.get(*col);
            let exp = expected.get(*col);

            let category = classify_column(act, exp);

            match category {
                ColumnCategory::Exact => col_exact += 1,
                ColumnCategory::Subset => {
                    col_subset += 1;
                    // Drill into the missing sources
                    let act_sources = act.unwrap();
                    let exp_sources = exp.unwrap();
                    let actual_tables: HashSet<String> =
                        act_sources.iter().map(|s| table_of(s)).collect();

                    for exp_ref in exp_sources {
                        if !act_sources.contains(exp_ref) {
                            subset_total_missing_refs += 1;
                            let missing_table = table_of(exp_ref);
                            if actual_tables.contains(&missing_table) {
                                subset_missing_same_table += 1;
                            } else {
                                subset_missing_diff_table += 1;
                            }
                        }
                    }
                }
                ColumnCategory::Superset => col_superset += 1,
                ColumnCategory::Overlap => {
                    col_overlap += 1;
                    // Also count missing refs for the overlap case (subset portion)
                    let act_sources = act.unwrap();
                    let exp_sources = exp.unwrap();
                    let actual_tables: HashSet<String> =
                        act_sources.iter().map(|s| table_of(s)).collect();

                    for exp_ref in exp_sources {
                        if !act_sources.contains(exp_ref) {
                            subset_total_missing_refs += 1;
                            let missing_table = table_of(exp_ref);
                            if actual_tables.contains(&missing_table) {
                                subset_missing_same_table += 1;
                            } else {
                                subset_missing_diff_table += 1;
                            }
                        }
                    }
                }
                ColumnCategory::MissingInActual => {
                    col_missing_in_actual += 1;
                    if is_dml {
                        missing_from_insert_merge += 1;
                    } else {
                        missing_from_other += 1;
                    }
                }
                ColumnCategory::ExtraInActual => {
                    col_extra_in_actual += 1;
                    *extra_col_names.entry(col.to_uppercase()).or_default() += 1;
                }
            }
        }

        // Progress every 2000 queries
        if total_queries % 2000 == 0 {
            eprintln!("  ... processed {total_queries} queries");
        }
    }

    // -----------------------------------------------------------------------
    // Report
    // -----------------------------------------------------------------------

    let col_total = col_exact
        + col_subset
        + col_superset
        + col_overlap
        + col_missing_in_actual
        + col_extra_in_actual;

    let pct = |n: u64, d: u64| -> f64 {
        if d > 0 {
            n as f64 / d as f64 * 100.0
        } else {
            0.0
        }
    };

    println!();
    println!("{}", "=".repeat(74));
    println!("  DETAILED COLUMN-LEVEL MISMATCH ANALYSIS");
    println!("{}", "=".repeat(74));

    println!();
    println!("--- Query-Level ---");
    println!("Total queries:       {total_queries}");
    println!("Parse failures:      {parse_fail_queries}");
    println!(
        "Analyzed:            {}",
        total_queries - parse_fail_queries
    );

    println!();
    println!("--- Per-Column Category Breakdown ({col_total} columns) ---");
    println!(
        "  exact:              {:>7}  ({:5.1}%)  actual == expected",
        col_exact,
        pct(col_exact, col_total)
    );
    println!(
        "  subset:             {:>7}  ({:5.1}%)  actual is strict subset of expected (missing some)",
        col_subset,
        pct(col_subset, col_total)
    );
    println!(
        "  superset:           {:>7}  ({:5.1}%)  actual is strict superset of expected (has extras)",
        col_superset,
        pct(col_superset, col_total)
    );
    println!(
        "  overlap:            {:>7}  ({:5.1}%)  partial overlap (some match, some missing, some extra)",
        col_overlap,
        pct(col_overlap, col_total)
    );
    println!(
        "  missing_in_actual:  {:>7}  ({:5.1}%)  column in expected but NOT in actual",
        col_missing_in_actual,
        pct(col_missing_in_actual, col_total)
    );
    println!(
        "  extra_in_actual:    {:>7}  ({:5.1}%)  column in actual but NOT in expected",
        col_extra_in_actual,
        pct(col_extra_in_actual, col_total)
    );

    // -----------------------------------------------------------------------
    // Subset drill-down: same table vs cross-table
    // -----------------------------------------------------------------------

    println!();
    println!("{}", "-".repeat(74));
    println!("  SUBSET DRILL-DOWN: Missing Source References");
    println!("{}", "-".repeat(74));
    println!(
        "Total missing source refs (from subset + overlap columns): {subset_total_missing_refs}"
    );
    println!(
        "  Same table as an actual source:   {:>7}  ({:5.1}%)",
        subset_missing_same_table,
        pct(subset_missing_same_table, subset_total_missing_refs)
    );
    println!(
        "  Different table (cross-table):    {:>7}  ({:5.1}%)",
        subset_missing_diff_table,
        pct(subset_missing_diff_table, subset_total_missing_refs)
    );

    // -----------------------------------------------------------------------
    // Extra-in-actual: top 20 column names
    // -----------------------------------------------------------------------

    println!();
    println!("{}", "-".repeat(74));
    println!("  EXTRA-IN-ACTUAL: Top 20 Output Column Names");
    println!("{}", "-".repeat(74));

    let mut extra_sorted: Vec<_> = extra_col_names.into_iter().collect();
    extra_sorted.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));

    for (i, (name, count)) in extra_sorted.iter().take(20).enumerate() {
        println!("  {:>2}. {:<50} {:>6}", i + 1, name, count);
    }

    if extra_sorted.len() > 20 {
        let remaining: u64 = extra_sorted.iter().skip(20).map(|(_, c)| c).sum();
        println!(
            "      ... and {} more distinct names ({} occurrences)",
            extra_sorted.len() - 20,
            remaining
        );
    }

    // -----------------------------------------------------------------------
    // Missing-in-actual: INSERT/MERGE breakdown
    // -----------------------------------------------------------------------

    println!();
    println!("{}", "-".repeat(74));
    println!("  MISSING-IN-ACTUAL: INSERT/MERGE Breakdown");
    println!("{}", "-".repeat(74));
    println!(
        "  From INSERT/MERGE queries:  {:>7}  ({:5.1}% of missing_in_actual)",
        missing_from_insert_merge,
        pct(missing_from_insert_merge, col_missing_in_actual)
    );
    println!(
        "  From other queries:         {:>7}  ({:5.1}% of missing_in_actual)",
        missing_from_other,
        pct(missing_from_other, col_missing_in_actual)
    );

    println!();
    println!("=== Done ({total_queries} queries, {col_total} columns analyzed) ===");
}
