//! Analyze per-column missing sources — cases where we have the expected output column
//! but are missing expected source references.

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;

#[derive(Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

fn normalize_to_short(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let normalized: Vec<String> = v
                .iter()
                .map(|ref_str| {
                    let parts: Vec<&str> =
                        ref_str.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        let table = parts[parts.len() - 2];
                        let col = parts[parts.len() - 1];
                        format!("\"{table}\".\"{col}\"")
                    } else {
                        ref_str.clone()
                    }
                })
                .collect();
            let mut sorted = normalized;
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

fn col_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    parts.last().unwrap_or(&"").to_uppercase()
}

fn table_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    if parts.len() >= 2 {
        parts[parts.len() - 2].to_uppercase()
    } else {
        String::new()
    }
}

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");
    let file = std::fs::File::open(&fixture_path).expect("open");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    let mut total_missing_refs = 0u64;
    let mut total_missing_queries = 0u64;

    // Categories of missing:
    let mut missing_empty_actual = 0u64; // column exists in actual but has empty sources
    let mut missing_col_absent = 0u64; // column not in actual at all
    let mut missing_source_present = 0u64; // column in actual, but specific source ref missing

    // For missing source refs: is the missing ref's TABLE in the schema?
    let mut missing_table_in_schema = 0u64;
    let mut missing_table_not_in_schema = 0u64;

    // When column is in actual but source is missing: what does actual have?
    let mut actual_has_same_col_diff_table = 0u64; // same col from wrong table
    let mut actual_has_no_match = 0u64;

    // Missing column frequency
    let mut missing_col_freq: HashMap<String, u64> = HashMap::new();

    // Samples
    let mut samples: Vec<(String, String, Vec<String>, Vec<String>, String)> = Vec::new();

    for line in reader.lines() {
        let line = line.unwrap();
        let line = line.trim();
        if line.is_empty() {
            continue;
        }
        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };

        let schema = schema_from_json(&case.schema);
        let schema_tables: HashSet<String> = schema.keys().map(|t| t.to_uppercase()).collect();

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => continue,
        };

        let actual = normalize_to_short(&result.column_dict);
        let expected = normalize(&case.expected);

        if actual == expected {
            continue;
        }

        let mut query_has_missing = false;

        for (col, exp_sources) in &expected {
            match actual.get(col) {
                None => {
                    // Column completely absent from actual
                    missing_col_absent += exp_sources.len() as u64;
                    total_missing_refs += exp_sources.len() as u64;
                    query_has_missing = true;
                    *missing_col_freq.entry(col.clone()).or_default() += 1;
                }
                Some(act_sources) => {
                    if act_sources.is_empty() {
                        missing_empty_actual += exp_sources.len() as u64;
                        total_missing_refs += exp_sources.len() as u64;
                        query_has_missing = true;
                        continue;
                    }

                    // Check each expected source
                    for exp_ref in exp_sources {
                        if act_sources.contains(exp_ref) {
                            continue; // present
                        }

                        // This expected source is missing
                        missing_source_present += 1;
                        total_missing_refs += 1;
                        query_has_missing = true;

                        let exp_table = table_of(exp_ref);
                        let exp_col = col_of(exp_ref);

                        if schema_tables.contains(&exp_table) {
                            missing_table_in_schema += 1;
                        } else {
                            missing_table_not_in_schema += 1;
                        }

                        // Does actual have same column from different table?
                        let has_same_col = act_sources.iter().any(|a| col_of(a) == exp_col);
                        if has_same_col {
                            actual_has_same_col_diff_table += 1;
                        } else {
                            actual_has_no_match += 1;
                        }

                        if samples.len() < 10 {
                            let sql_prefix: String = case.sql.chars().take(120).collect();
                            samples.push((
                                case.name.clone(),
                                col.clone(),
                                exp_sources.clone(),
                                act_sources.clone(),
                                sql_prefix,
                            ));
                        }
                    }
                }
            }
        }

        if query_has_missing {
            total_missing_queries += 1;
        }
    }

    println!("=== Missing Source Analysis ===");
    println!("Queries with missing refs:   {total_missing_queries}");
    println!("Total missing refs:          {total_missing_refs}");
    println!();
    println!("--- Breakdown ---");
    println!("Column absent from actual:   {missing_col_absent}");
    println!("Column empty sources:        {missing_empty_actual}");
    println!("Column present, ref missing: {missing_source_present}");
    println!();
    println!("Missing ref's table in schema:    {missing_table_in_schema}");
    println!("Missing ref's table NOT in schema:{missing_table_not_in_schema}");
    println!();
    println!("Actual has same col, diff table:  {actual_has_same_col_diff_table}");
    println!("Actual has no matching column:    {actual_has_no_match}");
    println!();

    println!("=== Top 20 Missing Column Names ===");
    let mut col_vec: Vec<_> = missing_col_freq.into_iter().collect();
    col_vec.sort_by(|a, b| b.1.cmp(&a.1));
    for (col, count) in col_vec.iter().take(20) {
        println!("  {col:<40} {count:>6}");
    }

    println!();
    println!("=== Samples ===");
    for (i, (name, col, exp, act, sql)) in samples.iter().enumerate() {
        println!("--- Sample {} ---", i + 1);
        println!("  Query:    {name}");
        println!("  Column:   {col}");
        println!("  Expected: {:?}", exp);
        println!("  Actual:   {:?}", act);
        println!("  SQL:      {sql}...");
        println!();
    }
}
