//! Find cases where our engine has extra sources that are correct
//! (ground truth is incomplete because referenced tables aren't in schema).
//!
//! Pattern: we emit sources from tables NOT in the provided schema,
//! meaning Snowflake ACCESS_USAGE couldn't trace them.

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use std::collections::{BTreeSet, HashMap, HashSet};
use std::io::BufRead;

#[derive(serde::Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

fn schema_tables(schema: &serde_json::Value) -> HashSet<String> {
    let mut tables = HashSet::new();
    if let serde_json::Value::Object(map) = schema {
        for k in map.keys() {
            tables.insert(k.to_uppercase());
        }
    }
    tables
}

fn extract_table_from_ref(source_ref: &str) -> Option<String> {
    // "TABLE"."COL" → TABLE
    let stripped = source_ref.replace('"', "");
    let parts: Vec<&str> = stripped.split('.').collect();
    if parts.len() >= 2 {
        Some(parts[parts.len() - 2].to_uppercase())
    } else {
        None
    }
}

fn main() {
    let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");
    let file = std::fs::File::open(&fixture_path).unwrap();
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    let mut total = 0;
    let mut superset_cases = 0;
    let mut extra_from_non_schema = 0; // All extra sources are from non-schema tables
    let mut extra_from_schema = 0; // Some extra sources ARE from schema tables
    let mut extra_mixed = 0; // Mix of schema and non-schema
    let mut correct_superset_names: Vec<String> = Vec::new();
    let mut wrong_superset_names: Vec<String> = Vec::new();
    let mut sample_correct: Vec<(String, String, Vec<String>, Vec<String>)> = Vec::new();
    let mut sample_wrong: Vec<(String, String, Vec<String>, Vec<String>)> = Vec::new();

    for line in reader.lines() {
        let line = line.unwrap();
        if line.trim().is_empty() {
            continue;
        }
        let case: Case = match serde_json::from_str(&line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let schema_map: HashMap<String, serde_json::Value> =
            match serde_json::from_value(case.schema.clone()) {
                Ok(s) => s,
                Err(_) => continue,
            };
        let schema_table_set = schema_tables(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema_map, None) {
            Ok(r) => r,
            Err(_) => continue,
        };

        // Normalize for comparison
        let actual: HashMap<String, BTreeSet<String>> = result
            .column_dict
            .iter()
            .map(|(k, v)| (k.clone(), v.iter().cloned().collect()))
            .collect();
        let expected: HashMap<String, BTreeSet<String>> = case
            .expected
            .iter()
            .map(|(k, v)| (k.clone(), v.iter().cloned().collect()))
            .collect();

        // Check if this is a superset case (we have extra sources but no missing ones)
        let mut is_superset = false;
        let mut all_extra_from_non_schema = true;
        let mut any_extra_from_schema = false;
        let mut all_extra: Vec<String> = Vec::new();
        let mut has_missing = false;

        for (col, exp_sources) in &expected {
            if let Some(act_sources) = actual.get(col) {
                let extra: Vec<String> = act_sources.difference(exp_sources).cloned().collect();
                let missing: Vec<String> = exp_sources.difference(act_sources).cloned().collect();
                if !missing.is_empty() {
                    has_missing = true;
                }
                if !extra.is_empty() {
                    is_superset = true;
                    for e in &extra {
                        if let Some(table) = extract_table_from_ref(e) {
                            if schema_table_set.contains(&table) {
                                any_extra_from_schema = true;
                                all_extra_from_non_schema = false;
                            }
                        }
                        all_extra.push(e.clone());
                    }
                }
            }
        }
        // Check for extra output columns
        for col in actual.keys() {
            if !expected.contains_key(col) {
                has_missing = true; // treat as "not pure superset"
            }
        }

        if is_superset && !has_missing {
            superset_cases += 1;
            if all_extra_from_non_schema {
                extra_from_non_schema += 1;
                correct_superset_names.push(case.name.clone());
                if sample_correct.len() < 10 {
                    let exp_cols: Vec<String> = expected.keys().cloned().collect();
                    sample_correct.push((
                        case.name.clone(),
                        String::new(),
                        exp_cols,
                        all_extra.clone(),
                    ));
                }
            } else if any_extra_from_schema && all_extra_from_non_schema {
                extra_mixed += 1;
            } else {
                extra_from_schema += 1;
                wrong_superset_names.push(case.name.clone());
                if sample_wrong.len() < 10 {
                    let exp_cols: Vec<String> = expected.keys().cloned().collect();
                    sample_wrong.push((
                        case.name.clone(),
                        String::new(),
                        exp_cols,
                        all_extra.clone(),
                    ));
                }
            }
        }

        if total % 5000 == 0 {
            eprintln!("  ... processed {total}");
        }
    }

    println!("=== Superset Analysis (extra sources, no missing sources) ===");
    println!("Total cases: {total}");
    println!("Pure superset cases: {superset_cases}");
    println!();
    println!("Extra sources ALL from non-schema tables: {extra_from_non_schema}");
    println!("  → These are likely CORRECT (ground truth is incomplete)");
    println!();
    println!("Extra sources from SCHEMA tables: {extra_from_schema}");
    println!("  → These may be engine bugs (extra sources from known tables)");
    println!();
    println!("Mixed: {extra_mixed}");
    println!();

    println!("--- Samples: correct supersets (extra from non-schema tables) ---");
    for (name, _, _, extras) in &sample_correct {
        println!("  {name}");
        println!("    extra: {extras:?}");
    }

    println!();
    println!("--- Samples: possibly wrong supersets (extra from schema tables) ---");
    for (name, _, _, extras) in &sample_wrong {
        println!("  {name}");
        println!("    extra: {extras:?}");
    }

    println!();
    println!(
        "--- All correct superset case names ({}) ---",
        correct_superset_names.len()
    );
    for n in &correct_superset_names {
        println!("  {n}");
    }
}
