//! Deep-dive into "wrong_sources — pure subset" cases.
//!
//! These are queries where every output column we produce is either an exact
//! match or a strict subset of the expected sources — meaning we are MISSING
//! source references that Snowflake ACCESS_USAGE reports.
//!
//! The analysis answers three questions:
//!   1. Which tables are we failing to trace through?
//!   2. Are those tables present in the per-case schema?
//!   3. What do representative failures look like?
//!
//! Run:  cargo run --release --example missing_sources_analysis

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

#[derive(Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

/// Normalize a column-lineage map: sort + dedup each source list.
fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

/// Extract the table name from a `"DB"."SCHEMA"."TABLE"."COL"` or
/// `"TABLE"."COL"` reference.  Returns uppercase for stable grouping.
fn table_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    if parts.len() >= 2 {
        parts[parts.len() - 2].to_uppercase()
    } else {
        String::new()
    }
}

/// Check whether `actual` is a subset-or-equal of `expected`.
fn is_subset_or_equal(actual: &[String], expected: &[String]) -> bool {
    actual.iter().all(|a| expected.contains(a))
}

// ---------------------------------------------------------------------------
// Per-case result
// ---------------------------------------------------------------------------

struct SubsetCase {
    name: String,
    sql_prefix: String,
    /// Missing `"TABLE"."COL"` references grouped by output column.
    missing_refs: Vec<(String, Vec<String>)>,
}

// ---------------------------------------------------------------------------
// Accumulators
// ---------------------------------------------------------------------------

#[derive(Default)]
struct MissingTableInfo {
    /// Number of (query, output_column) pairs where this table is missing.
    ref_count: u64,
    /// Number of distinct queries affected.
    query_names: HashSet<String>,
    /// Whether the table appeared in the per-case schema (at least once).
    ever_in_schema: bool,
    /// Whether the table was NOT in the per-case schema (at least once).
    ever_not_in_schema: bool,
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    if !fixture_path.exists() {
        eprintln!("Ground-truth fixture not found: {}", fixture_path.display());
        eprintln!("Expected at tests/fixtures/lineage/snowflake_ground_truth.jsonl");
        std::process::exit(1);
    }

    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    let mut total_queries = 0u64;
    let mut exact_match = 0u64;
    let mut parse_fail = 0u64;
    let mut empty_actual = 0u64;
    let mut pure_subset_queries = 0u64;
    let mut other_mismatch = 0u64;

    // Missing table -> info
    let mut missing_tables: HashMap<String, MissingTableInfo> = HashMap::new();

    // Missing "TABLE.COL" pair frequency
    let mut missing_table_col_freq: HashMap<String, u64> = HashMap::new();

    // Total missing source-ref count
    let mut total_missing_refs = 0u64;

    // Sample cases (capped at 10)
    let mut samples: Vec<SubsetCase> = Vec::new();

    for (line_idx, line) in reader.lines().enumerate() {
        let line = line.expect("failed to read line");
        let line = line.trim().to_string();
        if line.is_empty() {
            continue;
        }

        let case: Case = match serde_json::from_str(&line) {
            Ok(c) => c,
            Err(_) => continue,
        };

        total_queries += 1;

        let schema_map = schema_from_json(&case.schema);
        let schema_tables: HashSet<String> = schema_map.keys().map(|t| t.to_uppercase()).collect();
        let expected = normalize(&case.expected);

        // Run our lineage engine
        let result = match column_lineage(&case.sql, dialect, &schema_map, None) {
            Ok(r) => r,
            Err(_) => {
                parse_fail += 1;
                continue;
            }
        };

        let actual = normalize(&result.column_dict);

        // Fast path: exact match
        if actual == expected {
            exact_match += 1;
            continue;
        }

        // Empty actual
        if actual.is_empty() && !expected.is_empty() {
            empty_actual += 1;
            continue;
        }

        // -----------------------------------------------------------
        // Check "pure subset" at query level:
        //   - No extra output columns in actual that aren't in expected
        //   - For every output column in actual that IS in expected,
        //     actual sources are a subset-or-equal of expected sources
        //   - At least one column has a strict subset (otherwise it'd
        //     be exact match or only_missing_output_cols)
        // -----------------------------------------------------------

        let has_extra_output_cols = actual.keys().any(|k| !expected.contains_key(k));
        if has_extra_output_cols {
            other_mismatch += 1;
            continue;
        }

        // For every column in actual, check subset property
        let mut all_subset_or_equal = true;
        let mut any_strict_subset = false;

        for (col, act_sources) in &actual {
            if let Some(exp_sources) = expected.get(col) {
                if !is_subset_or_equal(act_sources, exp_sources) {
                    all_subset_or_equal = false;
                    break;
                }
                if act_sources != exp_sources {
                    any_strict_subset = true;
                }
            }
            // If col is in actual but not expected — would be extra output col,
            // already handled above
        }

        // Also: expected columns missing from actual count as "missing sources"
        // (all sources for that col are missing). We still classify this as
        // pure-subset if no actual column has EXTRA sources.
        let missing_output_cols: Vec<&String> = expected
            .keys()
            .filter(|k| !actual.contains_key(*k))
            .collect();

        if !missing_output_cols.is_empty() {
            any_strict_subset = true; // entire column missing = subset (empty)
        }

        if !all_subset_or_equal || !any_strict_subset {
            other_mismatch += 1;
            continue;
        }

        // ---- This is a "pure subset" case ----
        pure_subset_queries += 1;

        let mut case_missing_refs: Vec<(String, Vec<String>)> = Vec::new();

        // Collect missing sources for columns present in both
        for (col, exp_sources) in &expected {
            let act_sources = actual.get(col).map(|v| v.as_slice()).unwrap_or(&[]);
            let missing: Vec<String> = exp_sources
                .iter()
                .filter(|s| !act_sources.contains(s))
                .cloned()
                .collect();

            if missing.is_empty() {
                continue;
            }

            total_missing_refs += missing.len() as u64;

            for missing_ref in &missing {
                let tbl = table_of(missing_ref);
                if tbl.is_empty() {
                    continue;
                }

                // Track table-level info
                let info = missing_tables.entry(tbl.clone()).or_default();
                info.ref_count += 1;
                info.query_names.insert(case.name.clone());
                if schema_tables.contains(&tbl) {
                    info.ever_in_schema = true;
                } else {
                    info.ever_not_in_schema = true;
                }

                // Track table.col pair
                *missing_table_col_freq
                    .entry(missing_ref.clone())
                    .or_default() += 1;
            }

            case_missing_refs.push((col.clone(), missing));
        }

        // Sample collection
        if samples.len() < 10 && !case_missing_refs.is_empty() {
            let sql_prefix: String = case.sql.chars().take(200).collect();
            samples.push(SubsetCase {
                name: case.name.clone(),
                sql_prefix,
                missing_refs: case_missing_refs,
            });
        }

        // Progress
        if (line_idx + 1) % 2000 == 0 {
            eprint!("\r  processed {} lines ...", line_idx + 1);
        }
    }
    eprintln!("\r  done.                              ");

    // -----------------------------------------------------------------------
    // Report
    // -----------------------------------------------------------------------

    println!();
    println!("================================================================");
    println!("  Missing Sources Analysis — Pure Subset Cases");
    println!("================================================================");
    println!();
    println!("  Total queries:               {:>7}", total_queries);
    println!("  Exact match:                 {:>7}", exact_match);
    println!("  Parse fail:                  {:>7}", parse_fail);
    println!("  Empty actual:                {:>7}", empty_actual);
    println!(
        "  Pure subset (missing only):   {:>7}  ({:.1}%)",
        pure_subset_queries,
        if total_queries > 0 {
            pure_subset_queries as f64 / total_queries as f64 * 100.0
        } else {
            0.0
        }
    );
    println!("  Other mismatch:              {:>7}", other_mismatch);
    println!();
    println!(
        "  Total missing source refs (in pure-subset cases): {}",
        total_missing_refs
    );
    println!();

    // -- Top 30 missing tables by frequency --
    println!("================================================================");
    println!("  Top 30 Missing Tables (by # of missing source refs)");
    println!("================================================================");
    println!();
    println!(
        "  {:<45} {:>7} {:>7}  {:>10}",
        "TABLE", "REFS", "QUERIES", "IN_SCHEMA?"
    );
    println!("  {}", "-".repeat(78));

    let mut table_vec: Vec<(&String, &MissingTableInfo)> = missing_tables.iter().collect();
    table_vec.sort_by(|a, b| b.1.ref_count.cmp(&a.1.ref_count));

    for (tbl, info) in table_vec.iter().take(30) {
        let schema_status = match (info.ever_in_schema, info.ever_not_in_schema) {
            (true, false) => "YES",
            (false, true) => "NO",
            (true, true) => "MIXED",
            (false, false) => "?",
        };
        println!(
            "  {:<45} {:>7} {:>7}  {:>10}",
            tbl,
            info.ref_count,
            info.query_names.len(),
            schema_status
        );
    }

    if table_vec.len() > 30 {
        let tail_refs: u64 = table_vec.iter().skip(30).map(|(_, i)| i.ref_count).sum();
        let tail_queries: usize = table_vec
            .iter()
            .skip(30)
            .map(|(_, i)| i.query_names.len())
            .sum();
        println!(
            "  {:<45} {:>7} {:>7}",
            format!("... ({} more tables)", table_vec.len() - 30),
            tail_refs,
            tail_queries
        );
    }

    // -- Summary: in-schema vs not-in-schema --
    let tables_in_schema: usize = table_vec
        .iter()
        .filter(|(_, i)| i.ever_in_schema && !i.ever_not_in_schema)
        .count();
    let tables_not_in_schema: usize = table_vec
        .iter()
        .filter(|(_, i)| !i.ever_in_schema && i.ever_not_in_schema)
        .count();
    let tables_mixed: usize = table_vec
        .iter()
        .filter(|(_, i)| i.ever_in_schema && i.ever_not_in_schema)
        .count();

    let refs_from_in_schema: u64 = table_vec
        .iter()
        .filter(|(_, i)| i.ever_in_schema && !i.ever_not_in_schema)
        .map(|(_, i)| i.ref_count)
        .sum();
    let refs_from_not_in_schema: u64 = table_vec
        .iter()
        .filter(|(_, i)| !i.ever_in_schema && i.ever_not_in_schema)
        .map(|(_, i)| i.ref_count)
        .sum();
    let refs_from_mixed: u64 = table_vec
        .iter()
        .filter(|(_, i)| i.ever_in_schema && i.ever_not_in_schema)
        .map(|(_, i)| i.ref_count)
        .sum();

    println!();
    println!("================================================================");
    println!("  Schema Presence Summary");
    println!("================================================================");
    println!();
    println!(
        "  Tables always IN schema:      {:>5} tables, {:>7} missing refs",
        tables_in_schema, refs_from_in_schema
    );
    println!(
        "  Tables always NOT in schema:  {:>5} tables, {:>7} missing refs",
        tables_not_in_schema, refs_from_not_in_schema
    );
    println!(
        "  Tables MIXED (sometimes):     {:>5} tables, {:>7} missing refs",
        tables_mixed, refs_from_mixed
    );
    println!();

    // -- Interpretation --
    println!("  Interpretation:");
    println!("    IN schema     = we SEE the table but fail to trace columns through it");
    println!("                    (likely a view, or our engine loses the column path)");
    println!("    NOT in schema = table is entirely absent from our case schema");
    println!("                    (Snowflake traced through a view to a base table");
    println!("                     that we don't have a definition for)");
    println!("    MIXED         = same table name, but schema presence varies by query");
    println!();

    // -- Top 20 missing table.column pairs --
    println!("================================================================");
    println!("  Top 20 Missing TABLE.COL Pairs");
    println!("================================================================");
    println!();
    let mut tc_vec: Vec<_> = missing_table_col_freq.iter().collect();
    tc_vec.sort_by(|a, b| b.1.cmp(a.1));
    for (tc, count) in tc_vec.iter().take(20) {
        println!("  {:<55} {:>6}", tc, count);
    }

    // -- Sample cases --
    if !samples.is_empty() {
        println!();
        println!("================================================================");
        println!("  Sample Pure-Subset Cases (up to 10)");
        println!("================================================================");
        for (i, sample) in samples.iter().enumerate() {
            println!();
            println!("  --- Sample {} ---", i + 1);
            println!("  Name: {}", sample.name);
            println!("  SQL:  {}...", sample.sql_prefix);
            println!("  Missing:");
            for (col, missing) in &sample.missing_refs {
                println!("    output col {col}:");
                for m in missing {
                    let tbl = table_of(m);
                    println!("      - {m}  (table: {tbl})");
                }
            }
        }
    }

    // -- Cross-tabulation: top tables by # queries --
    println!();
    println!("================================================================");
    println!("  Top 20 Missing Tables (by # distinct queries affected)");
    println!("================================================================");
    println!();

    let mut by_queries: Vec<(&String, &MissingTableInfo)> = missing_tables.iter().collect();
    by_queries.sort_by(|a, b| b.1.query_names.len().cmp(&a.1.query_names.len()));
    println!(
        "  {:<45} {:>7} {:>7}  {:>10}",
        "TABLE", "QUERIES", "REFS", "IN_SCHEMA?"
    );
    println!("  {}", "-".repeat(78));
    for (tbl, info) in by_queries.iter().take(20) {
        let schema_status = match (info.ever_in_schema, info.ever_not_in_schema) {
            (true, false) => "YES",
            (false, true) => "NO",
            (true, true) => "MIXED",
            (false, false) => "?",
        };
        println!(
            "  {:<45} {:>7} {:>7}  {:>10}",
            tbl,
            info.query_names.len(),
            info.ref_count,
            schema_status
        );
    }

    // -- Sorted alphabetically for completeness (all tables) --
    println!();
    println!("================================================================");
    println!("  All Missing Tables (alphabetical)");
    println!("================================================================");
    println!();
    let mut alpha: BTreeMap<&String, &MissingTableInfo> = BTreeMap::new();
    for (tbl, info) in &missing_tables {
        alpha.insert(tbl, info);
    }
    let total_unique = alpha.len();
    println!("  Total unique missing tables: {}", total_unique);
    println!();
    if total_unique <= 80 {
        println!(
            "  {:<45} {:>7} {:>7}  {:>10}",
            "TABLE", "REFS", "QUERIES", "IN_SCHEMA?"
        );
        println!("  {}", "-".repeat(78));
        for (tbl, info) in &alpha {
            let schema_status = match (info.ever_in_schema, info.ever_not_in_schema) {
                (true, false) => "YES",
                (false, true) => "NO",
                (true, true) => "MIXED",
                (false, false) => "?",
            };
            println!(
                "  {:<45} {:>7} {:>7}  {:>10}",
                tbl,
                info.ref_count,
                info.query_names.len(),
                schema_status
            );
        }
    } else {
        println!("  (too many to list — showing top 30 by refs above)");
    }

    println!();
    println!("================================================================");
}
