//! Analyze "disjoint" column lineage — cases where actual and expected sources
//! for the same output column have ZERO overlap.
//!
//! This is the hardest mismatch category: we're not partially right (superset
//! or subset), we're pointing to completely different source tables/columns.
//! Understanding _which_ tables appear on each side reveals whether the issue
//! is CTE resolution, view expansion, alias confusion, or something else.
//!
//! Run: cargo run --example disjoint_analysis --release

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;

#[derive(Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

fn normalize_to_short_names(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let normalized: Vec<String> = v
                .iter()
                .map(|ref_str| {
                    let parts: Vec<&str> =
                        ref_str.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        let table = parts[parts.len() - 2];
                        let col = parts[parts.len() - 1];
                        format!("\"{table}\".\"{col}\"")
                    } else {
                        ref_str.clone()
                    }
                })
                .collect();
            let mut sorted = normalized;
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

/// Extract the table part from a `"TABLE"."COL"` reference.
fn table_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    if parts.len() >= 2 {
        parts[parts.len() - 2].to_uppercase()
    } else {
        ref_str.trim_matches('"').to_uppercase()
    }
}

/// Extract CTE names from SQL by scanning for `WITH name AS` and `, name AS` patterns.
fn extract_cte_names(sql: &str) -> HashSet<String> {
    let upper = sql.to_uppercase();
    let mut names = HashSet::new();

    // Find WITH keyword positions, then scan for `name AS (`
    let tokens: Vec<&str> = upper.split_whitespace().collect();
    let mut i = 0;
    while i < tokens.len() {
        if (tokens[i] == "WITH" || tokens[i] == ",") && i + 2 < tokens.len() {
            // Next token could be RECURSIVE — skip it
            let name_idx = if tokens[i + 1] == "RECURSIVE" {
                i + 2
            } else {
                i + 1
            };
            if name_idx + 1 < tokens.len() {
                let candidate =
                    tokens[name_idx].trim_matches(|c: char| !c.is_alphanumeric() && c != '_');
                let next = tokens[name_idx + 1];
                if next.starts_with("AS") && !candidate.is_empty() {
                    names.insert(candidate.to_string());
                }
            }
        }
        i += 1;
    }

    names
}

/// A single disjoint column observation.
struct DisjointColumn {
    query_name: String,
    sql_prefix: String,
    column: String,
    actual_sources: Vec<String>,
    expected_sources: Vec<String>,
}

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");
    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    let mut total_queries = 0u64;
    let mut parse_fail = 0u64;
    let mut disjoint_column_count = 0u64;
    let mut disjoint_query_count = 0u64;

    // Frequency maps
    let mut actual_table_freq: HashMap<String, u64> = HashMap::new();
    let mut expected_table_freq: HashMap<String, u64> = HashMap::new();
    let mut pair_freq: HashMap<(String, String), u64> = HashMap::new();

    // CTE analysis
    let mut actual_is_cte_name = 0u64;
    let mut actual_total_table_refs = 0u64;

    // Samples
    let mut samples: Vec<DisjointColumn> = Vec::new();

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total_queries += 1;

        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => {
                parse_fail += 1;
                continue;
            }
        };

        let actual = normalize_to_short_names(&result.column_dict);
        let expected = normalize(&case.expected);

        let cte_names = extract_cte_names(&case.sql);

        let mut query_has_disjoint = false;

        // Only look at columns present in BOTH actual and expected
        for (col, exp_sources) in &expected {
            let act_sources = match actual.get(col) {
                Some(s) if !s.is_empty() => s,
                _ => continue,
            };

            if exp_sources.is_empty() {
                continue;
            }

            // Check for disjoint: zero overlap
            let act_set: HashSet<&String> = act_sources.iter().collect();
            let exp_set: HashSet<&String> = exp_sources.iter().collect();
            let overlap: HashSet<_> = act_set.intersection(&exp_set).collect();

            if !overlap.is_empty() {
                continue;
            }

            // This column is disjoint
            disjoint_column_count += 1;
            query_has_disjoint = true;

            // Extract table names from each side
            let act_tables: Vec<String> = act_sources.iter().map(|r| table_of(r)).collect();
            let exp_tables: Vec<String> = exp_sources.iter().map(|r| table_of(r)).collect();

            // Count each unique actual table
            let act_table_set: HashSet<&String> = act_tables.iter().collect();
            let exp_table_set: HashSet<&String> = exp_tables.iter().collect();

            for t in &act_table_set {
                *actual_table_freq.entry((*t).clone()).or_default() += 1;
            }
            for t in &exp_table_set {
                *expected_table_freq.entry((*t).clone()).or_default() += 1;
            }

            // Count pairs (actual_table, expected_table)
            for at in &act_table_set {
                actual_total_table_refs += 1;
                if cte_names.contains(at.as_str()) {
                    actual_is_cte_name += 1;
                }
                for et in &exp_table_set {
                    *pair_freq.entry(((*at).clone(), (*et).clone())).or_default() += 1;
                }
            }

            // Collect sample
            if samples.len() < 20 {
                samples.push(DisjointColumn {
                    query_name: case.name.chars().take(50).collect(),
                    sql_prefix: case.sql.chars().take(200).collect(),
                    column: col.clone(),
                    actual_sources: act_sources.clone(),
                    expected_sources: exp_sources.clone(),
                });
            }
        }

        if query_has_disjoint {
            disjoint_query_count += 1;
        }
    }

    // -----------------------------------------------------------------------
    // Print results
    // -----------------------------------------------------------------------
    println!("\n{}", "=".repeat(70));
    println!("  DISJOINT ANALYSIS — Zero-Overlap Column Sources");
    println!("{}\n", "=".repeat(70));

    println!("=== Summary ===");
    println!("Total queries processed:   {total_queries}");
    println!("Parse failures:            {parse_fail}");
    println!(
        "Queries with disjoint:     {disjoint_query_count} ({:.1}%)",
        disjoint_query_count as f64 / (total_queries - parse_fail).max(1) as f64 * 100.0
    );
    println!("Disjoint columns total:    {disjoint_column_count}");

    // 1. Top 30 actual source tables
    println!("\n=== Top 30 Actual Source Tables (in disjoint columns) ===");
    let mut act_vec: Vec<_> = actual_table_freq.iter().collect();
    act_vec.sort_by(|a, b| b.1.cmp(a.1));
    for (i, (tbl, count)) in act_vec.iter().take(30).enumerate() {
        println!(
            "  {:>2}. {:<45} {:>6} ({:.1}%)",
            i + 1,
            tbl,
            count,
            **count as f64 / disjoint_column_count.max(1) as f64 * 100.0
        );
    }

    // 2. Top 30 expected source tables
    println!("\n=== Top 30 Expected Source Tables (in disjoint columns) ===");
    let mut exp_vec: Vec<_> = expected_table_freq.iter().collect();
    exp_vec.sort_by(|a, b| b.1.cmp(a.1));
    for (i, (tbl, count)) in exp_vec.iter().take(30).enumerate() {
        println!(
            "  {:>2}. {:<45} {:>6} ({:.1}%)",
            i + 1,
            tbl,
            count,
            **count as f64 / disjoint_column_count.max(1) as f64 * 100.0
        );
    }

    // 3. Top 30 actual->expected table pairs
    println!("\n=== Top 30 Actual->Expected Table Pairs ===");
    let mut pair_vec: Vec<_> = pair_freq.iter().collect();
    pair_vec.sort_by(|a, b| b.1.cmp(a.1));
    for (i, ((act, exp), count)) in pair_vec.iter().take(30).enumerate() {
        println!("  {:>2}. {:<30} -> {:<30} {:>5}", i + 1, act, exp, count);
    }

    // 4. CTE name analysis
    println!("\n=== CTE Name Analysis ===");
    println!(
        "Actual table refs that are CTE names: {actual_is_cte_name} / {actual_total_table_refs} ({:.1}%)",
        actual_is_cte_name as f64 / actual_total_table_refs.max(1) as f64 * 100.0
    );

    // 5. Sample disjoint columns
    println!("\n{}", "=".repeat(70));
    println!("  20 SAMPLE DISJOINT COLUMNS");
    println!("{}", "=".repeat(70));
    for (i, s) in samples.iter().enumerate() {
        println!("\n--- Sample {} ---", i + 1);
        println!("Query:    {}", s.query_name);
        println!("SQL:      {}...", s.sql_prefix);
        println!("Column:   {}", s.column);
        println!("Actual:   {:?}", s.actual_sources);
        println!("Expected: {:?}", s.expected_sources);
    }

    println!("\n=== Done ===");
}
