//! Debug: what sources do the top extra output columns have?
//!
//! Run: cargo run --example extra_cols_debug --release

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{BTreeSet, HashMap};
use std::io::BufRead;
use std::path::Path;

#[derive(Debug, Deserialize)]
struct Case {
    #[allow(dead_code)]
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, BTreeSet<String>> {
    map.iter()
        .map(|(k, v)| {
            let set: BTreeSet<String> = v.iter().cloned().collect();
            (k.clone(), set)
        })
        .collect()
}

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");
    let file = std::fs::File::open(&fixture_path).expect("open fixture");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);
    let dialect = SqlDialect::Snowflake;

    // Track: for each extra output column name, what are its typical source patterns?
    let mut extra_col_source_patterns: HashMap<String, Vec<(String, Vec<String>)>> = HashMap::new();
    // Track: how many extra cols have sources all outside schema vs mixed
    let mut all_outside = 0u64;
    let mut some_inside = 0u64;
    let mut empty_sources = 0u64;
    let mut total_extra = 0u64;

    for line in reader.lines() {
        let line = line.unwrap();
        let line = line.trim();
        if line.is_empty() {
            continue;
        }
        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };

        let schema = schema_from_json(&case.schema);
        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => continue,
        };

        let actual = normalize(&result.column_dict);
        let expected = normalize(&case.expected);

        // Find extra output columns
        for (col, sources) in &actual {
            if expected.contains_key(col) {
                continue;
            }
            total_extra += 1;

            if sources.is_empty() {
                empty_sources += 1;
                continue;
            }

            // Check if ANY source is in schema
            let has_schema_source = sources.iter().any(|ref_str| {
                let parts: Vec<&str> = ref_str.split('.').collect();
                if parts.len() < 2 {
                    return false;
                }
                let table_part = parts[..parts.len() - 1]
                    .iter()
                    .map(|p| p.trim_matches('"'))
                    .collect::<Vec<_>>()
                    .join(".");
                // Check if table is in schema (case-insensitive)
                schema.keys().any(|k| k.eq_ignore_ascii_case(&table_part))
            });

            if has_schema_source {
                some_inside += 1;
            } else {
                all_outside += 1;
            }

            // Collect first 3 examples per extra column name
            let entry = extra_col_source_patterns.entry(col.clone()).or_default();
            if entry.len() < 3 {
                entry.push((case.name.clone(), sources.iter().cloned().collect()));
            }
        }
    }

    println!("=== Extra Output Column Analysis ===");
    println!("Total extra column occurrences: {total_extra}");
    println!("  empty sources:     {empty_sources}");
    println!("  all outside schema: {all_outside}");
    println!("  some inside schema: {some_inside}");
    println!();

    // Show top 20 extra columns with their source patterns
    let mut by_count: Vec<(String, Vec<(String, Vec<String>)>)> =
        extra_col_source_patterns.into_iter().collect();
    by_count.sort_by(|a, b| b.1.len().cmp(&a.1.len()));

    println!("Top 30 extra columns with example sources:");
    for (col, examples) in by_count.iter().take(30) {
        println!("\n  {} ({} occurrences):", col, examples.len());
        for (name, sources) in examples.iter().take(2) {
            println!("    case: {name}");
            for s in sources {
                println!("      src: {s}");
            }
        }
    }
}
