//! Per-column superset detail diagnostic.
//!
//! For each output column where actual lineage is a strict superset of expected
//! (all expected sources present, plus extras), this diagnostic:
//!
//! 1. Categorizes every extra source into one of three buckets:
//!    a) Same table as an expected source (different column from that table)
//!    b) Different table entirely (table present in schema but not in expected)
//!    c) Non-schema alias (table name not found in schema -- subquery/CTE alias)
//!
//! 2. Tracks the distribution of extra-source counts per column (1, 2, 3+).
//!
//! 3. Prints the top 20 most-frequently-appearing extra column names.
//!
//! 4. Prints 10 sample cases with query name, output column, expected sources,
//!    extra sources (the diff), and the first 80 chars of SQL.
//!
//! Run: cargo run --example superset_detail --release

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;

// ---------------------------------------------------------------------------
// Fixture types
// ---------------------------------------------------------------------------

#[derive(Debug, Deserialize)]
struct Case {
    name: String,
    sql: String,
    schema: serde_json::Value,
    expected: HashMap<String, Vec<String>>,
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

/// Normalize a column-lineage map: sort + dedup each source list.
fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

/// Normalize actual output to short `"TABLE"."COL"` form (strip leading db/schema qualifiers).
fn normalize_to_short(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let normalized: Vec<String> = v
                .iter()
                .map(|ref_str| {
                    let parts: Vec<&str> =
                        ref_str.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        let table = parts[parts.len() - 2];
                        let col = parts[parts.len() - 1];
                        format!("\"{table}\".\"{col}\"")
                    } else {
                        ref_str.clone()
                    }
                })
                .collect();
            let mut sorted = normalized;
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

/// Extract the table portion from a `"TABLE"."COL"` reference.
fn table_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    if parts.len() >= 2 {
        parts[parts.len() - 2].to_uppercase()
    } else {
        String::new()
    }
}

/// Extract the column portion from a `"TABLE"."COL"` reference.
fn column_of(ref_str: &str) -> String {
    let parts: Vec<&str> = ref_str.split('.').map(|p| p.trim_matches('"')).collect();
    parts.last().unwrap_or(&"").to_uppercase()
}

// ---------------------------------------------------------------------------
// Extra-source categories
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
enum ExtraKind {
    /// Extra source comes from the same table as an expected source (different column).
    SameTable,
    /// Extra source comes from a table that exists in the schema but is not among
    /// the expected tables for this output column.
    DifferentTable,
    /// Extra source's table name does not appear in the schema at all -- it is a
    /// subquery alias, CTE name, or other non-schema identifier.
    NonSchemaAlias,
}

impl std::fmt::Display for ExtraKind {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            ExtraKind::SameTable => write!(f, "same_table"),
            ExtraKind::DifferentTable => write!(f, "different_table"),
            ExtraKind::NonSchemaAlias => write!(f, "non_schema_alias"),
        }
    }
}

// ---------------------------------------------------------------------------
// Sample record
// ---------------------------------------------------------------------------

struct Sample {
    query_name: String,
    output_column: String,
    expected_sources: Vec<String>,
    extra_sources: Vec<(String, ExtraKind)>,
    sql_prefix: String,
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

fn main() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);

    let dialect = SqlDialect::Snowflake;

    // Counters
    let mut total_queries = 0u64;
    let mut parse_failures = 0u64;
    let mut superset_column_count = 0u64;
    let mut total_extra_count = 0u64;

    // Category distribution
    let mut kind_counts: BTreeMap<ExtraKind, u64> = BTreeMap::new();

    // Extras-per-column histogram (key = number of extras)
    let mut extras_histogram: BTreeMap<usize, u64> = BTreeMap::new();

    // Frequency of extra column names (the column part of the ref)
    let mut extra_col_freq: HashMap<String, u64> = HashMap::new();

    // Collected samples (up to 10)
    let mut samples: Vec<Sample> = Vec::new();

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: Case = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total_queries += 1;

        // Build schema table set (uppercased) for alias detection.
        let schema = schema_from_json(&case.schema);
        let schema_table_set: HashSet<String> = schema.keys().map(|t| t.to_uppercase()).collect();

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => {
                parse_failures += 1;
                continue;
            }
        };

        let actual = normalize_to_short(&result.column_dict);
        let expected = normalize(&case.expected);

        // Per-column superset check: actual must contain ALL expected sources AND
        // have at least one extra.  We iterate expected columns individually.
        for (col, exp_sources) in &expected {
            let act_sources = match actual.get(col) {
                Some(s) => s,
                None => continue, // missing column -- not a superset case
            };

            // All expected present?
            let all_expected_present = exp_sources.iter().all(|e| act_sources.contains(e));
            if !all_expected_present {
                continue;
            }

            // Compute extras.
            let extras: Vec<&String> = act_sources
                .iter()
                .filter(|a| !exp_sources.contains(a))
                .collect();
            if extras.is_empty() {
                continue; // exact match on this column
            }

            // -- This column is a superset case --
            superset_column_count += 1;
            let n_extras = extras.len();
            total_extra_count += n_extras as u64;

            // Histogram bucket: 1, 2, or 3+
            let bucket = if n_extras >= 3 { 3 } else { n_extras };
            *extras_histogram.entry(bucket).or_default() += 1;

            // Tables present among expected sources for this column.
            let expected_tables: HashSet<String> =
                exp_sources.iter().map(|s| table_of(s)).collect();

            let mut classified_extras: Vec<(String, ExtraKind)> = Vec::new();

            for extra in &extras {
                let extra_table = table_of(extra);
                let extra_col = column_of(extra);

                // Track column frequency.
                *extra_col_freq.entry(extra_col.clone()).or_default() += 1;

                // Classify.
                let kind = if !schema_table_set.contains(&extra_table) {
                    // Table is not in the schema at all -- subquery/CTE alias.
                    ExtraKind::NonSchemaAlias
                } else if expected_tables.contains(&extra_table) {
                    // Table matches an expected source's table -- same table, different column.
                    ExtraKind::SameTable
                } else {
                    // Table exists in schema but is not among the expected tables.
                    ExtraKind::DifferentTable
                };

                *kind_counts.entry(kind).or_default() += 1;
                classified_extras.push(((*extra).clone(), kind));
            }

            // Collect sample (up to 10).
            if samples.len() < 10 {
                let name_prefix: String = case.name.chars().take(30).collect();
                let sql_prefix: String = case
                    .sql
                    .chars()
                    .filter(|c| !c.is_ascii_control() || *c == ' ')
                    .take(80)
                    .collect();

                samples.push(Sample {
                    query_name: name_prefix,
                    output_column: col.clone(),
                    expected_sources: exp_sources.clone(),
                    extra_sources: classified_extras,
                    sql_prefix,
                });
            }
        }
    }

    // -----------------------------------------------------------------------
    // Report
    // -----------------------------------------------------------------------

    println!();
    println!("{}", "=".repeat(70));
    println!("  SUPERSET DETAIL -- Per-Column Extra-Source Diagnostic");
    println!("{}", "=".repeat(70));

    println!();
    println!("--- Overview ---");
    println!("Total queries processed:      {total_queries}");
    println!("Parse failures (skipped):     {parse_failures}");
    println!("Per-column superset cases:    {superset_column_count}");
    println!("Total extra sources:          {total_extra_count}");
    if superset_column_count > 0 {
        println!(
            "Average extras per column:    {:.2}",
            total_extra_count as f64 / superset_column_count as f64
        );
    }

    // -- Category breakdown --
    println!();
    println!("--- Extra-Source Categories ---");
    let cat_order = [
        ExtraKind::SameTable,
        ExtraKind::DifferentTable,
        ExtraKind::NonSchemaAlias,
    ];
    for kind in &cat_order {
        let count = kind_counts.get(kind).copied().unwrap_or(0);
        let pct = if total_extra_count > 0 {
            count as f64 / total_extra_count as f64 * 100.0
        } else {
            0.0
        };
        println!("  {kind:<25} {count:>6}  ({pct:5.1}%)");
    }

    // -- Extras-per-column distribution --
    println!();
    println!("--- Extras-per-Column Distribution ---");
    for (&bucket, &count) in &extras_histogram {
        let label = if bucket >= 3 {
            "3+ extra".to_string()
        } else {
            format!("{bucket} extra")
        };
        let pct = if superset_column_count > 0 {
            count as f64 / superset_column_count as f64 * 100.0
        } else {
            0.0
        };
        println!("  {label:<15} {count:>6} columns  ({pct:5.1}%)");
    }

    // -- Top 20 extra column names --
    println!();
    println!("--- Top 20 Extra Source Column Names ---");
    let mut col_vec: Vec<_> = extra_col_freq.into_iter().collect();
    col_vec.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
    for (col, count) in col_vec.iter().take(20) {
        println!("  {col:<40} {count:>6}");
    }

    // -- 10 Sample cases --
    println!();
    println!("{}", "=".repeat(70));
    println!("  10 SAMPLE PER-COLUMN SUPERSET CASES");
    println!("{}", "=".repeat(70));

    for (i, s) in samples.iter().enumerate() {
        println!();
        println!("--- Sample {} ---", i + 1);
        println!("Query:     {}", s.query_name);
        println!("Column:    {}", s.output_column);
        println!("Expected:  [{}]", s.expected_sources.join(", "));
        let extra_strs: Vec<String> = s
            .extra_sources
            .iter()
            .map(|(ref_str, kind)| format!("{ref_str} ({kind})"))
            .collect();
        println!("Extras:    [{}]", extra_strs.join(", "));
        println!("SQL:       {}...", s.sql_prefix);
    }

    println!();
    println!("=== Done ===");
}
