//! Ground-truth column lineage tests from Snowflake access_history.
//!
//! Streams 19,878 real production queries from Rakuten's Snowflake warehouse
//! and compares our lineage engine output against Snowflake's own column-level
//! lineage tracking.
//!
//! Run with: cargo test --test ground_truth_lineage_tests -- --nocapture

use altimate_core::lineage::column_lineage;
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::HashMap;
use std::io::BufRead;
use std::path::Path;

#[derive(Debug, Deserialize)]
struct GroundTruthCase {
    name: String,
    sql: String,
    schema: serde_json::Value,
    #[allow(dead_code)]
    dialect: String,
    #[allow(dead_code)]
    mode: String,
    expected: HashMap<String, Vec<String>>,
    #[allow(dead_code)]
    query_hash: Option<String>,
    #[allow(dead_code)]
    source_query_type: Option<String>,
    #[allow(dead_code)]
    database_name: Option<String>,
    #[allow(dead_code)]
    schema_name: Option<String>,
}

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

fn normalize(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

/// Normalize actual output to use short table names (last segment only).
/// Our engine may produce "schema"."table"."col" while ground truth uses "table"."col".
fn normalize_to_short_names(map: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    map.iter()
        .map(|(k, v)| {
            let normalized: Vec<String> = v
                .iter()
                .map(|ref_str| {
                    // Parse quoted parts: "A"."B"."C" → ["A", "B", "C"]
                    let parts: Vec<&str> =
                        ref_str.split('.').map(|p| p.trim_matches('"')).collect();
                    if parts.len() >= 3 {
                        // Take last 2 parts: table.column
                        let table = parts[parts.len() - 2];
                        let col = parts[parts.len() - 1];
                        format!("\"{table}\".\"{col}\"")
                    } else {
                        ref_str.clone()
                    }
                })
                .collect();
            let mut sorted = normalized;
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

#[test]
fn test_snowflake_ground_truth() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "Ground-truth fixture not found: {}. Skipping.",
            fixture_path.display()
        );
        return;
    }

    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);

    let dialect = SqlDialect::Snowflake;
    let mut total = 0u64;
    let mut exact_match = 0u64;
    let mut partial_match = 0u64;
    let mut parse_errors = 0u64;
    let mut mismatch = 0u64;
    let mut lenient_exact = 0u64; // exact match ignoring extra columns in actual
    let mut only_extra_cols = 0u64; // queries that fail ONLY because of extra columns

    // Track error categories
    let mut error_categories: HashMap<String, u64> = HashMap::new();

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: GroundTruthCase = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(e) => {
                parse_errors += 1;
                let cat = if e.contains("panicked") {
                    "panic"
                } else if e.contains("spawn") {
                    "thread_spawn"
                } else if e.contains("nesting depth") {
                    "nesting_depth"
                } else {
                    "parse_error"
                };
                *error_categories.entry(cat.to_string()).or_default() += 1;
                continue;
            }
        };

        let actual = normalize_to_short_names(&result.column_dict);
        let expected = normalize(&case.expected);

        if actual == expected {
            exact_match += 1;
            lenient_exact += 1;
        } else {
            // Check lenient match: all expected cols have correct sources
            let mut all_expected_match = true;
            let mut has_extra = false;
            for (col, exp_sources) in &expected {
                match actual.get(col) {
                    Some(act_sources) if act_sources == exp_sources => {}
                    _ => {
                        all_expected_match = false;
                        break;
                    }
                }
            }
            for col in actual.keys() {
                if !expected.contains_key(col) {
                    has_extra = true;
                    break;
                }
            }
            if all_expected_match {
                lenient_exact += 1;
                if has_extra {
                    only_extra_cols += 1;
                }
            }
            // Categorize the mismatch
            let mut cols_correct = 0u64;
            let mut cols_missing_in_actual = 0u64;
            let mut cols_extra_in_actual = 0u64;
            let mut cols_wrong_sources = 0u64;

            for (col, exp_sources) in &expected {
                match actual.get(col) {
                    Some(act_sources) if act_sources == exp_sources => cols_correct += 1,
                    Some(_) => cols_wrong_sources += 1,
                    None => cols_missing_in_actual += 1,
                }
            }
            for col in actual.keys() {
                if !expected.contains_key(col) {
                    cols_extra_in_actual += 1;
                }
            }

            if cols_correct > 0 {
                partial_match += 1;
            } else {
                mismatch += 1;
            }

            // Track mismatch reasons
            if cols_missing_in_actual > 0 {
                *error_categories
                    .entry("cols_missing".to_string())
                    .or_default() += 1;
            }
            if cols_extra_in_actual > 0 {
                *error_categories
                    .entry("cols_extra".to_string())
                    .or_default() += 1;
            }
            if cols_wrong_sources > 0 {
                *error_categories
                    .entry("wrong_sources".to_string())
                    .or_default() += 1;
            }
            if actual.is_empty() && !expected.is_empty() {
                *error_categories
                    .entry("empty_actual".to_string())
                    .or_default() += 1;
            }

            // Collect first 5 detailed failure samples
            if (mismatch + partial_match) <= 5 {
                eprintln!("\n--- MISMATCH: {} ---", case.name);
                eprintln!(
                    "  SQL (first 200): {}",
                    &case.sql[..case.sql.len().min(200)]
                );
                eprintln!("  Expected cols: {}", expected.len());
                eprintln!("  Actual cols: {}", actual.len());
                for (col, exp) in expected.iter().take(3) {
                    let act = actual.get(col).cloned().unwrap_or_default();
                    eprintln!("  col '{}': expected={:?} actual={:?}", col, exp, act);
                }
            }
        }

        if total % 2000 == 0 {
            eprintln!(
                "  ... {total}: exact={exact_match} partial={partial_match} \
                 mismatch={mismatch} err={parse_errors}"
            );
        }
    }

    let pass_rate = if total > 0 {
        exact_match as f64 / total as f64 * 100.0
    } else {
        0.0
    };
    let parseable = total - parse_errors;
    let parse_rate = if total > 0 {
        parseable as f64 / total as f64 * 100.0
    } else {
        0.0
    };

    let lenient_rate = if total > 0 {
        lenient_exact as f64 / total as f64 * 100.0
    } else {
        0.0
    };
    eprintln!(
        "\n=== Snowflake Ground-Truth Results ===\n\
         Total:         {total}\n\
         Exact match:   {exact_match} ({pass_rate:.1}%)\n\
         Lenient exact: {lenient_exact} ({lenient_rate:.1}%) (ignoring extra output cols)\n\
         Only-extra:    {only_extra_cols} (would become exact with lenient)\n\
         Partial match: {partial_match}\n\
         Mismatch:      {mismatch}\n\
         Parse errors:  {parse_errors}\n\
         Parse rate:    {parse_rate:.1}%"
    );

    if !error_categories.is_empty() {
        eprintln!("\nError categories:");
        let mut cats: Vec<_> = error_categories.iter().collect();
        cats.sort_by(|a, b| b.1.cmp(a.1));
        for (cat, count) in cats {
            eprintln!("  {cat}: {count}");
        }
    }

    // Analyze wrong sources patterns
    eprintln!("\n=== WRONG SOURCES ANALYSIS ===");

    // Re-read fixtures and collect detailed wrong-sources info
    let file2 = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader2 = std::io::BufReader::with_capacity(1024 * 1024, file2);

    let mut near_miss_count = 0u64; // queries where only 1 column is wrong
    let mut near_miss_2 = 0u64; // queries where only 2 columns are wrong
    let mut wrong_source_patterns: HashMap<String, u64> = HashMap::new();
    let mut extra_col_names: HashMap<String, u64> = HashMap::new();
    let mut superset_count = 0u64; // actual is superset of expected (extra sources)
    let mut subset_count = 0u64; // actual is subset of expected (missing sources)
    let mut near_miss_reasons: HashMap<String, u64> = HashMap::new();

    for line in reader2.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: GroundTruthCase = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };

        let schema_map = schema_from_json(&case.schema);
        let result = match column_lineage(&case.sql, dialect, &schema_map, None) {
            Ok(r) => r,
            Err(_) => continue,
        };

        let actual = normalize_to_short_names(&result.column_dict);
        let expected = normalize(&case.expected);

        if actual == expected {
            continue;
        }

        // Count wrong columns
        let mut wrong_cols = 0u64;
        for (col, exp_sources) in &expected {
            if let Some(act_sources) = actual.get(col) {
                if act_sources != exp_sources {
                    wrong_cols += 1;

                    // Classify the type of wrong
                    let exp_set: std::collections::BTreeSet<_> = exp_sources.iter().collect();
                    let act_set: std::collections::BTreeSet<_> = act_sources.iter().collect();

                    if act_set.is_superset(&exp_set) {
                        superset_count += 1;
                        // We have extra sources — what are they?
                        for extra in act_set.difference(&exp_set) {
                            *wrong_source_patterns
                                .entry(format!("EXTRA_SRC:{}", extra))
                                .or_default() += 1;
                        }
                    } else if act_set.is_subset(&exp_set) {
                        subset_count += 1;
                        // We're missing sources — what are they?
                        for missing in exp_set.difference(&act_set) {
                            *wrong_source_patterns
                                .entry(format!("MISSING_SRC:{}", missing))
                                .or_default() += 1;
                        }
                    } else {
                        // Partial overlap or disjoint
                        for extra in act_set.difference(&exp_set) {
                            *wrong_source_patterns
                                .entry(format!("WRONG:{}", extra))
                                .or_default() += 1;
                        }
                    }
                }
            }
        }

        // Track extra columns we produce
        for col in actual.keys() {
            if !expected.contains_key(col) {
                *extra_col_names.entry(col.clone()).or_default() += 1;
            }
        }

        if wrong_cols == 1 {
            near_miss_count += 1;
            // Track what type of error for the single wrong column
            for (col, exp_sources) in &expected {
                if let Some(act_sources) = actual.get(col) {
                    if act_sources != exp_sources {
                        let exp_set: std::collections::BTreeSet<_> = exp_sources.iter().collect();
                        let act_set: std::collections::BTreeSet<_> = act_sources.iter().collect();
                        let reason = if act_set.is_superset(&exp_set) {
                            "superset"
                        } else if act_set.is_subset(&exp_set) {
                            "subset"
                        } else if exp_set.intersection(&act_set).count() > 0 {
                            "overlap"
                        } else {
                            "disjoint"
                        };
                        *near_miss_reasons.entry(reason.to_string()).or_default() += 1;

                        // Print first 10 disjoint cases for analysis
                        if reason == "disjoint"
                            && near_miss_reasons.get("disjoint").copied().unwrap_or(0) <= 10
                        {
                            eprintln!(
                                "\n--- DISJOINT #{} [{}] ---",
                                near_miss_reasons["disjoint"], case.name
                            );
                            eprintln!("  Wrong col: {col}");
                            eprintln!("  Expected: {:?}", exp_sources);
                            eprintln!("  Actual:   {:?}", act_sources);
                            eprintln!("  SQL: {}", &case.sql[..case.sql.len().min(150)]);
                            eprintln!(
                                "  Schema tables: {:?}",
                                case.schema
                                    .as_object()
                                    .map(|o| o.keys().collect::<Vec<_>>())
                                    .unwrap_or_default()
                            );
                        }

                        // Track extra source columns for superset near-misses
                        if reason == "superset" {
                            for extra in act_set.difference(&exp_set) {
                                // Classify: same table as expected or different table?
                                let extra_table: &str = extra.split('.').next().unwrap_or("");
                                let expected_tables: std::collections::BTreeSet<&str> = exp_set
                                    .iter()
                                    .map(|s| s.split('.').next().unwrap_or(""))
                                    .collect();
                                let tag = if expected_tables.contains(extra_table) {
                                    "same_table"
                                } else {
                                    "diff_table"
                                };
                                *near_miss_reasons
                                    .entry(format!("superset_{tag}"))
                                    .or_default() += 1;
                            }
                        }

                        // Track missing source columns for subset near-misses
                        if reason == "subset" {
                            for missing in exp_set.difference(&act_set) {
                                let missing_table: &str = missing.split('.').next().unwrap_or("");
                                let actual_tables: std::collections::BTreeSet<&str> = act_set
                                    .iter()
                                    .map(|s| s.split('.').next().unwrap_or(""))
                                    .collect();
                                let tag = if actual_tables.contains(missing_table) {
                                    "same_table"
                                } else {
                                    "diff_table"
                                };
                                *near_miss_reasons
                                    .entry(format!("subset_{tag}"))
                                    .or_default() += 1;
                            }
                        }
                        break;
                    }
                }
            }
        }
        if wrong_cols == 2 {
            near_miss_2 += 1;
        }
    }

    eprintln!("Near-miss (1 wrong col): {near_miss_count}");
    eprintln!("Near-miss (2 wrong cols): {near_miss_2}");
    eprintln!("Superset sources (extra): {superset_count}");
    eprintln!("Subset sources (missing): {subset_count}");

    eprintln!("\nNear-miss (1 wrong) error type:");
    let mut nm_sorted: Vec<_> = near_miss_reasons.iter().collect();
    nm_sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (reason, count) in &nm_sorted {
        eprintln!("  {count:5} {reason}");
    }

    eprintln!("\nTop 20 wrong source patterns:");
    let mut ws_sorted: Vec<_> = wrong_source_patterns.iter().collect();
    ws_sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (pat, count) in ws_sorted.iter().take(20) {
        eprintln!("  {count:5} {pat}");
    }

    eprintln!("\nTop 20 extra column names:");
    let mut extra_sorted: Vec<_> = extra_col_names.iter().collect();
    extra_sorted.sort_by(|a, b| b.1.cmp(a.1));
    for (name, count) in extra_sorted.iter().take(20) {
        eprintln!("  {count:5} {name}");
    }

    // Assert minimum parse rate — the engine should handle most real-world queries
    assert!(
        parse_rate > 50.0,
        "Parse rate too low: {parse_rate:.1}% ({parseable}/{total})"
    );
}

/// Regression test for the known-passing subset.
///
/// Reads `snowflake_passing.jsonl` (downloaded from S3 via `scripts/download_lineage_tests.sh`)
/// and asserts 100% lenient pass rate. If the engine regresses and breaks a previously-passing
/// query, this test catches it.
///
/// Skips gracefully if the file doesn't exist (it's not checked into git).
#[test]
fn test_snowflake_passing_regression() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_passing.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "Passing fixture not found: {}. Download with: bash scripts/download_lineage_tests.sh",
            fixture_path.display()
        );
        return;
    }

    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);

    let dialect = SqlDialect::Snowflake;
    let mut total = 0u64;
    let mut pass = 0u64;
    let mut regressions: Vec<String> = Vec::new();

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: GroundTruthCase = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(e) => {
                if regressions.len() < 10 {
                    regressions.push(format!("{}: parse error: {}", case.name, e));
                }
                continue;
            }
        };

        let actual = normalize_to_short_names(&result.column_dict);
        let expected = normalize(&case.expected);

        // Lenient match: all expected columns must have correct sources
        let mut ok = true;
        for (col, exp_sources) in &expected {
            match actual.get(col) {
                Some(act_sources) if act_sources == exp_sources => {}
                _ => {
                    ok = false;
                    break;
                }
            }
        }

        if ok {
            pass += 1;
        } else if regressions.len() < 10 {
            regressions.push(format!("{}: lenient mismatch", case.name));
        }

        if total % 5000 == 0 {
            eprintln!("  ... {total}: pass={pass}");
        }
    }

    let regression_count = total - pass;

    eprintln!(
        "\n=== Passing Regression Test ===\n\
         Total:       {total}\n\
         Pass:        {pass}\n\
         Regressions: {regression_count}"
    );

    if !regressions.is_empty() {
        eprintln!("\nFirst {} regressions:", regressions.len());
        for r in &regressions {
            eprintln!("  - {r}");
        }
    }

    assert_eq!(
        regression_count, 0,
        "Found {regression_count} regressions in the known-passing set. \
         These queries previously passed but now fail."
    );
}

/// Validate that direct links from `column_lineage` (flat list) produce the
/// same column_dict as the existing projection pipeline.
///
/// This ensures the flat list is a faithful representation of the same data —
/// direct links reconstructed into a dict must match `column_dict` exactly.
/// If column_dict matches Snowflake, so do the direct links (by transitivity).
#[test]
fn test_flat_lineage_direct_links_match_column_dict() {
    use altimate_core::lineage::complete::LineageType;

    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "Ground-truth fixture not found: {}. Skipping.",
            fixture_path.display()
        );
        return;
    }

    let file = std::fs::File::open(&fixture_path).expect("failed to open fixture");
    let reader = std::io::BufReader::with_capacity(1024 * 1024, file);

    let dialect = SqlDialect::Snowflake;
    let mut total = 0u64;
    let mut consistent = 0u64;
    let mut inconsistent = 0u64;
    let mut parse_errors = 0u64;

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: GroundTruthCase = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => {
                parse_errors += 1;
                continue;
            }
        };

        // Reconstruct column_dict from direct links in the flat list
        let mut reconstructed: HashMap<String, Vec<String>> = HashMap::new();
        for entry in &result.column_lineage {
            if entry.lineage_type == LineageType::Direct {
                reconstructed
                    .entry(entry.target.clone())
                    .or_default()
                    .push(entry.source.clone());
            }
        }
        // Sort + dedup to match column_dict format
        for sources in reconstructed.values_mut() {
            sources.sort();
            sources.dedup();
        }

        let column_dict = normalize(&result.column_dict);

        if reconstructed == column_dict {
            consistent += 1;
        } else {
            inconsistent += 1;
            if inconsistent <= 3 {
                eprintln!("\n--- INCONSISTENT #{inconsistent}: {} ---", case.name);
                for (col, dict_srcs) in &column_dict {
                    let flat_srcs = reconstructed.get(col).cloned().unwrap_or_default();
                    if flat_srcs != *dict_srcs {
                        eprintln!("  col '{col}': dict={dict_srcs:?} flat_direct={flat_srcs:?}");
                    }
                }
            }
        }

        if total % 5000 == 0 {
            eprintln!("  ... {total}: consistent={consistent} inconsistent={inconsistent}");
        }
    }

    let parseable = total - parse_errors;
    let consistency_rate = if parseable > 0 {
        consistent as f64 / parseable as f64 * 100.0
    } else {
        0.0
    };

    eprintln!(
        "\n=== Flat Lineage Direct Links Consistency ===\n\
         Total:        {total}\n\
         Parseable:    {parseable}\n\
         Consistent:   {consistent} ({consistency_rate:.1}%)\n\
         Inconsistent: {inconsistent}\n\
         Parse errors: {parse_errors}"
    );

    // Direct links MUST be 100% consistent with column_dict
    assert_eq!(
        inconsistent, 0,
        "Direct links from column_lineage must exactly match column_dict. \
         Found {inconsistent} inconsistent queries out of {parseable} parseable."
    );
}
