//! Data-driven column lineage tests.
//!
//! Loads test cases from JSONL fixtures and validates our Rust implementation
//! produces matching output. Supports both "complete" (inject_shared) and
//! "projection" (projection-only) modes across all 34 dialects.

use altimate_core::lineage::{ColumnLineageOptions, column_lineage};
use altimate_core::schema::types::SqlDialect;
use serde::Deserialize;
use std::collections::HashMap;
use std::fs;
use std::path::Path;

#[derive(Debug, Deserialize)]
struct LineageTestCase {
    name: String,
    sql: String,
    #[allow(dead_code)]
    schema: serde_json::Value,
    dialect: String,
    #[serde(default = "default_mode")]
    mode: String,
    expected: HashMap<String, Vec<String>>,
}

fn default_mode() -> String {
    "complete".to_string()
}

/// Map dialect strings from fixture data to SqlDialect variants.
/// Supports all 34 dialects plus common aliases.
fn parse_dialect(s: &str) -> SqlDialect {
    match s {
        "generic" => SqlDialect::Generic,
        "snowflake" => SqlDialect::Snowflake,
        "postgres" => SqlDialect::Postgres,
        "bigquery" => SqlDialect::BigQuery,
        "databricks" => SqlDialect::Databricks,
        "duckdb" => SqlDialect::DuckDB,
        "athena" => SqlDialect::Athena,
        "clickhouse" => SqlDialect::ClickHouse,
        "cockroachdb" => SqlDialect::CockroachDB,
        "datafusion" => SqlDialect::DataFusion,
        "doris" => SqlDialect::Doris,
        "dremio" => SqlDialect::Dremio,
        "drill" => SqlDialect::Drill,
        "druid" => SqlDialect::Druid,
        "dune" => SqlDialect::Dune,
        "exasol" => SqlDialect::Exasol,
        "fabric" => SqlDialect::Fabric,
        "hive" => SqlDialect::Hive,
        "materialize" => SqlDialect::Materialize,
        "mysql" => SqlDialect::MySQL,
        "oracle" => SqlDialect::Oracle,
        "presto" => SqlDialect::Presto,
        "redshift" => SqlDialect::Redshift,
        "risingwave" => SqlDialect::RisingWave,
        "singlestore" => SqlDialect::SingleStore,
        "solr" => SqlDialect::Solr,
        "spark" => SqlDialect::Spark,
        "sqlite" => SqlDialect::SQLite,
        "starrocks" => SqlDialect::StarRocks,
        "tableau" => SqlDialect::Tableau,
        "teradata" => SqlDialect::Teradata,
        "tidb" => SqlDialect::TiDB,
        "trino" => SqlDialect::Trino,
        "tsql" | "mssql" => SqlDialect::TSQL,
        other => {
            eprintln!("WARNING: unknown dialect '{other}', falling back to Generic");
            SqlDialect::Generic
        }
    }
}

fn load_jsonl(path: &Path) -> Vec<LineageTestCase> {
    let content = fs::read_to_string(path)
        .unwrap_or_else(|e| panic!("failed to read fixture {}: {e}", path.display()));
    content
        .lines()
        .filter(|l| !l.trim().is_empty())
        .enumerate()
        .map(|(i, line)| {
            serde_json::from_str(line)
                .unwrap_or_else(|e| panic!("failed to parse line {}: {e}", i + 1))
        })
        .collect()
}

fn normalize_expected(expected: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    expected
        .iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

fn normalize_actual(actual: &HashMap<String, Vec<String>>) -> HashMap<String, Vec<String>> {
    actual
        .iter()
        .map(|(k, v)| {
            let mut sorted = v.clone();
            sorted.sort();
            sorted.dedup();
            (k.clone(), sorted)
        })
        .collect()
}

fn schema_from_json(value: &serde_json::Value) -> HashMap<String, serde_json::Value> {
    match value {
        serde_json::Value::Object(map) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
        _ => HashMap::new(),
    }
}

/// Core fixture test — runs all cases from column_lineage_complete.jsonl.
/// Handles both "complete" (inject_shared=true) and "projection" (None) modes.
#[test]
fn test_column_lineage_from_fixtures() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/column_lineage_complete.jsonl");

    if !fixture_path.exists() {
        panic!(
            "Fixture file not found: {}. Run scripts/extract_lineage_fixtures.py first.",
            fixture_path.display()
        );
    }

    let cases = load_jsonl(&fixture_path);
    assert!(!cases.is_empty(), "No test cases loaded from fixture file");

    let mut failures: Vec<String> = Vec::new();
    let mut passes = 0;

    for case in &cases {
        let dialect = parse_dialect(&case.dialect);
        let schema = schema_from_json(&case.schema);

        let opts = match case.mode.as_str() {
            "projection" => None,
            _ => Some(ColumnLineageOptions {
                inject_shared: true,
                ..Default::default()
            }),
        };

        let result = match column_lineage(&case.sql, dialect, &schema, opts.as_ref()) {
            Ok(r) => r,
            Err(e) => {
                failures.push(format!("{}: engine error: {e}", case.name));
                continue;
            }
        };

        let actual = normalize_actual(&result.column_dict);
        let expected = normalize_expected(&case.expected);

        if actual != expected {
            let mut diff = String::new();
            for (key, exp_vals) in &expected {
                match actual.get(key) {
                    Some(act_vals) if act_vals != exp_vals => {
                        let missing: Vec<_> =
                            exp_vals.iter().filter(|v| !act_vals.contains(v)).collect();
                        let extra: Vec<_> =
                            act_vals.iter().filter(|v| !exp_vals.contains(v)).collect();
                        diff.push_str(&format!("  column '{key}':"));
                        if !missing.is_empty() {
                            diff.push_str(&format!(" missing={missing:?}"));
                        }
                        if !extra.is_empty() {
                            diff.push_str(&format!(" extra={extra:?}"));
                        }
                        diff.push('\n');
                    }
                    None => {
                        diff.push_str(&format!("  column '{key}': MISSING in actual\n"));
                    }
                    _ => {}
                }
            }
            for key in actual.keys() {
                if !expected.contains_key(key) {
                    diff.push_str(&format!("  column '{key}': UNEXPECTED in actual\n"));
                }
            }
            failures.push(format!("{}:\n{diff}", case.name));
        } else {
            passes += 1;
        }
    }

    if !failures.is_empty() {
        let total = passes + failures.len();
        let names: Vec<_> = failures
            .iter()
            .map(|f| f.split(':').next().unwrap_or("?"))
            .collect();
        let sample_size = failures.len().min(40);
        panic!(
            "\n\nColumn lineage failures: {}/{total} tests failed\n\nAll failing names:\n{}\n\nFirst {sample_size} details:\n{}\n",
            failures.len(),
            names.join("\n"),
            failures[..sample_size].join("\n---\n")
        );
    }

    eprintln!("All {} column lineage tests passed", passes);
}

/// Snowflake ground-truth benchmark (19,875 production queries).
#[test]
fn test_column_lineage_snowflake_ground_truth() {
    use std::io::BufRead;

    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/snowflake_ground_truth.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "Ground-truth fixture not found: {}. Skipping.",
            fixture_path.display()
        );
        return;
    }

    let file = fs::File::open(&fixture_path).expect("failed to open fixture file");
    let reader = std::io::BufReader::new(file);

    let dialect = parse_dialect("snowflake");
    let mut strict_passes = 0;
    let mut lenient_passes = 0;
    let mut parse_errors = 0;
    let mut mismatches = 0;
    let mut total = 0;

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: LineageTestCase = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(_) => {
                parse_errors += 1;
                continue;
            }
        };

        let actual = normalize_actual(&result.column_dict);
        let expected = normalize_expected(&case.expected);

        if actual == expected {
            strict_passes += 1;
            lenient_passes += 1;
        } else {
            let lenient_ok = expected.iter().all(|(col, exp_sources)| {
                actual
                    .get(col)
                    .map_or(false, |act_sources| act_sources == exp_sources)
            });
            if lenient_ok {
                lenient_passes += 1;
            } else {
                mismatches += 1;
            }
        }

        if total % 1000 == 0 {
            eprintln!(
                "  ... processed {total} cases ({strict_passes} strict, {lenient_passes} lenient, {mismatches} mismatch, {parse_errors} err)"
            );
        }
    }

    let strict_rate = if total > 0 {
        strict_passes as f64 / total as f64 * 100.0
    } else {
        0.0
    };
    let lenient_rate = if total > 0 {
        lenient_passes as f64 / total as f64 * 100.0
    } else {
        0.0
    };

    eprintln!(
        "\nSnowflake ground-truth results: {total} cases\n  \
         Strict match:  {strict_passes} ({strict_rate:.1}%)\n  \
         Lenient match: {lenient_passes} ({lenient_rate:.1}%)\n  \
         Mismatch:      {mismatches}\n  \
         Parse errors:  {parse_errors}"
    );

    let parseable = strict_passes + lenient_passes - strict_passes + mismatches;
    let parse_rate = (parseable + strict_passes) as f64 / total as f64 * 100.0;
    assert!(
        parse_rate > 50.0,
        "Parse rate too low: {parse_rate:.1}% parseable"
    );
}

/// DataHub golden regression tests (calibrated against our engine).
#[test]
fn test_column_lineage_datahub_goldens() {
    use std::io::BufRead;

    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/lineage_datahub.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "DataHub fixture not found: {}. Skipping.",
            fixture_path.display()
        );
        return;
    }

    let file = fs::File::open(&fixture_path).expect("failed to open fixture file");
    let reader = std::io::BufReader::new(file);

    let mut strict_passes = 0;
    let mut engine_errors = 0;
    let mut mismatches = 0;
    let mut total = 0;
    let mut dialect_stats: HashMap<String, (usize, usize)> = HashMap::new();

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: LineageTestCase = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let dialect = parse_dialect(&case.dialect);
        let schema = schema_from_json(&case.schema);
        let entry = dialect_stats.entry(case.dialect.clone()).or_insert((0, 0));
        entry.0 += 1;

        let opts = match case.mode.as_str() {
            "projection" => None,
            _ => Some(ColumnLineageOptions {
                inject_shared: true,
                ..Default::default()
            }),
        };

        let result = match column_lineage(&case.sql, dialect, &schema, opts.as_ref()) {
            Ok(r) => r,
            Err(_) => {
                engine_errors += 1;
                continue;
            }
        };

        let actual = normalize_actual(&result.column_dict);
        let expected = normalize_expected(&case.expected);

        if actual == expected {
            strict_passes += 1;
            entry.1 += 1;
        } else {
            mismatches += 1;
        }
    }

    let strict_rate = if total > 0 {
        strict_passes as f64 / total as f64 * 100.0
    } else {
        0.0
    };

    eprintln!(
        "\nDataHub golden results: {total} cases\n  \
         Strict match:  {strict_passes} ({strict_rate:.1}%)\n  \
         Mismatch:      {mismatches}\n  \
         Engine errors: {engine_errors}"
    );

    let mut dialects: Vec<_> = dialect_stats.iter().collect();
    dialects.sort_by_key(|(name, _)| (*name).clone());
    for (name, (total_d, pass_d)) in &dialects {
        let rate = if *total_d > 0 {
            *pass_d as f64 / *total_d as f64 * 100.0
        } else {
            0.0
        };
        eprintln!("  {name}: {pass_d}/{total_d} ({rate:.0}%)");
    }

    assert_eq!(
        mismatches, 0,
        "DataHub regression: {mismatches}/{total} tests changed behavior"
    );
}

/// S3 failed production requests — must-not-crash regression tests.
#[test]
fn test_column_lineage_s3_failures() {
    use std::io::BufRead;

    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/lineage_s3_failures.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "S3 failures fixture not found: {}. Skipping.",
            fixture_path.display()
        );
        return;
    }

    let file = fs::File::open(&fixture_path).expect("failed to open fixture file");
    let reader = std::io::BufReader::new(file);

    let mut total = 0;
    let mut ok = 0;
    let mut engine_errors = 0;
    let mut dialect_stats: HashMap<String, (usize, usize)> = HashMap::new();

    for line in reader.lines() {
        let line = line.expect("failed to read line");
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        let case: LineageTestCase = match serde_json::from_str(line) {
            Ok(c) => c,
            Err(_) => continue,
        };
        total += 1;

        let dialect = parse_dialect(&case.dialect);
        let schema = schema_from_json(&case.schema);
        let entry = dialect_stats.entry(case.dialect.clone()).or_insert((0, 0));
        entry.0 += 1;

        match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(_) => {
                ok += 1;
                entry.1 += 1;
            }
            Err(_) => {
                engine_errors += 1;
            }
        }
    }

    let ok_rate = if total > 0 {
        ok as f64 / total as f64 * 100.0
    } else {
        0.0
    };

    eprintln!(
        "\nS3 failure regression results: {total} cases\n  \
         OK (no crash): {ok} ({ok_rate:.1}%)\n  \
         Engine errors: {engine_errors}"
    );

    let mut dialects: Vec<_> = dialect_stats.iter().collect();
    dialects.sort_by_key(|(name, _)| (*name).clone());
    for (name, (total_d, pass_d)) in &dialects {
        let rate = if *total_d > 0 {
            *pass_d as f64 / *total_d as f64 * 100.0
        } else {
            0.0
        };
        eprintln!("  {name}: {pass_d}/{total_d} ({rate:.0}%)");
    }

    assert!(ok_rate > 70.0, "S3 failure OK rate too low: {ok_rate:.1}%");
}

/// Community-sourced column lineage tests from clgraph, OpenLineage,
/// sqlglot, sqllineage, DataHub, and OpenMetadata complex query patterns.
///
/// These fixtures were scraped from open-source SQL lineage tools, calibrated
/// through our engine, and deduplicated against existing fixtures.
/// Strict regression: any mismatch means a regression in our lineage engine.
#[test]
fn test_column_lineage_community_goldens() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/lineage_community.jsonl");

    if !fixture_path.exists() {
        eprintln!(
            "Community fixture not found: {}. Skipping.",
            fixture_path.display()
        );
        return;
    }

    let cases = load_jsonl(&fixture_path);
    assert!(
        !cases.is_empty(),
        "No test cases loaded from community fixture"
    );

    let mut mismatches: Vec<String> = Vec::new();
    let mut passes = 0;

    for case in &cases {
        let dialect = parse_dialect(&case.dialect);
        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(e) => {
                mismatches.push(format!("{}: engine error: {e}", case.name));
                continue;
            }
        };

        let actual = normalize_actual(&result.column_dict);
        let expected = normalize_expected(&case.expected);

        if actual != expected {
            let mut diff = String::new();
            for (key, exp_vals) in &expected {
                match actual.get(key) {
                    Some(act_vals) if act_vals != exp_vals => {
                        let missing: Vec<_> =
                            exp_vals.iter().filter(|v| !act_vals.contains(v)).collect();
                        let extra: Vec<_> =
                            act_vals.iter().filter(|v| !exp_vals.contains(v)).collect();
                        diff.push_str(&format!("  column '{key}':"));
                        if !missing.is_empty() {
                            diff.push_str(&format!(" missing={missing:?}"));
                        }
                        if !extra.is_empty() {
                            diff.push_str(&format!(" extra={extra:?}"));
                        }
                        diff.push('\n');
                    }
                    None => {
                        diff.push_str(&format!("  column '{key}': MISSING in actual\n"));
                    }
                    _ => {}
                }
            }
            for key in actual.keys() {
                if !expected.contains_key(key) {
                    diff.push_str(&format!("  column '{key}': UNEXPECTED in actual\n"));
                }
            }
            mismatches.push(format!("{}:\n{diff}", case.name));
        } else {
            passes += 1;
        }
    }

    let total = passes + mismatches.len();
    eprintln!("Community goldens: {passes}/{total} passed");

    assert_eq!(
        mismatches.len(),
        0,
        "\n\nCommunity lineage regressions: {}/{total} tests failed\n\nFirst {} details:\n{}\n",
        mismatches.len(),
        mismatches.len().min(20),
        mismatches[..mismatches.len().min(20)].join("\n---\n")
    );
}

/// Calibrated column lineage tests from OpenMetadata and reata/sqllineage
/// test suites.
///
/// These fixtures were extracted from Python test files, DML-wrapper-stripped
/// to pure SELECT/WITH queries, run through our engine in projection mode,
/// and calibrated with the actual results. Strict regression: any mismatch
/// means a regression in our lineage engine.
#[test]
fn test_column_lineage_openmetadata_goldens() {
    let fixture_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("../../tests/fixtures/lineage/lineage_openmetadata.jsonl");

    if !fixture_path.exists() {
        panic!(
            "OpenMetadata fixture not found: {}. Run the calibration pipeline first.",
            fixture_path.display()
        );
    }

    let cases = load_jsonl(&fixture_path);
    assert!(
        !cases.is_empty(),
        "No test cases loaded from OpenMetadata fixture"
    );

    let mut mismatches: Vec<String> = Vec::new();
    let mut passes = 0;

    for case in &cases {
        let dialect = parse_dialect(&case.dialect);
        let schema = schema_from_json(&case.schema);

        let result = match column_lineage(&case.sql, dialect, &schema, None) {
            Ok(r) => r,
            Err(e) => {
                mismatches.push(format!("{}: engine error: {e}", case.name));
                continue;
            }
        };

        let actual = normalize_actual(&result.column_dict);
        let expected = normalize_expected(&case.expected);

        if actual != expected {
            let mut diff = String::new();
            for (key, exp_vals) in &expected {
                match actual.get(key) {
                    Some(act_vals) if act_vals != exp_vals => {
                        let missing: Vec<_> =
                            exp_vals.iter().filter(|v| !act_vals.contains(v)).collect();
                        let extra: Vec<_> =
                            act_vals.iter().filter(|v| !exp_vals.contains(v)).collect();
                        diff.push_str(&format!("  column '{key}':"));
                        if !missing.is_empty() {
                            diff.push_str(&format!(" missing={missing:?}"));
                        }
                        if !extra.is_empty() {
                            diff.push_str(&format!(" extra={extra:?}"));
                        }
                        diff.push('\n');
                    }
                    None => {
                        diff.push_str(&format!("  column '{key}': MISSING in actual\n"));
                    }
                    _ => {}
                }
            }
            for key in actual.keys() {
                if !expected.contains_key(key) {
                    diff.push_str(&format!("  column '{key}': UNEXPECTED in actual\n"));
                }
            }
            mismatches.push(format!("{}:\n{diff}", case.name));
        } else {
            passes += 1;
        }
    }

    let total = passes + mismatches.len();
    eprintln!("OpenMetadata/sqllineage goldens: {passes}/{total} passed");

    assert_eq!(
        mismatches.len(),
        0,
        "\n\nOpenMetadata lineage regressions: {}/{total} tests failed\n\nFirst {} details:\n{}\n",
        mismatches.len(),
        mismatches.len().min(20),
        mismatches[..mismatches.len().min(20)].join("\n---\n")
    );
}
