//! Fellegi-Sunter probabilistic matching integration tests.
//!
//! These tests validate the full pipeline: YAML parsing → IR compilation →
//! engine scoring → decision output, plus plan observability and spec diff.

use cannon_common::spec::IdentitySpec;
use cannon_common::ir::ScoringMethod;
use cannon_core::engine::compiler::SpecCompiler;
use cannon_core::engine::ReconciliationEngine;
use cannon_core::engine::overrides::OverrideResolver;
use cannon_core::engine::types::{NormalizedEntity, Decision};
use std::collections::HashMap;
use uuid::Uuid;

// ============================================================================
// Helpers
// ============================================================================

fn make_entity(id: &str, data: Vec<(&str, &str)>) -> NormalizedEntity {
    let mut map = HashMap::new();
    for (k, v) in data {
        map.insert(k.to_string(), v.to_string());
    }
    NormalizedEntity {
        id: Uuid::new_v4(),
        tenant_id: Uuid::nil(),
        external_id: id.to_string(),
        entity_type: "customer".to_string(),
        data: map,
        source_name: "test".to_string(),
        valid_from: None,
        valid_to: None,
        last_updated: chrono::Utc::now(),
    }
}

fn fs_spec_yaml() -> &'static str {
    r#"
api_version: kanoniv/v2
identity_version: fs_test_v1

entity:
  name: customer

sources:
  - name: crm
    system: salesforce
    table: contacts
    id: contact_id
    attributes:
      first_name: first_name
      email: email_address

blocking:
  keys:
    - [first_name]

decision:
  scoring:
    strategy: fellegi_sunter
    fields:
      - name: first_name
        comparator: jaro_winkler
        weight: 1.0
        m_probability: 0.92
        u_probability: 0.02
      - name: email
        comparator: exact
        weight: 2.0
        m_probability: 0.99
        u_probability: 0.001
    thresholds:
      match: 6.0
      possible: 2.0
      non_match: -4.0
  thresholds:
    match: 0.9

governance:
  log_probabilities: true
  require_shadow_on_threshold_change: true
"#
}

fn fs_spec_with_em_yaml() -> &'static str {
    r#"
api_version: kanoniv/v2
identity_version: fs_em_test_v1

entity:
  name: customer

sources:
  - name: crm
    system: test
    table: contacts
    id: id
    attributes:
      first_name: first_name
      email: email

blocking:
  keys:
    - [first_name]

decision:
  scoring:
    strategy: fellegi_sunter
    training:
      method: em
      max_iterations: 10
      convergence_threshold: 0.01
    fields:
      - name: first_name
        comparator: jaro_winkler
        weight: 1.0
        m_probability: 0.9
        u_probability: 0.1
      - name: email
        comparator: exact
        weight: 1.0
        m_probability: 0.9
        u_probability: 0.1
    thresholds:
      match: 3.0
      possible: 1.0
      non_match: -3.0
  thresholds:
    match: 0.9
"#
}

// ============================================================================
// Test: fs_spec_parses — YAML with strategy: fellegi_sunter deserializes
// ============================================================================

#[test]
fn fs_spec_parses() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).expect("Failed to parse FS spec");

    let scoring = spec.decision.scoring.as_ref().expect("scoring should be present");
    assert_eq!(
        scoring.strategy,
        Some(cannon_common::spec::ScoringStrategy::FellegiSunter)
    );

    let fields = scoring.fields.as_ref().expect("fields should be present");
    assert_eq!(fields.len(), 2);
    assert_eq!(fields[0].name, "first_name");
    assert_eq!(fields[0].comparator, cannon_common::spec::FsComparator::JaroWinkler);
    assert!((fields[0].m_probability - 0.92).abs() < f64::EPSILON);
    assert!((fields[0].u_probability - 0.02).abs() < f64::EPSILON);

    assert_eq!(fields[1].name, "email");
    assert_eq!(fields[1].comparator, cannon_common::spec::FsComparator::Exact);
    assert!((fields[1].weight - 2.0).abs() < f64::EPSILON);

    let thresholds = scoring.thresholds.as_ref().expect("thresholds should be present");
    assert!((thresholds.match_threshold - 6.0).abs() < f64::EPSILON);
    assert!((thresholds.possible - 2.0).abs() < f64::EPSILON);
    assert!((thresholds.non_match - -4.0).abs() < f64::EPSILON);

    // Governance fields
    let gov = spec.governance.as_ref().expect("governance should be present");
    assert!(gov.log_probabilities);
    assert!(gov.require_shadow_on_threshold_change);
}

// ============================================================================
// Test: fs_compiles_to_ir — compiler produces CompiledFellegiSunterPlan
// ============================================================================

#[test]
fn fs_compiles_to_ir() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).expect("Compilation should succeed");

    assert_eq!(plan.decision.scoring_method, ScoringMethod::FellegiSunter);

    let fs = plan.decision.fellegi_sunter.as_ref().expect("FS plan should be present");
    assert_eq!(fs.fields.len(), 2);

    // Verify precomputed weights
    let first_name = &fs.fields[0];
    assert_eq!(first_name.name, "first_name");
    let expected_w_agree = (0.92_f64 / 0.02).ln() * 1.0;
    assert!((first_name.w_agree - expected_w_agree).abs() < 1e-6,
        "first_name w_agree: expected {}, got {}", expected_w_agree, first_name.w_agree);

    let email = &fs.fields[1];
    assert_eq!(email.name, "email");
    let expected_email_agree = (0.99_f64 / 0.001).ln() * 2.0;
    assert!((email.w_agree - expected_email_agree).abs() < 1e-6,
        "email w_agree: expected {}, got {}", expected_email_agree, email.w_agree);

    // Verify max/min composite
    let expected_max = expected_w_agree + expected_email_agree;
    assert!((fs.max_composite - expected_max).abs() < 1e-6);

    // Verify thresholds
    assert!((fs.match_threshold - 6.0).abs() < f64::EPSILON);
    assert!((fs.possible_threshold - 2.0).abs() < f64::EPSILON);

    // Verify implicit rules were generated
    assert!(plan.match_graph.rules.iter().any(|r| r.name == "first_name"),
        "Should have implicit rule for first_name");
    assert!(plan.match_graph.rules.iter().any(|r| r.name == "email"),
        "Should have implicit rule for email");
}

// ============================================================================
// Test: fs_perfect_match — all fields agree → composite > match → Merge
// ============================================================================

#[test]
fn fs_perfect_match() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let engine = ReconciliationEngine::from_plan(&plan);
    let resolver = OverrideResolver::new(vec![]);

    let a = make_entity("a1", vec![
        ("first_name", "john"),
        ("email", "john@example.com"),
    ]);
    let b = make_entity("b1", vec![
        ("first_name", "john"),
        ("email", "john@example.com"),
    ]);

    let decision = engine.evaluate_pair(&a, &b, &resolver, 1.0);
    assert_eq!(decision.decision, Decision::Merge,
        "Perfect match should be Merge, got {:?} with confidence {}", decision.decision, decision.confidence);
    // Confidence should be the sum of all w_agree values (max composite)
    let fs = plan.decision.fellegi_sunter.as_ref().unwrap();
    assert!((decision.confidence - fs.max_composite).abs() < 1.0,
        "Confidence should be near max composite {}, got {}", fs.max_composite, decision.confidence);
}

// ============================================================================
// Test: fs_no_match — all fields disagree → composite < non_match → NoMerge
// ============================================================================

#[test]
fn fs_no_match() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let engine = ReconciliationEngine::from_plan(&plan);
    let resolver = OverrideResolver::new(vec![]);

    let a = make_entity("a1", vec![
        ("first_name", "alice"),
        ("email", "alice@example.com"),
    ]);
    let b = make_entity("b1", vec![
        ("first_name", "bob"),
        ("email", "bob@other.com"),
    ]);

    let decision = engine.evaluate_pair(&a, &b, &resolver, 1.0);
    assert_eq!(decision.decision, Decision::NoMerge,
        "Complete mismatch should be NoMerge, got {:?} with confidence {}", decision.decision, decision.confidence);
}

// ============================================================================
// Test: fs_partial_review — mixed scores → Review zone
// ============================================================================

#[test]
fn fs_partial_review() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let engine = ReconciliationEngine::from_plan(&plan);
    let resolver = OverrideResolver::new(vec![]);
    let fs = plan.decision.fellegi_sunter.as_ref().unwrap();

    // Same first name (high similarity), different email (no match)
    let a = make_entity("a1", vec![
        ("first_name", "john"),
        ("email", "john@company.com"),
    ]);
    let b = make_entity("b1", vec![
        ("first_name", "john"),
        ("email", "different@other.com"),
    ]);

    let decision = engine.evaluate_pair(&a, &b, &resolver, 1.0);
    // first_name matches perfectly → gets w_agree for that field (~3.83)
    // email doesn't match → gets w_disagree for that field (very negative due to weight=2.0, u=0.001)
    // With these parameters, even a perfect name match can't overcome email disagreement,
    // which correctly shows that email is the dominant discriminator
    assert!(decision.confidence < fs.match_threshold,
        "Partial match confidence {} should be below match threshold {}",
        decision.confidence, fs.match_threshold);
    // The confidence should still reflect the contribution of the matching name field
    assert!(decision.confidence > fs.min_composite,
        "Partial match confidence {} should be above min composite {}",
        decision.confidence, fs.min_composite);
}

// ============================================================================
// Test: fs_rare_field_dominates — email (u=0.001) agreement swings composite
// ============================================================================

#[test]
fn fs_rare_field_dominates() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let engine = ReconciliationEngine::from_plan(&plan);
    let resolver = OverrideResolver::new(vec![]);

    // Different first names but same email
    let a = make_entity("a1", vec![
        ("first_name", "alice"),
        ("email", "shared@example.com"),
    ]);
    let b = make_entity("b1", vec![
        ("first_name", "bob"),
        ("email", "shared@example.com"),
    ]);

    let _decision_email_match = engine.evaluate_pair(&a, &b, &resolver, 1.0);

    // Same first names but different email
    let c = make_entity("c1", vec![
        ("first_name", "john"),
        ("email", "john@company.com"),
    ]);
    let d = make_entity("d1", vec![
        ("first_name", "john"),
        ("email", "different@other.com"),
    ]);

    let _decision_name_match = engine.evaluate_pair(&c, &d, &resolver, 1.0);

    // Email match should contribute more to confidence than name match alone
    // because email has weight=2.0 and u=0.001 (very rare coincidence)
    let fs = plan.decision.fellegi_sunter.as_ref().unwrap();
    let email_w_agree = fs.fields.iter().find(|f| f.name == "email").unwrap().w_agree;
    let name_w_agree = fs.fields.iter().find(|f| f.name == "first_name").unwrap().w_agree;

    assert!(email_w_agree > name_w_agree,
        "Email w_agree ({}) should exceed first_name w_agree ({})", email_w_agree, name_w_agree);
}

// ============================================================================
// Test: fs_weight_multiplier — weight: 2.0 has double impact vs weight: 1.0
// ============================================================================

#[test]
fn fs_weight_multiplier() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let fs = plan.decision.fellegi_sunter.as_ref().unwrap();

    let first_name = fs.fields.iter().find(|f| f.name == "first_name").unwrap();
    let email = fs.fields.iter().find(|f| f.name == "email").unwrap();

    // email has weight=2.0, first_name has weight=1.0
    // If they had the same m/u, email's w_agree would be exactly 2x
    // Since they have different m/u, just verify the weight multiplier effect
    let m_fn = first_name.m_probability;
    let u_fn = first_name.u_probability;
    let base_w_agree_fn = (m_fn / u_fn).ln();

    assert!((first_name.w_agree - base_w_agree_fn * 1.0).abs() < 1e-6,
        "first_name w_agree should equal base * weight(1.0)");

    let m_em = email.m_probability;
    let u_em = email.u_probability;
    let base_w_agree_em = (m_em / u_em).ln();

    assert!((email.w_agree - base_w_agree_em * 2.0).abs() < 1e-6,
        "email w_agree should equal base * weight(2.0)");
}

// ============================================================================
// Test: fs_plan_summary_risk_flags — plan detects FS-specific risks
// ============================================================================

#[test]
fn fs_plan_summary_risk_flags() {
    // Spec with narrow threshold gap and dominant field
    let yaml = r#"
api_version: kanoniv/v2
identity_version: risk_test_v1

entity:
  name: customer

sources:
  - name: crm
    system: test
    table: t
    id: id
    attributes:
      email: email
      name: name

blocking:
  keys:
    - [email]

decision:
  scoring:
    strategy: fellegi_sunter
    fields:
      - name: email
        comparator: exact
        weight: 5.0
        m_probability: 0.99
        u_probability: 0.001
      - name: name
        comparator: jaro_winkler
        weight: 0.1
        m_probability: 0.8
        u_probability: 0.3
    thresholds:
      match: 6.0
      possible: 5.5
      non_match: -4.0
  thresholds:
    match: 0.9
"#;

    let spec = IdentitySpec::from_yaml(yaml).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let summary = SpecCompiler::summarize(&spec, &plan);

    let risk_codes: Vec<&str> = summary.risk_analysis.flags.iter()
        .map(|f| f.code.as_str())
        .collect();

    // Should detect field dominance (email contributes >50% of max composite)
    assert!(risk_codes.contains(&"fs_field_dominance"),
        "Should detect field dominance. Flags: {:?}", risk_codes);

    // Should detect narrow threshold gap (6.0 - 5.5 = 0.5 < 1.0)
    assert!(risk_codes.contains(&"fs_narrow_threshold_gap"),
        "Should detect narrow threshold gap. Flags: {:?}", risk_codes);

    // Should detect m near u for name field (0.8 - 0.3 = 0.5, but that's >0.1, so no)
    // Actually m-u = 0.5 which is > 0.1 so this won't trigger
}

// ============================================================================
// Test: fs_m_near_u_risk_flag
// ============================================================================

#[test]
fn fs_m_near_u_risk_flag() {
    let yaml = r#"
api_version: kanoniv/v2
identity_version: risk_test_v2

entity:
  name: customer

sources:
  - name: crm
    system: test
    table: t
    id: id
    attributes:
      city: city

blocking:
  keys:
    - [city]

decision:
  scoring:
    strategy: fellegi_sunter
    fields:
      - name: city
        comparator: exact
        weight: 1.0
        m_probability: 0.55
        u_probability: 0.50
    thresholds:
      match: 0.5
      possible: 0.1
      non_match: -1.0
  thresholds:
    match: 0.9
"#;

    let spec = IdentitySpec::from_yaml(yaml).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let summary = SpecCompiler::summarize(&spec, &plan);

    let risk_codes: Vec<&str> = summary.risk_analysis.flags.iter()
        .map(|f| f.code.as_str())
        .collect();

    assert!(risk_codes.contains(&"fs_m_near_u"),
        "Should detect m near u (0.55 - 0.50 = 0.05 < 0.1). Flags: {:?}", risk_codes);
}

// ============================================================================
// Test: fs_spec_diff_detects_probability_change
// (Note: FS spec diff is tested in oss/validator/src/commands/diff.rs)
// ============================================================================

#[test]
fn fs_spec_diff_via_recompilation() {
    // Test that two FS specs with different m/u produce different compiled plans
    let spec_a_yaml = r#"
api_version: kanoniv/v2
identity_version: v1
entity:
  name: customer
sources:
  - name: crm
    system: test
    table: t
    id: id
    attributes:
      email: email
blocking:
  keys:
    - [email]
decision:
  scoring:
    strategy: fellegi_sunter
    fields:
      - name: email
        comparator: exact
        weight: 1.0
        m_probability: 0.95
        u_probability: 0.01
    thresholds:
      match: 5.0
      possible: 2.0
      non_match: -3.0
  thresholds:
    match: 0.9
"#;

    let spec_b_yaml = r#"
api_version: kanoniv/v2
identity_version: v2
entity:
  name: customer
sources:
  - name: crm
    system: test
    table: t
    id: id
    attributes:
      email: email
blocking:
  keys:
    - [email]
decision:
  scoring:
    strategy: fellegi_sunter
    fields:
      - name: email
        comparator: exact
        weight: 1.0
        m_probability: 0.99
        u_probability: 0.001
    thresholds:
      match: 5.0
      possible: 2.0
      non_match: -3.0
  thresholds:
    match: 0.9
"#;

    let spec_a = IdentitySpec::from_yaml(spec_a_yaml).unwrap();
    let spec_b = IdentitySpec::from_yaml(spec_b_yaml).unwrap();
    let plan_a = SpecCompiler::compile(&spec_a).unwrap();
    let plan_b = SpecCompiler::compile(&spec_b).unwrap();

    let fs_a = plan_a.decision.fellegi_sunter.as_ref().unwrap();
    let fs_b = plan_b.decision.fellegi_sunter.as_ref().unwrap();

    // Different m/u → different w_agree values
    assert!((fs_a.fields[0].w_agree - fs_b.fields[0].w_agree).abs() > 0.1,
        "Different m/u should produce different w_agree: {} vs {}", fs_a.fields[0].w_agree, fs_b.fields[0].w_agree);

    // Different max composites
    assert!((fs_a.max_composite - fs_b.max_composite).abs() > 0.1,
        "Different m/u should produce different max composites");

    // Plan hashes should differ
    assert_ne!(plan_a.plan_hash, plan_b.plan_hash,
        "Plans with different m/u should have different hashes");
}

// ============================================================================
// Test: fs_em_training_config_compiles
// ============================================================================

#[test]
fn fs_em_training_config_compiles() {
    let spec = IdentitySpec::from_yaml(fs_spec_with_em_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();

    let fs = plan.decision.fellegi_sunter.as_ref().expect("FS plan should be present");
    let em = fs.em_training.as_ref().expect("EM training config should be present");
    assert_eq!(em.max_iterations, 10);
    assert!((em.convergence_threshold - 0.01).abs() < 1e-10);
}

// ============================================================================
// Test: fs_plan_summary_has_probability_model
// ============================================================================

#[test]
fn fs_plan_summary_has_probability_model() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let summary = SpecCompiler::summarize(&spec, &plan);

    let model = summary.matching.probability_model.as_ref()
        .expect("Plan summary should have probability model");

    assert_eq!(model.fields_configured, 2);
    assert!(model.independence_assumption);
    assert!(model.max_possible_log_likelihood > 0.0);
    assert!(model.min_possible_log_likelihood < 0.0);
    assert_eq!(model.field_weights.len(), 2);

    // Dominance percentages should sum to ~100%
    let total_pct: f64 = model.field_weights.iter().map(|f| f.dominance_pct).sum();
    assert!((total_pct - 100.0).abs() < 1.0,
        "Dominance percentages should sum to ~100%, got {}", total_pct);
}

// ============================================================================
// Test: fs_rules_optional_when_strategy_set
// ============================================================================

#[test]
fn fs_rules_optional_when_strategy_set() {
    let yaml = r#"
api_version: kanoniv/v2
identity_version: no_rules_v1

entity:
  name: customer

sources:
  - name: crm
    system: test
    table: contacts
    id: id
    attributes:
      email: email

blocking:
  keys:
    - [email]

decision:
  scoring:
    strategy: fellegi_sunter
    fields:
      - name: email
        comparator: exact
        weight: 1.0
        m_probability: 0.99
        u_probability: 0.001
    thresholds:
      match: 5.0
      possible: 2.0
      non_match: -3.0
  thresholds:
    match: 0.9
"#;

    // Should parse without explicit rules section
    let spec = IdentitySpec::from_yaml(yaml).expect("Should parse without rules");
    assert!(spec.rules.is_empty());

    // Should compile successfully — implicit rules generated from FS fields
    let plan = SpecCompiler::compile(&spec).expect("Should compile without explicit rules");
    assert!(!plan.match_graph.rules.is_empty(), "Should have implicit rules from FS fields");
    assert_eq!(plan.decision.scoring_method, ScoringMethod::FellegiSunter);
}

// ============================================================================
// Test: fs_posterior_probability — P(match|data) via sigmoid
// ============================================================================

#[test]
fn fs_posterior_probability() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let engine = ReconciliationEngine::from_plan(&plan);
    let resolver = OverrideResolver::new(vec![]);

    // Perfect match: high confidence → posterior near 1.0
    let a = make_entity("a1", vec![
        ("first_name", "john"),
        ("email", "john@example.com"),
    ]);
    let b = make_entity("b1", vec![
        ("first_name", "john"),
        ("email", "john@example.com"),
    ]);

    let decision = engine.evaluate_pair(&a, &b, &resolver, 1.0);
    let posterior = decision.posterior_probability.expect("FS should have posterior probability");
    assert!(posterior > 0.99, "Perfect match posterior should be near 1.0, got {}", posterior);
    assert!(posterior <= 1.0, "Posterior should be <= 1.0");

    // Complete mismatch: low confidence → posterior near 0.0
    let c = make_entity("c1", vec![
        ("first_name", "alice"),
        ("email", "alice@example.com"),
    ]);
    let d = make_entity("d1", vec![
        ("first_name", "bob"),
        ("email", "bob@other.com"),
    ]);

    let decision2 = engine.evaluate_pair(&c, &d, &resolver, 1.0);
    let posterior2 = decision2.posterior_probability.expect("FS should have posterior probability");
    assert!(posterior2 < 0.01, "Mismatch posterior should be near 0.0, got {}", posterior2);
    assert!(posterior2 >= 0.0, "Posterior should be >= 0.0");
}

// ============================================================================
// Test: fs_posterior_none_for_weighted_sum — non-FS has no posterior
// ============================================================================

#[test]
fn fs_posterior_none_for_weighted_sum() {
    // Use a non-FS spec
    let yaml = r#"
api_version: kanoniv/v2
identity_version: weighted_v1

entity:
  name: customer

sources:
  - name: crm
    system: test
    table: t
    id: id
    attributes:
      email: email

blocking:
  keys:
    - [email]

rules:
  - name: email_exact
    type: exact
    field: email
    weight: 1.0

decision:
  thresholds:
    match: 0.5
"#;
    let spec = IdentitySpec::from_yaml(yaml).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let engine = ReconciliationEngine::from_plan(&plan);
    let resolver = OverrideResolver::new(vec![]);

    let a = make_entity("a1", vec![("email", "test@example.com")]);
    let b = make_entity("b1", vec![("email", "test@example.com")]);

    let decision = engine.evaluate_pair(&a, &b, &resolver, 1.0);
    assert!(decision.posterior_probability.is_none(),
        "Non-FS scoring should not have posterior probability");
}

// ============================================================================
// Test: fs_deterministic_output — same input produces identical output
// ============================================================================

#[test]
fn fs_deterministic_output() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let resolver = OverrideResolver::new(vec![]);

    // Create entities that share a blocking key
    let entities: Vec<NormalizedEntity> = (0..5).map(|i| {
        make_entity(&format!("e{}", i), vec![
            ("first_name", "john"),
            ("email", &format!("john{}@example.com", i)),
        ])
    }).collect();

    // Run reconciliation multiple times
    let mut results: Vec<Vec<(Uuid, Uuid, f64)>> = Vec::new();
    for _ in 0..3 {
        let engine = ReconciliationEngine::from_plan(&plan);
        let all_decisions = engine.reconcile(&entities, &resolver);
        let mut decisions: Vec<(Uuid, Uuid, f64)> = all_decisions.iter()
            .map(|d| {
                let min = d.entity_a_id.min(d.entity_b_id);
                let max = d.entity_a_id.max(d.entity_b_id);
                (min, max, d.confidence)
            })
            .collect();
        decisions.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
        results.push(decisions);
    }

    // All runs should produce identical output
    for i in 1..results.len() {
        assert_eq!(results[0].len(), results[i].len(),
            "Run {} has different number of decisions", i);
        for (j, (a, b)) in results[0].iter().zip(results[i].iter()).enumerate() {
            assert_eq!(a.0, b.0, "Decision {} entity_a differs in run {}", j, i);
            assert_eq!(a.1, b.1, "Decision {} entity_b differs in run {}", j, i);
            assert!((a.2 - b.2).abs() < 1e-10,
                "Decision {} confidence differs in run {}: {} vs {}", j, i, a.2, b.2);
        }
    }
}

// ============================================================================
// Test: fs_tie_breaking_lowest_entity_id — default tie-breaker
// ============================================================================

#[test]
fn fs_tie_breaking_lowest_entity_id() {
    let yaml = r#"
api_version: kanoniv/v2
identity_version: tie_test_v1

entity:
  name: customer

sources:
  - name: crm
    system: test
    table: t
    id: id
    attributes:
      name: name

blocking:
  keys:
    - [name]

decision:
  scoring:
    strategy: fellegi_sunter
    fields:
      - name: name
        comparator: exact
        weight: 1.0
        m_probability: 0.95
        u_probability: 0.05
    thresholds:
      match: 1.0
      possible: 0.0
      non_match: -2.0
  thresholds:
    match: 0.9
  tie_breaking:
    - lowest_entity_id
"#;
    let spec = IdentitySpec::from_yaml(yaml).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();

    assert!(!plan.decision.tie_breaking.is_empty(),
        "Should have tie-breaking rules");
    assert_eq!(
        plan.decision.tie_breaking[0],
        cannon_common::ir::CompiledTieBreaker::LowestEntityId,
    );
}

// ============================================================================
// Test: fs_cluster_with_confidence — per-cluster metrics
// ============================================================================

#[test]
fn fs_cluster_with_confidence() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let engine = ReconciliationEngine::from_plan(&plan);
    let resolver = OverrideResolver::new(vec![]);

    // Two entities that match perfectly
    let a = make_entity("a1", vec![
        ("first_name", "john"),
        ("email", "john@example.com"),
    ]);
    let b = make_entity("b1", vec![
        ("first_name", "john"),
        ("email", "john@example.com"),
    ]);

    let entities = vec![a, b];
    let all_decisions = engine.reconcile(&entities, &resolver);
    let clusters = engine.cluster_decisions_with_confidence(&entities, &all_decisions);

    // Should have one cluster with both entities
    let merged_clusters: Vec<_> = clusters.iter().filter(|c| c.members.len() > 1).collect();
    assert!(!merged_clusters.is_empty(), "Should have at least one merged cluster");

    let cluster = &merged_clusters[0];
    assert_eq!(cluster.members.len(), 2);
    assert!(cluster.avg_confidence > 0.0, "Cluster should have positive confidence");
    assert_eq!(cluster.pair_count, 1, "Two members = one pair");
    assert!((cluster.min_confidence - cluster.max_confidence).abs() < 1e-10,
        "Single pair: min should equal max");
}

// ============================================================================
// Test: fs_em_training_updates_weights — EM modifies compiled plan
// ============================================================================

#[test]
fn fs_em_training_updates_weights() {
    let spec = IdentitySpec::from_yaml(fs_spec_with_em_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();

    let fs_before = plan.decision.fellegi_sunter.as_ref().unwrap();
    let m_before: Vec<f64> = fs_before.fields.iter().map(|f| f.m_probability).collect();

    let mut engine = ReconciliationEngine::from_plan(&plan);

    // Create training data with a clear match pattern
    let entities: Vec<NormalizedEntity> = vec![
        make_entity("e1", vec![("first_name", "alice"), ("email", "alice@test.com")]),
        make_entity("e2", vec![("first_name", "alice"), ("email", "alice@test.com")]),
        make_entity("e3", vec![("first_name", "bob"), ("email", "bob@test.com")]),
        make_entity("e4", vec![("first_name", "bob"), ("email", "bob@test.com")]),
        make_entity("e5", vec![("first_name", "charlie"), ("email", "charlie@test.com")]),
        make_entity("e6", vec![("first_name", "charlie"), ("email", "diff@test.com")]),
    ];

    // This should trigger EM training since em_training is configured
    engine.train_fellegi_sunter(&entities);

    // After EM, weights should have been updated
    let fs_after = engine.fellegi_sunter_config.as_ref().unwrap();
    let m_after: Vec<f64> = fs_after.fields.iter().map(|f| f.m_probability).collect();

    // EM should have adjusted at least one m probability
    let any_changed = m_before.iter().zip(m_after.iter())
        .any(|(before, after)| (before - after).abs() > 1e-6);
    assert!(any_changed,
        "EM training should update m probabilities: before={:?}, after={:?}", m_before, m_after);

    // w_agree/w_disagree should be recomputed
    for field in &fs_after.fields {
        let expected_w_agree = (field.m_probability / field.u_probability).ln() * field.weight;
        assert!((field.w_agree - expected_w_agree).abs() < 1e-6,
            "w_agree should be recomputed after EM for field {}", field.name);
    }
}

// ============================================================================
// Test: fs_em_skipped_without_config — no EM config = no training
// ============================================================================

#[test]
fn fs_em_skipped_without_config() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();

    let fs_before = plan.decision.fellegi_sunter.as_ref().unwrap().clone();
    let mut engine = ReconciliationEngine::from_plan(&plan);

    let entities = vec![
        make_entity("e1", vec![("first_name", "alice"), ("email", "alice@test.com")]),
        make_entity("e2", vec![("first_name", "alice"), ("email", "alice@test.com")]),
    ];

    engine.train_fellegi_sunter(&entities);

    // No EM config → weights unchanged
    let fs_after = engine.fellegi_sunter_config.as_ref().unwrap();
    for (before, after) in fs_before.fields.iter().zip(fs_after.fields.iter()) {
        assert!((before.w_agree - after.w_agree).abs() < 1e-10,
            "w_agree should not change without EM config");
        assert!((before.w_disagree - after.w_disagree).abs() < 1e-10,
            "w_disagree should not change without EM config");
    }
}

// ============================================================================
// Test: fs_sensitivity_report — sensitivity analysis
// ============================================================================

#[test]
fn fs_sensitivity_report() {
    use cannon_core::engine::fellegi_sunter::SensitivityReport;

    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let fs = plan.decision.fellegi_sunter.as_ref().unwrap();

    // Generate sample scores spanning the range
    let n_samples = 100;
    let range = fs.max_composite - fs.min_composite;
    let sample_scores: Vec<f64> = (0..n_samples)
        .map(|i| fs.min_composite + (i as f64 / n_samples as f64) * range)
        .collect();

    let report = SensitivityReport::analyze(fs, &sample_scores);

    // Should have one entry per field
    assert_eq!(report.fields.len(), fs.fields.len(),
        "Sensitivity report should cover all fields");

    for field_report in &report.fields {
        // Elasticity values should be finite
        assert!(field_report.m_elasticity.is_finite(),
            "m_elasticity should be finite for {}", field_report.name);
        assert!(field_report.u_elasticity.is_finite(),
            "u_elasticity should be finite for {}", field_report.name);
        // Removal impact: match_rate change when field is removed (typically negative or zero)
        assert!(field_report.removal_impact.is_finite(),
            "removal_impact should be finite for {}", field_report.name);
    }

    // Threshold sensitivity
    assert!(report.threshold_sensitivity.match_rate_per_unit_threshold.is_finite(),
        "match_rate_per_unit_threshold should be finite");
}

// ============================================================================
// Test: fs_governance_compiled — governance flags in IR
// ============================================================================

#[test]
fn fs_governance_compiled() {
    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();

    let gov = plan.governance.as_ref().expect("Governance should be compiled");
    assert!(gov.log_probabilities, "log_probabilities should be true");
    assert!(gov.require_shadow_on_threshold_change,
        "require_shadow_on_threshold_change should be true");
}

// ============================================================================
// Test: fs_confidence_distribution — bimodality + percentiles
// ============================================================================

#[test]
fn fs_confidence_distribution() {
    use cannon_core::engine::observability::ConfidenceDistribution;

    let spec = IdentitySpec::from_yaml(fs_spec_yaml()).unwrap();
    let plan = SpecCompiler::compile(&spec).unwrap();
    let fs = plan.decision.fellegi_sunter.as_ref().unwrap();

    // Simulate a bimodal distribution: half near max, half near min
    let mut scores = Vec::new();
    for _ in 0..50 {
        scores.push(fs.max_composite * 0.9);
    }
    for _ in 0..50 {
        scores.push(fs.min_composite * 0.9);
    }

    let dist = ConfidenceDistribution::from_scores(&scores);

    // Should have buckets
    assert!(!dist.histogram.is_empty(), "Should have histogram buckets");

    // Percentiles should be ordered
    assert!(dist.percentiles.p5 <= dist.percentiles.p25);
    assert!(dist.percentiles.p25 <= dist.percentiles.p50);
    assert!(dist.percentiles.p50 <= dist.percentiles.p75);
    assert!(dist.percentiles.p75 <= dist.percentiles.p95);

    // Bimodality score should be high for bimodal data
    assert!(dist.bimodality_score > 0.3,
        "Bimodal data should have bimodality score > 0.3, got {}", dist.bimodality_score);
}

// ============================================================================
// Test: fs_build_cluster_assignment — from cluster list
// ============================================================================

#[test]
fn fs_build_cluster_assignment() {
    use cannon_common::observability::build_cluster_assignment;

    let e1 = Uuid::from_u128(1);
    let e2 = Uuid::from_u128(2);
    let e3 = Uuid::from_u128(3);
    let e4 = Uuid::from_u128(4);

    let clusters = vec![
        vec![e1, e2],  // cluster with smallest member = e1
        vec![e3, e4],  // cluster with smallest member = e3
    ];

    let assignment = build_cluster_assignment(&clusters);

    assert_eq!(assignment.len(), 4);
    assert_eq!(assignment[&e1], e1); // smallest in cluster
    assert_eq!(assignment[&e2], e1); // maps to smallest
    assert_eq!(assignment[&e3], e3); // smallest in cluster
    assert_eq!(assignment[&e4], e3); // maps to smallest
}
