pub mod types;
pub mod normalization;
pub mod matching;
pub mod blocking;
pub mod survivorship;
pub mod temporal;
pub mod overrides;
pub mod compiler;
pub mod observability;
pub mod expression;
pub mod fellegi_sunter;

use types::{NormalizedEntity, MatchDecision, Decision, ClusterWithConfidence};
use matching::MatchRule;
use blocking::BlockingStrategy;
use overrides::OverrideResolver;
use cannon_common::ir::{IdentityPlan, TemporalStrategy};
use temporal::{TemporalMatcher, TemporalResult};
use observability::{ReconciliationTelemetry, TelemetryCollector};
use hashbrown::{HashMap, HashSet};
use uuid::Uuid;
use rayon::prelude::*;
use std::sync::Mutex;

pub struct ReconciliationEngine {
    pub rules: Vec<Box<dyn MatchRule>>,
    pub blocking_strategies: Vec<Box<dyn BlockingStrategy>>,
    pub merge_threshold: f64,
    pub review_threshold: f64,
    pub max_block_size: usize,
    pub neighborhood_window: Option<usize>,
    pub sort_fields: Vec<String>,
    pub temporal_strategy: TemporalStrategy,
    pub reference_identifiers: Vec<cannon_common::ir::CompiledReferenceIdentifier>,
    pub relations: Vec<cannon_common::ir::CompiledRelation>,
    pub exclusions: Vec<cannon_common::ir::CompiledExclusion>,
    pub scoring_method: cannon_common::ir::ScoringMethod,
    pub ml_ensemble_config: Option<cannon_common::ir::CompiledMlEnsembleConfig>,
    pub custom_scoring_ops: Option<Vec<expression::Op>>,
    pub fellegi_sunter_config: Option<cannon_common::ir::CompiledFellegiSunterPlan>,
    /// Whether to log per-field FS probabilities in rule explanations
    pub log_probabilities: bool,
    /// Tie-breaking rules for deterministic ordering of equal-confidence pairs
    pub tie_breakers: Vec<cannon_common::ir::CompiledTieBreaker>,
}

impl ReconciliationEngine {
    pub fn new(
        rules: Vec<Box<dyn MatchRule>>,
        blocking_strategies: Vec<Box<dyn BlockingStrategy>>,
        merge_threshold: f64,
        review_threshold: f64,
    ) -> Self {
        Self {
            rules,
            blocking_strategies,
            merge_threshold,
            review_threshold,
            max_block_size: 1000,
            neighborhood_window: Some(50),
            sort_fields: vec![],
            temporal_strategy: TemporalStrategy::default(),
            reference_identifiers: vec![],
            relations: vec![],
            exclusions: vec![],
            scoring_method: cannon_common::ir::ScoringMethod::WeightedSum,
            ml_ensemble_config: None,
            custom_scoring_ops: None,
            fellegi_sunter_config: None,
            log_probabilities: false,
            tie_breakers: vec![cannon_common::ir::CompiledTieBreaker::LowestEntityId],
        }
    }

    pub fn from_plan(plan: &IdentityPlan) -> Self {
        let rules = plan.match_graph.rules.iter()
            .map(|r| Self::compile_rule(r))
            .collect();

        let mut blocking_strategies: Vec<Box<dyn BlockingStrategy>> = vec![
            Box::new(blocking::PlanBlocking { keys: plan.blocking.keys.clone() })
        ];

        if plan.blocking.strategy == cannon_common::ir::BlockingStrategy::Lsh {
            let fields: Vec<String> = plan.blocking.keys.iter()
                .flat_map(|k| k.fields.clone())
                .collect();
            let bands = plan.blocking.lsh_bands.unwrap_or(20);
            let rows = plan.blocking.lsh_rows.unwrap_or(5);
            blocking_strategies.push(Box::new(blocking::lsh::LshBlocking::new(fields, bands, rows)));
        }

        // Pre-compile custom scoring expression if present
        let custom_scoring_ops = plan.decision.custom_scoring_config.as_ref().and_then(|cfg| {
            match expression::compile_expression(&cfg.expression) {
                Ok(ops) => Some(ops),
                Err(e) => {
                    tracing::error!("Failed to compile custom scoring expression: {}", e);
                    None
                }
            }
        });

        // Determine log_probabilities from governance config
        let log_probabilities = plan.governance
            .as_ref()
            .map(|g| g.log_probabilities)
            .unwrap_or(false);

        let mut engine = Self {
            rules,
            blocking_strategies,
            merge_threshold: plan.decision.match_threshold,
            review_threshold: plan.decision.review_threshold.unwrap_or(plan.decision.match_threshold),
            max_block_size: 1000,
            neighborhood_window: plan.blocking.neighborhood_window,
            sort_fields: plan.blocking.sort_fields.clone(),
            temporal_strategy: TemporalStrategy::default(),
            reference_identifiers: plan.reference_identifiers.clone(),
            relations: plan.relations.clone(),
            exclusions: plan.exclusions.clone(),
            scoring_method: plan.decision.scoring_method.clone(),
            ml_ensemble_config: plan.decision.ml_ensemble_config.clone(),
            custom_scoring_ops,
            fellegi_sunter_config: plan.decision.fellegi_sunter.clone(),
            log_probabilities,
            tie_breakers: plan.decision.tie_breaking.clone(),
        };

        if let Some(first_source) = plan.sources.iter().find(|s| s.temporal.is_some()) {
            if let Some(ref t) = first_source.temporal {
                engine.temporal_strategy = t.strategy.clone();
            }
        }

        engine
    }

    /// Run EM training to estimate m/u probabilities from data.
    /// Call this before `reconcile()` when `training.method == "em"` in the spec.
    /// Generates blocked candidate pairs, runs EM, and updates the FS config in place.
    pub fn train_fellegi_sunter(&mut self, entities: &[NormalizedEntity]) {
        let fs = match self.fellegi_sunter_config.as_ref() {
            Some(fs) if fs.em_training.is_some() => fs.clone(),
            _ => return,
        };

        let em_config = fs.em_training.as_ref().unwrap();
        let trainer = fellegi_sunter::EmTrainer::new(
            em_config.max_iterations,
            em_config.convergence_threshold,
        );

        // Generate candidate pairs from blocking
        let mut groups: std::collections::BTreeMap<String, Vec<usize>> = std::collections::BTreeMap::new();
        for (idx, entity) in entities.iter().enumerate() {
            for strategy in &self.blocking_strategies {
                let keys = strategy.generate_block_key(entity);
                for key in keys {
                    groups.entry(key).or_default().push(idx);
                }
            }
        }

        // Collect pairs (cap at 10k for performance)
        let mut pairs: Vec<(NormalizedEntity, NormalizedEntity)> = Vec::new();
        let max_pairs = 10_000;
        'outer: for ids in groups.values() {
            for i in 0..ids.len() {
                for j in i + 1..ids.len() {
                    pairs.push((entities[ids[i]].clone(), entities[ids[j]].clone()));
                    if pairs.len() >= max_pairs {
                        break 'outer;
                    }
                }
            }
        }

        if pairs.is_empty() {
            return;
        }

        let result = trainer.train(&pairs, &fs.fields);

        if result.converged || result.iterations > 0 {
            if let Some(ref mut fs_config) = self.fellegi_sunter_config {
                fellegi_sunter::apply_em_results(fs_config, &result);
                tracing::info!(
                    "EM training completed: {} iterations, converged={}, fields={}",
                    result.iterations, result.converged, fs_config.fields.len()
                );
            }
        }
    }

    fn compile_rule(rule: &cannon_common::ir::CompiledRule) -> Box<dyn MatchRule> {
        match &rule.rule_type {
            cannon_common::ir::CompiledRuleType::Exact { field, weight, case_insensitive, normalize, normalizer } => {
                Box::new(matching::ExactMatch {
                    field: field.clone(),
                    weight: *weight,
                    case_insensitive: *case_insensitive,
                    normalize: *normalize,
                    normalizer: normalizer.clone(),
                    rule_name: rule.name.clone(),
                })
            },
            cannon_common::ir::CompiledRuleType::Similarity { field, algorithm, threshold, weight, normalizer } => {
                use cannon_common::ir::SimilarityAlgorithm;
                match algorithm {
                     SimilarityAlgorithm::JaroWinkler => Box::new(matching::FuzzyStringMatch {
                          field: field.clone(),
                          threshold: *threshold,
                          weight: *weight,
                          normalizer: normalizer.clone(),
                          rule_name: rule.name.clone(),
                      }),
                      SimilarityAlgorithm::Levenshtein => Box::new(matching::LevenshteinMatch {
                          field: field.clone(),
                          threshold: *threshold,
                          weight: *weight,
                          normalizer: normalizer.clone(),
                          rule_name: rule.name.clone(),
                      }),
                      SimilarityAlgorithm::Soundex => Box::new(matching::SoundexMatch {
                          field: field.clone(),
                          threshold: *threshold,
                          weight: *weight,
                          rule_name: rule.name.clone(),
                      }),
                      SimilarityAlgorithm::Metaphone => Box::new(matching::MetaphoneMatch {
                          field: field.clone(),
                          threshold: *threshold,
                          weight: *weight,
                          rule_name: rule.name.clone(),
                      }),
                      SimilarityAlgorithm::Cosine => Box::new(matching::CosineMatch {
                          field: field.clone(),
                          threshold: *threshold,
                          weight: *weight,
                          rule_name: rule.name.clone(),
                      }),
                      SimilarityAlgorithm::Haversine => Box::new(matching::HaversineMatch {
                          field: field.clone(),
                          threshold: *threshold,
                          weight: *weight,
                          rule_name: rule.name.clone(),
                      }),
                }
            },
            cannon_common::ir::CompiledRuleType::Range { field, tolerance, weight, normalizer } => {
                Box::new(matching::RangeMatch {
                    field: field.clone(),
                    tolerance: *tolerance,
                    weight: *weight,
                    normalizer: normalizer.clone(),
                    rule_name: rule.name.clone(),
                })
            },
            cannon_common::ir::CompiledRuleType::Composite { operator, children } => {
                let child_rules = children.iter().map(|c| Self::compile_rule(c)).collect();
                use cannon_common::ir::CompositeOperator;
                Box::new(matching::CompositeMatch {
                    rules: child_rules,
                    weight: 1.0,
                    require_all: match operator {
                        CompositeOperator::And => true,
                        CompositeOperator::Or => false,
                    },
                    rule_name: rule.name.clone(),
                })
            },
             cannon_common::ir::CompiledRuleType::Ml { model_id, features, threshold, weight } => {
                 Box::new(matching::MlMatch {
                     model_id: model_id.clone(),
                     features: features.clone(),
                     threshold: *threshold,
                     weight: *weight,
                     rule_name: rule.name.clone(),
                 })
             }
        }
    }

    /// Check if an entity matches any FieldValue exclusion.
    fn matches_exclusion(entity: &NormalizedEntity, exclusion: &cannon_common::ir::CompiledExclusion) -> bool {
        match exclusion {
            cannon_common::ir::CompiledExclusion::FieldValue { field, operator, value } => {
                if let Some(entity_val) = entity.data.get(field) {
                    match operator.as_str() {
                        "eq" | "==" => {
                            if let Some(s) = value.as_str() {
                                entity_val == s
                            } else {
                                false
                            }
                        }
                        "ne" | "!=" => {
                            if let Some(s) = value.as_str() {
                                entity_val != s
                            } else {
                                false
                            }
                        }
                        "contains" => {
                            if let Some(s) = value.as_str() {
                                entity_val.contains(s)
                            } else {
                                false
                            }
                        }
                        "is_empty" => entity_val.is_empty(),
                        _ => false,
                    }
                } else {
                    // Field not present
                    operator == "is_empty"
                }
            }
            _ => false,
        }
    }

    /// Check if a pair should be excluded.
    fn is_pair_excluded(&self, a: &NormalizedEntity, b: &NormalizedEntity) -> bool {
        for excl in &self.exclusions {
            match excl {
                cannon_common::ir::CompiledExclusion::FieldValue { .. } => {
                    if Self::matches_exclusion(a, excl) || Self::matches_exclusion(b, excl) {
                        return true;
                    }
                }
                cannon_common::ir::CompiledExclusion::ExplicitPair { left_id, right_id } => {
                    let a_ext = &a.external_id;
                    let b_ext = &b.external_id;
                    if (a_ext == left_id && b_ext == right_id) || (a_ext == right_id && b_ext == left_id) {
                        return true;
                    }
                }
                cannon_common::ir::CompiledExclusion::CrossTenant { enabled } => {
                    if *enabled && a.tenant_id != b.tenant_id {
                        return true;
                    }
                }
            }
        }
        false
    }

    pub fn reconcile(&self, entities: &[NormalizedEntity], override_resolver: &OverrideResolver) -> Vec<MatchDecision> {
        let entity_map: HashMap<Uuid, &NormalizedEntity> = entities.iter()
            .map(|e| (e.id, e))
            .collect();

        // Use BTreeMap for deterministic blocking group iteration order
        let mut groups: std::collections::BTreeMap<String, Vec<Uuid>> = std::collections::BTreeMap::new();
        for entity in entities {
            // Pre-filter: skip entities matching FieldValue exclusions
            if self.exclusions.iter().any(|e| Self::matches_exclusion(entity, e)) {
                continue;
            }
            for strategy in &self.blocking_strategies {
                let keys = strategy.generate_block_key(entity);
                for key in keys {
                    groups.entry(key).or_default().push(entity.id);
                }
            }
        }

        // Sort entity IDs within each blocking group for deterministic pair ordering
        for ids in groups.values_mut() {
            ids.sort();
        }

        // Collect sorted keys for deterministic parallel iteration
        let sorted_groups: Vec<(String, Vec<Uuid>)> = groups.into_iter().collect();

        let raw_decisions: Vec<MatchDecision> = sorted_groups.par_iter()
            .filter(|(_, ids)| ids.len() >= 2 && ids.len() <= self.max_block_size)
            .flat_map(|(_key, ids)| {
                let mut block_decisions = Vec::new();
                for i in 0..ids.len() {
                    for j in i + 1..ids.len() {
                        let id_a = ids[i];
                        let id_b = ids[j];

                        let entity_a = entity_map.get(&id_a).expect("Entity not found in map");
                        let entity_b = entity_map.get(&id_b).expect("Entity not found in map");

                        if let Some(mut decision) = self.reconcile_pair(entity_a, entity_b, override_resolver) {
                            decision.blocking_key = Some(_key.clone());
                            block_decisions.push(decision);
                        }
                    }
                }
                block_decisions
            })
            .collect();

        // Dedup using BTreeSet for deterministic insertion order
        let mut final_decisions = Vec::new();
        let mut seen_pairs: std::collections::BTreeSet<(Uuid, Uuid)> = std::collections::BTreeSet::new();
        for d in raw_decisions {
            let pair = if d.entity_a_id < d.entity_b_id { (d.entity_a_id, d.entity_b_id) } else { (d.entity_b_id, d.entity_a_id) };
            if seen_pairs.insert(pair) {
                final_decisions.push(d);
            }
        }

        if let Some(window) = self.neighborhood_window {
            if !self.sort_fields.is_empty() {
                let mut sorted_entities: Vec<_> = entities.iter().collect();
                sorted_entities.sort_by_key(|e| {
                    let mut key = String::new();
                    for field in &self.sort_fields {
                        let val = e.data.get(field).map(|v| v.to_lowercase()).unwrap_or_default();
                        key.push_str(&val);
                        key.push(':');
                    }
                    key
                });

                for i in 0..sorted_entities.len() {
                    for j in i + 1..std::cmp::min(i + window, sorted_entities.len()) {
                        let entity_a = sorted_entities[i];
                        let entity_b = sorted_entities[j];

                        let pair = if entity_a.id < entity_b.id { (entity_a.id, entity_b.id) } else { (entity_b.id, entity_a.id) };
                        if seen_pairs.insert(pair) {
                            if let Some(mut decision) = self.reconcile_pair(entity_a, entity_b, override_resolver) {
                                decision.blocking_key = Some("SORTED_NEIGHBORHOOD".to_string());
                                final_decisions.push(decision);
                            }
                        }
                    }
                }
            }
        }

        // Deterministic sort: by confidence desc, then tie-breakers, then entity IDs
        self.sort_decisions_deterministic(&mut final_decisions, &entity_map);

        final_decisions
    }

    pub fn reconcile_with_telemetry(&self, entities: &[NormalizedEntity], override_resolver: &OverrideResolver) -> (Vec<MatchDecision>, ReconciliationTelemetry) {
        let entity_map: HashMap<Uuid, &NormalizedEntity> = entities.iter()
            .map(|e| (e.id, e))
            .collect();

        // Use BTreeMap for deterministic blocking group iteration order
        let mut groups: std::collections::BTreeMap<String, Vec<Uuid>> = std::collections::BTreeMap::new();
        for entity in entities {
            if self.exclusions.iter().any(|e| Self::matches_exclusion(entity, e)) {
                continue;
            }
            for strategy in &self.blocking_strategies {
                let keys = strategy.generate_block_key(entity);
                for key in keys {
                    groups.entry(key).or_default().push(entity.id);
                }
            }
        }

        // Sort entity IDs within each blocking group for deterministic pair ordering
        for ids in groups.values_mut() {
            ids.sort();
        }

        let collector = Mutex::new(TelemetryCollector::new());
        collector.lock().unwrap().blocking_groups = groups.len();

        // Collect sorted keys for deterministic parallel iteration
        let sorted_groups: Vec<(String, Vec<Uuid>)> = groups.into_iter().collect();

        let raw_decisions: Vec<MatchDecision> = sorted_groups.par_iter()
            .filter(|(_, ids)| ids.len() >= 2 && ids.len() <= self.max_block_size)
            .flat_map(|(_key, ids)| {
                let mut block_decisions = Vec::new();
                for i in 0..ids.len() {
                    for j in i + 1..ids.len() {
                        let id_a = ids[i];
                        let id_b = ids[j];

                        let entity_a = entity_map.get(&id_a).expect("Entity not found in map");
                        let entity_b = entity_map.get(&id_b).expect("Entity not found in map");

                        let (maybe_decision, maybe_telemetry) = self.reconcile_pair_with_telemetry(entity_a, entity_b, override_resolver);

                        if let Some((rule_results, decision_type)) = maybe_telemetry {
                            if let Ok(mut c) = collector.lock() {
                                c.record_pair(&rule_results, &decision_type);
                            }
                        }

                        if let Some(mut decision) = maybe_decision {
                            decision.blocking_key = Some(_key.clone());
                            block_decisions.push(decision);
                        }
                    }
                }
                block_decisions
            })
            .collect();

        let mut final_decisions = Vec::new();
        let mut seen_pairs: std::collections::BTreeSet<(Uuid, Uuid)> = std::collections::BTreeSet::new();
        for d in raw_decisions {
            let pair = if d.entity_a_id < d.entity_b_id { (d.entity_a_id, d.entity_b_id) } else { (d.entity_b_id, d.entity_a_id) };
            if seen_pairs.insert(pair) {
                final_decisions.push(d);
            }
        }

        if let Some(window) = self.neighborhood_window {
            if !self.sort_fields.is_empty() {
                let mut sorted_entities: Vec<_> = entities.iter().collect();
                sorted_entities.sort_by_key(|e| {
                    let mut key = String::new();
                    for field in &self.sort_fields {
                        let val = e.data.get(field).map(|v| v.to_lowercase()).unwrap_or_default();
                        key.push_str(&val);
                        key.push(':');
                    }
                    key
                });

                for i in 0..sorted_entities.len() {
                    for j in i + 1..std::cmp::min(i + window, sorted_entities.len()) {
                        let entity_a = sorted_entities[i];
                        let entity_b = sorted_entities[j];

                        let pair = if entity_a.id < entity_b.id { (entity_a.id, entity_b.id) } else { (entity_b.id, entity_a.id) };
                        if seen_pairs.insert(pair) {
                            let (maybe_decision, maybe_telemetry) = self.reconcile_pair_with_telemetry(entity_a, entity_b, override_resolver);

                            if let Some((rule_results, decision_type)) = maybe_telemetry {
                                if let Ok(mut c) = collector.lock() {
                                    c.record_pair(&rule_results, &decision_type);
                                }
                            }

                            if let Some(mut decision) = maybe_decision {
                                decision.blocking_key = Some("SORTED_NEIGHBORHOOD".to_string());
                                final_decisions.push(decision);
                            }
                        }
                    }
                }
            }
        }

        // Deterministic sort: by confidence desc, then tie-breakers, then entity IDs
        self.sort_decisions_deterministic(&mut final_decisions, &entity_map);

        let telemetry = collector.into_inner().unwrap().into_telemetry();
        (final_decisions, telemetry)
    }

    fn reconcile_pair_with_telemetry(&self, a: &NormalizedEntity, b: &NormalizedEntity, resolver: &OverrideResolver) -> (Option<MatchDecision>, Option<(Vec<types::RuleResult>, String)>) {
        let (temporal_res, temporal_adj) = TemporalMatcher::evaluate(
            &self.temporal_strategy,
            a,
            b,
            None
        );

        if temporal_res == TemporalResult::Historical && temporal_adj == 0.0 {
            return (None, None);
        }

        let decision = self.evaluate_pair(a, b, resolver, temporal_adj);
        let decision_type = format!("{:?}", decision.decision).to_lowercase();
        let rule_results = decision.rule_results.clone();
        let telemetry = Some((rule_results, decision_type));

        if decision.decision != Decision::NoMerge {
            (Some(decision), telemetry)
        } else {
            (None, telemetry)
        }
    }

    fn reconcile_pair(&self, a: &NormalizedEntity, b: &NormalizedEntity, resolver: &OverrideResolver) -> Option<MatchDecision> {
        let (temporal_res, temporal_adj) = TemporalMatcher::evaluate(
            &self.temporal_strategy,
            a,
            b,
            None
        );

        if temporal_res == TemporalResult::Historical && temporal_adj == 0.0 {
            return None;
        }

        let decision = self.evaluate_pair(a, b, resolver, temporal_adj);
        if decision.decision != Decision::NoMerge {
            Some(decision)
        } else {
            None
        }
    }

    pub fn cluster_decisions(&self, entities: &[NormalizedEntity], decisions: &[MatchDecision]) -> Vec<Vec<Uuid>> {
        use petgraph::unionfind::UnionFind;

        let id_to_idx: HashMap<Uuid, usize> = entities.iter().enumerate().map(|(i, e)| (e.id, i)).collect();
        let idx_to_id: Vec<Uuid> = entities.iter().map(|e| e.id).collect();

        let mut uf = UnionFind::new(entities.len());

        for decision in decisions {
            if decision.decision == Decision::Merge {
                if let (Some(&idx_a), Some(&idx_b)) = (id_to_idx.get(&decision.entity_a_id), id_to_idx.get(&decision.entity_b_id)) {
                    uf.union(idx_a, idx_b);
                }
            }
        }

        let mut groups: HashMap<usize, Vec<Uuid>> = HashMap::new();
        for i in 0..entities.len() {
            let root = uf.find(i);
            groups.entry(root).or_default().push(idx_to_id[i]);
        }

        let mut clusters: Vec<Vec<Uuid>> = groups.into_values().collect();
        // Sort clusters deterministically: by smallest member ID
        for cluster in &mut clusters {
            cluster.sort();
        }
        clusters.sort_by(|a, b| a[0].cmp(&b[0]));
        clusters
    }

    /// Cluster with per-cluster aggregate confidence metrics.
    pub fn cluster_decisions_with_confidence(
        &self,
        entities: &[NormalizedEntity],
        decisions: &[MatchDecision],
    ) -> Vec<ClusterWithConfidence> {
        let clusters = self.cluster_decisions(entities, decisions);

        // Build a lookup: (entity_a, entity_b) -> MatchDecision
        let mut pair_decisions: std::collections::BTreeMap<(Uuid, Uuid), &MatchDecision> = std::collections::BTreeMap::new();
        for d in decisions {
            let pair = if d.entity_a_id < d.entity_b_id {
                (d.entity_a_id, d.entity_b_id)
            } else {
                (d.entity_b_id, d.entity_a_id)
            };
            pair_decisions.insert(pair, d);
        }

        clusters.into_iter().map(|members| {
            // Collect all decision confidences within this cluster
            let mut confidences = Vec::new();
            for i in 0..members.len() {
                for j in i + 1..members.len() {
                    let pair = if members[i] < members[j] {
                        (members[i], members[j])
                    } else {
                        (members[j], members[i])
                    };
                    if let Some(d) = pair_decisions.get(&pair) {
                        confidences.push(d.confidence);
                    }
                }
            }

            let (avg_confidence, min_confidence, max_confidence) = if confidences.is_empty() {
                (0.0, 0.0, 0.0)
            } else {
                let sum: f64 = confidences.iter().sum();
                let min = confidences.iter().cloned().fold(f64::INFINITY, f64::min);
                let max = confidences.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
                (sum / confidences.len() as f64, min, max)
            };

            ClusterWithConfidence {
                members,
                avg_confidence,
                min_confidence,
                max_confidence,
                pair_count: confidences.len(),
            }
        }).collect()
    }

    /// Sort decisions deterministically using configured tie-breaking rules.
    fn sort_decisions_deterministic(&self, decisions: &mut [MatchDecision], entity_map: &HashMap<Uuid, &NormalizedEntity>) {
        let tie_breakers = &self.tie_breakers;
        decisions.sort_by(|a, b| {
            // Primary: confidence descending
            let conf_cmp = b.confidence.partial_cmp(&a.confidence).unwrap_or(std::cmp::Ordering::Equal);
            if conf_cmp != std::cmp::Ordering::Equal {
                return conf_cmp;
            }

            // Apply tie-breakers in order
            for tb in tie_breakers {
                let result = match tb {
                    cannon_common::ir::CompiledTieBreaker::ExactMatchPriority => {
                        // Prefer pair with more exact matches (score == 1.0)
                        let a_exact = a.rule_results.iter().filter(|r| (r.score - 1.0).abs() < f64::EPSILON).count();
                        let b_exact = b.rule_results.iter().filter(|r| (r.score - 1.0).abs() < f64::EPSILON).count();
                        b_exact.cmp(&a_exact) // more exact matches first
                    }
                    cannon_common::ir::CompiledTieBreaker::LowestEntityId => {
                        // Compare by smaller entity ID of the pair
                        let a_min = a.entity_a_id.min(a.entity_b_id);
                        let b_min = b.entity_a_id.min(b.entity_b_id);
                        a_min.cmp(&b_min)
                    }
                    cannon_common::ir::CompiledTieBreaker::EarliestTimestamp => {
                        // Compare by earliest last_updated of the pair's entities
                        let a_ts = [
                            entity_map.get(&a.entity_a_id).map(|e| e.last_updated),
                            entity_map.get(&a.entity_b_id).map(|e| e.last_updated),
                        ].iter().filter_map(|t| *t).min();
                        let b_ts = [
                            entity_map.get(&b.entity_a_id).map(|e| e.last_updated),
                            entity_map.get(&b.entity_b_id).map(|e| e.last_updated),
                        ].iter().filter_map(|t| *t).min();
                        a_ts.cmp(&b_ts)
                    }
                };
                if result != std::cmp::Ordering::Equal {
                    return result;
                }
            }

            // Ultimate fallback: lexicographic on entity ID pair
            let a_pair = (a.entity_a_id.min(a.entity_b_id), a.entity_a_id.max(a.entity_b_id));
            let b_pair = (b.entity_a_id.min(b.entity_b_id), b.entity_a_id.max(b.entity_b_id));
            a_pair.cmp(&b_pair)
        });
    }

    pub fn evaluate_pair(&self, a: &NormalizedEntity, b: &NormalizedEntity, resolver: &OverrideResolver, scale: f64) -> MatchDecision {
        // Check exclusions first
        if self.is_pair_excluded(a, b) {
            return MatchDecision {
                entity_a_id: a.id,
                entity_b_id: b.id,
                decision: Decision::NoMerge,
                confidence: 0.0,
                rule_results: vec![],
                matched_on: vec!["EXCLUDED".to_string()],
                posterior_probability: None,
                blocking_key: None,
                temporal_adjustment: Some(scale),
            };
        }

        if resolver.is_split(a.id, b.id) {
            return MatchDecision {
                entity_a_id: a.id,
                entity_b_id: b.id,
                decision: Decision::NoMerge,
                confidence: 0.0,
                rule_results: vec![],
                matched_on: vec!["MANUAL_SPLIT".to_string()],
                posterior_probability: None,
                blocking_key: None,
                temporal_adjustment: Some(scale),
            };
        }

        if resolver.is_locked(a.id, b.id) {
            return MatchDecision {
                entity_a_id: a.id,
                entity_b_id: b.id,
                decision: Decision::NoMerge,
                confidence: 0.0,
                rule_results: vec![],
                matched_on: vec!["MANUAL_LOCK".to_string()],
                posterior_probability: None,
                blocking_key: None,
                temporal_adjustment: Some(scale),
            };
        }

        if resolver.is_forced_merge(a.id, b.id) {
            return MatchDecision {
                entity_a_id: a.id,
                entity_b_id: b.id,
                decision: Decision::Merge,
                confidence: 1.0,
                rule_results: vec![],
                matched_on: vec!["MANUAL_MERGE".to_string()],
                posterior_probability: None,
                blocking_key: None,
                temporal_adjustment: Some(scale),
            };
        }

        for rid in &self.reference_identifiers {
            let val_a = a.data.get(&rid.field);
            let val_b = b.data.get(&rid.field);

            if let (Some(v_a), Some(v_b)) = (val_a, val_b) {
                if v_a == v_b && !v_a.is_empty() {
                    return MatchDecision {
                        entity_a_id: a.id,
                        entity_b_id: b.id,
                        decision: Decision::Merge,
                        confidence: 1.0,
                        posterior_probability: Some(1.0),
                        rule_results: vec![],
                        matched_on: vec![format!("REF_ID:{}", rid.name)],
                        blocking_key: None,
                        temporal_adjustment: Some(scale),
                    };
                }
            }
        }

        let mut rule_results = Vec::new();
        let mut total_score = 0.0;
        let mut total_weight = 0.0;
        let mut matched_on = Vec::new();

        for rule in &self.rules {
            let result = rule.evaluate(a, b);
            total_score += result.score * result.weight;
            total_weight += result.weight;
            if result.matched {
                matched_on.push(result.rule_name.clone());
            }
            rule_results.push(result);
        }

        let confidence = match self.scoring_method {
            cannon_common::ir::ScoringMethod::WeightedSum => {
                if total_weight > 0.0 {
                    (total_score / total_weight) * scale
                } else {
                    0.0
                }
            }
            cannon_common::ir::ScoringMethod::MlEnsemble => {
                if let Some(ref config) = self.ml_ensemble_config {
                    // Logistic regression: sigmoid(bias + sum(coeff_i * raw_score_i))
                    let mut logit = config.bias;
                    for result in &rule_results {
                        let coeff = config.coefficients.get(&result.rule_name).copied().unwrap_or(0.0);
                        logit += coeff * result.score;
                    }
                    let raw = 1.0 / (1.0 + (-logit).exp());
                    // Clamp to [0.0, 1.0] for safety (sigmoid already does this but guard against NaN)
                    let clamped = raw.clamp(0.0, 1.0);
                    clamped * scale
                } else {
                    tracing::warn!("ML ensemble scoring selected but no ml_ensemble config provided, falling back to weighted sum");
                    if total_weight > 0.0 {
                        (total_score / total_weight) * scale
                    } else {
                        0.0
                    }
                }
            }
            cannon_common::ir::ScoringMethod::Custom => {
                if let Some(ref ops) = self.custom_scoring_ops {
                    // Build variable map from rule results: rule_name -> raw score
                    let vars: hashbrown::HashMap<String, f64> = rule_results.iter()
                        .map(|r| (r.rule_name.clone(), r.score))
                        .collect();
                    // Convert to std HashMap for the expression evaluator
                    let std_vars: std::collections::HashMap<String, f64> = vars.into_iter().collect();
                    match expression::evaluate(ops, &std_vars) {
                        Ok(raw) => {
                            let clamped = raw.clamp(0.0, 1.0);
                            clamped * scale
                        }
                        Err(e) => {
                            tracing::error!("Custom scoring expression evaluation failed: {}, falling back to weighted sum", e);
                            if total_weight > 0.0 {
                                (total_score / total_weight) * scale
                            } else {
                                0.0
                            }
                        }
                    }
                } else {
                    tracing::warn!("Custom scoring selected but no expression configured, falling back to weighted sum");
                    if total_weight > 0.0 {
                        (total_score / total_weight) * scale
                    } else {
                        0.0
                    }
                }
            }
            cannon_common::ir::ScoringMethod::FellegiSunter => {
                if let Some(ref fs) = self.fellegi_sunter_config {
                    let mut composite = 0.0_f64;
                    for field in &fs.fields {
                        let score = rule_results.iter()
                            .find(|r| r.rule_name == field.name)
                            .map(|r| r.score)
                            .unwrap_or(0.0);
                        // Interpolate between disagree and agree weights based on similarity score
                        let field_contribution = field.w_disagree + score * (field.w_agree - field.w_disagree);
                        composite += field_contribution;
                    }
                    composite // raw log-likelihood, NOT normalized to [0,1]
                } else {
                    tracing::warn!("Fellegi-Sunter scoring selected but no config provided, falling back to weighted sum");
                    if total_weight > 0.0 {
                        (total_score / total_weight) * scale
                    } else {
                        0.0
                    }
                }
            }
        };

        // Enrich rule explanations with FS probability details if logging enabled (A9: telemetry)
        if self.log_probabilities {
            if let Some(ref fs) = self.fellegi_sunter_config {
                for field in &fs.fields {
                    let score = rule_results.iter()
                        .find(|r| r.rule_name == field.name)
                        .map(|r| r.score)
                        .unwrap_or(0.0);
                    let field_contribution = field.w_disagree + score * (field.w_agree - field.w_disagree);
                    if let Some(r) = rule_results.iter_mut().find(|r| r.rule_name == field.name) {
                        r.explanation = format!(
                            "{}: score={:.2}, w={:.2} (agree={:.2}, disagree={:.2})",
                            field.name, score, field_contribution, field.w_agree, field.w_disagree
                        );
                    }
                }
            }
        }

        // Decision logic: FS uses its own thresholds, other methods use normalized thresholds
        let decision = if self.scoring_method == cannon_common::ir::ScoringMethod::FellegiSunter {
            if let Some(ref fs) = self.fellegi_sunter_config {
                if confidence >= fs.match_threshold {
                    Decision::Merge
                } else if confidence >= fs.possible_threshold {
                    Decision::Review
                } else {
                    Decision::NoMerge
                }
            } else {
                if confidence >= self.merge_threshold {
                    Decision::Merge
                } else if confidence >= self.review_threshold {
                    Decision::Review
                } else {
                    Decision::NoMerge
                }
            }
        } else if confidence >= self.merge_threshold {
            Decision::Merge
        } else if confidence >= self.review_threshold {
            Decision::Review
        } else {
            Decision::NoMerge
        };

        // Compute posterior probability P(match|data) for FS scoring
        let posterior_probability = if self.scoring_method == cannon_common::ir::ScoringMethod::FellegiSunter {
            if let Some(ref _fs) = self.fellegi_sunter_config {
                // P(match|data) = sigmoid(log-likelihood) when using uniform prior
                // This maps the raw log-likelihood to [0, 1] probability
                let odds = confidence.exp(); // e^(log-likelihood ratio)
                let posterior = odds / (1.0 + odds);
                Some(posterior.clamp(0.0, 1.0))
            } else {
                None
            }
        } else {
            None
        };

        MatchDecision {
            entity_a_id: a.id,
            entity_b_id: b.id,
            decision,
            confidence,
            posterior_probability,
            rule_results,
            matched_on,
            blocking_key: None,
            temporal_adjustment: Some(scale),
        }
    }

    pub fn generate_block_keys(&self, entity: &NormalizedEntity) -> Vec<String> {
        let mut keys = Vec::new();
        for strategy in &self.blocking_strategies {
            keys.extend(strategy.generate_block_key(entity));
        }
        keys
    }

    pub fn propagate_relations(
        &self,
        clusters: &[Vec<Uuid>],
        entities: &[NormalizedEntity],
        resolver: &OverrideResolver,
        initial_decisions: &[MatchDecision],
    ) -> Vec<MatchDecision> {
        let mut all_propagated_decisions = Vec::new();
        let mut current_clusters = clusters.to_vec();

        let mut entity_map: HashMap<Uuid, &NormalizedEntity> = HashMap::new();
        // Index entities by (type, value) for lookup by any attribute value
        let mut entities_by_type_val: HashMap<(String, String), Vec<Uuid>> = HashMap::new();

        for e in entities {
            entity_map.insert(e.id, e);
            for (_attr, val) in &e.data {
                entities_by_type_val.entry((e.entity_type.clone(), val.clone())).or_default().push(e.id);
            }
            entities_by_type_val.entry((e.entity_type.clone(), e.id.to_string())).or_default().push(e.id);
            entities_by_type_val.entry((e.entity_type.clone(), e.external_id.clone())).or_default().push(e.id);
        }

        let max_hops = 5;
        let mut seen_pairs = HashSet::new();

        for hop in 0..max_hops {
            let mut new_decisions_this_hop = Vec::new();

            for relation in &self.relations {
                if !relation.propagate_match {
                    continue;
                }

                for cluster in &current_clusters {
                    if cluster.len() < 2 {
                        continue;
                    }

                    // Collect target values from join key for all from_entity entities in this cluster
                    let mut target_vals = HashSet::new();
                    for &id in cluster {
                        if let Some(entity) = entity_map.get(&id) {
                            if entity.entity_type == relation.from_entity {
                                if let Some(target_val) = entity.data.get(&relation.join_key) {
                                    target_vals.insert(target_val.clone());
                                }
                            }
                        }
                    }

                    // First try to find matching entities of `to_entity` type
                    let mut target_ids = HashSet::new();
                    for target_val in &target_vals {
                        if let Some(found_ids) = entities_by_type_val.get(&(relation.to_entity.clone(), target_val.clone())) {
                            for &fid in found_ids {
                                target_ids.insert(fid);
                            }
                        }
                    }

                    if target_ids.len() >= 2 {
                        // Found multiple target entities to link
                        let mut sorted_targets: Vec<_> = target_ids.into_iter().collect();
                        sorted_targets.sort();
                        for i in 0..sorted_targets.len() {
                            for j in i + 1..sorted_targets.len() {
                                let id_a = sorted_targets[i];
                                let id_b = sorted_targets[j];

                                if seen_pairs.contains(&(id_a, id_b)) {
                                    continue;
                                }

                                if resolver.is_split(id_a, id_b) || resolver.is_locked(id_a, id_b) {
                                    continue;
                                }

                                seen_pairs.insert((id_a, id_b));
                                new_decisions_this_hop.push(MatchDecision {
                                    entity_a_id: id_a,
                                    entity_b_id: id_b,
                                    decision: Decision::Merge,
                                    confidence: 1.0,
                                    posterior_probability: Some(1.0),
                                    rule_results: vec![],
                                    matched_on: vec![format!("RELATION:{}:hop:{}", relation.name, hop)],
                                    blocking_key: None,
                                    temporal_adjustment: None,
                                });
                            }
                        }
                    } else if target_vals.len() >= 2 && target_ids.is_empty() {
                        // No actual target entities found, but we have distinct join key values
                        // from matched source entities. Link the join key values as virtual IDs.
                        let mut sorted_vals: Vec<_> = target_vals.into_iter().collect();
                        sorted_vals.sort();
                        for i in 0..sorted_vals.len() {
                            for j in i + 1..sorted_vals.len() {
                                // Try to parse join key values as UUIDs for the decision
                                let uuid_a = sorted_vals[i].parse::<Uuid>();
                                let uuid_b = sorted_vals[j].parse::<Uuid>();

                                if let (Ok(id_a), Ok(id_b)) = (uuid_a, uuid_b) {
                                    let (id_a, id_b) = if id_a < id_b { (id_a, id_b) } else { (id_b, id_a) };

                                    if seen_pairs.contains(&(id_a, id_b)) {
                                        continue;
                                    }

                                    if resolver.is_split(id_a, id_b) || resolver.is_locked(id_a, id_b) {
                                        continue;
                                    }

                                    seen_pairs.insert((id_a, id_b));
                                    new_decisions_this_hop.push(MatchDecision {
                                        entity_a_id: id_a,
                                        entity_b_id: id_b,
                                        decision: Decision::Merge,
                                        confidence: 1.0,
                                        posterior_probability: Some(1.0),
                                        rule_results: vec![],
                                        matched_on: vec![format!("RELATION:{}:hop:{}", relation.name, hop)],
                                        blocking_key: None,
                                        temporal_adjustment: None,
                                    });
                                }
                            }
                        }
                    }
                }
            }

            if new_decisions_this_hop.is_empty() {
                break;
            }

            all_propagated_decisions.extend(new_decisions_this_hop);

            let mut all_current_decisions = initial_decisions.to_vec();
            all_current_decisions.extend(all_propagated_decisions.clone());

            current_clusters = self.cluster_decisions(entities, &all_current_decisions);
        }

        all_propagated_decisions
    }
}
