//! Field profiler: discovers identity signals from data and bootstraps an IdentityPlan.
//!
//! This is the Rust equivalent of the Python SDK's `_profiler.py`. It scans
//! `external_entities.raw_data` keys, detects identity signals (email, phone,
//! name, company), computes uniqueness/null rates, and builds a compiled
//! `IdentityPlan` ready for the `ReconciliationEngine`.

use crate::ir::{
    BlockingKey, BlockingStrategy, BlockingTransformation, CompiledAudit, CompiledBlocking,
    CompiledClusteringConfig, CompiledEmTraining, CompiledFsField,
    CompiledFellegiSunterPlan, CompiledRule, CompiledRuleType, ConflictStrategy, DecisionConfig,
    FsComparatorType, IdentityPlan, MatchGraph, PlanMetadata, ResolvedSource, ScoringMethod,
    SimilarityAlgorithm, SurvivorshipPolicy, SurvivorshipStrategy,
};
use regex::Regex;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::sync::OnceLock;
use uuid::Uuid;

// ---------------------------------------------------------------------------
// Signal patterns (port of Python _SIGNAL_PATTERNS, lines 26-47)
// ---------------------------------------------------------------------------

/// Canonical identity signal types detected from column names or values.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum IdentitySignal {
    // Person signals
    Email,
    Phone,
    FirstName,
    LastName,
    FullName,
    CompanyName,
    // Company signals
    Domain,
    TaxId,
    Industry,
    // Product signals
    Sku,
    Upc,
    ProductName,
    Brand,
    Category,
    // Transaction signals
    TransactionId,
    Amount,
    Currency,
    // Healthcare signals
    Mrn,
    Npi,
    PatientName,
    DateOfBirth,
}

impl IdentitySignal {
    /// Canonical field name used in the IdentityPlan.
    pub fn canonical_name(&self) -> &'static str {
        match self {
            Self::Email => "email",
            Self::Phone => "phone",
            Self::FirstName => "first_name",
            Self::LastName => "last_name",
            Self::FullName => "full_name",
            Self::CompanyName => "company_name",
            Self::Domain => "domain",
            Self::TaxId => "tax_id",
            Self::Industry => "industry",
            Self::Sku => "sku",
            Self::Upc => "upc",
            Self::ProductName => "product_name",
            Self::Brand => "brand",
            Self::Category => "category",
            Self::TransactionId => "transaction_id",
            Self::Amount => "amount",
            Self::Currency => "currency",
            Self::Mrn => "mrn",
            Self::Npi => "npi",
            Self::PatientName => "patient_name",
            Self::DateOfBirth => "date_of_birth",
        }
    }
}

/// Entity type inferred from data signals. This is a profiler concept - the engine
/// just executes whatever rules/fields are in the `IdentityPlan`. The profiler uses
/// `EntityType` to select which signals, defaults, and blocking keys to generate.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EntityType {
    Person,
    Company,
    Product,
    Transaction,
    Healthcare,
    Custom(String),
}

impl EntityType {
    /// Entity name for use in `IdentityPlan.entity`.
    pub fn to_entity_name(&self) -> &str {
        match self {
            Self::Person => "person",
            Self::Company => "company",
            Self::Product => "product",
            Self::Transaction => "transaction",
            Self::Healthcare => "healthcare",
            Self::Custom(name) => name.as_str(),
        }
    }

    /// Parse from a string. Unknown strings become `Custom`.
    pub fn from_str_loose(s: &str) -> Self {
        match s.to_lowercase().as_str() {
            "person" | "sf_person" | "contact" | "lead" => Self::Person,
            "company" | "organization" | "org" | "account" => Self::Company,
            "product" | "item" | "sku" => Self::Product,
            "transaction" | "order" | "payment" | "invoice" => Self::Transaction,
            "healthcare" | "patient" | "provider" | "medical" => Self::Healthcare,
            _ => Self::Custom(s.to_string()),
        }
    }

    /// Display name for CLI output and API responses.
    pub fn display_name(&self) -> &str {
        match self {
            Self::Person => "person",
            Self::Company => "company",
            Self::Product => "product",
            Self::Transaction => "transaction",
            Self::Healthcare => "healthcare",
            Self::Custom(name) => name.as_str(),
        }
    }
}

struct SignalPattern {
    signal: IdentitySignal,
    pattern: Regex,
}

fn signal_patterns() -> &'static Vec<SignalPattern> {
    static PATTERNS: OnceLock<Vec<SignalPattern>> = OnceLock::new();
    PATTERNS.get_or_init(|| {
        vec![
            // --- Person signals ---
            SignalPattern {
                signal: IdentitySignal::Email,
                pattern: Regex::new(
                    r"(?i)^(e_?mail|email_address|user_email|contact_email)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Phone,
                pattern: Regex::new(
                    r"(?i)^(phone|phone_number|telephone|mobile|cell|tel)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::FirstName,
                pattern: Regex::new(r"(?i)^(first_name|given_name|fname|first)$").unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::LastName,
                pattern: Regex::new(r"(?i)^(last_name|family_name|surname|lname|last)$").unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::FullName,
                pattern: Regex::new(
                    r"(?i)^(full_name|display_name|contact_name|customer_name|name)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::CompanyName,
                pattern: Regex::new(
                    r"(?i)^(company|company_name|account_name|org|organization|employer)$",
                )
                .unwrap(),
            },
            // --- Company signals ---
            SignalPattern {
                signal: IdentitySignal::Domain,
                pattern: Regex::new(
                    r"(?i)^(domain|website_domain|company_domain|web_domain)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::TaxId,
                pattern: Regex::new(
                    r"(?i)^(tax_id|ein|vat_number|tax_number|vat_id|tax_identifier)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Industry,
                pattern: Regex::new(
                    r"(?i)^(industry|sector|business_type|vertical)$",
                )
                .unwrap(),
            },
            // --- Product signals ---
            SignalPattern {
                signal: IdentitySignal::Sku,
                pattern: Regex::new(
                    r"(?i)^(sku|item_code|product_code|item_number|stock_keeping_unit)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Upc,
                pattern: Regex::new(
                    r"(?i)^(upc|ean|barcode|gtin|upc_code|ean_code)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::ProductName,
                pattern: Regex::new(
                    r"(?i)^(product_name|item_name|product_title|product_description|product)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Brand,
                pattern: Regex::new(
                    r"(?i)^(brand|brand_name|manufacturer|vendor|maker)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Category,
                pattern: Regex::new(
                    r"(?i)^(category|product_category|department|product_type|item_category)$",
                )
                .unwrap(),
            },
            // --- Transaction signals ---
            SignalPattern {
                signal: IdentitySignal::TransactionId,
                pattern: Regex::new(
                    r"(?i)^(transaction_id|txn_id|order_id|payment_id|invoice_id|order_number)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Amount,
                pattern: Regex::new(
                    r"(?i)^(amount|total|price|subtotal|grand_total|order_total|payment_amount)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Currency,
                pattern: Regex::new(
                    r"(?i)^(currency|currency_code|currency_type)$",
                )
                .unwrap(),
            },
            // --- Healthcare signals ---
            SignalPattern {
                signal: IdentitySignal::Mrn,
                pattern: Regex::new(
                    r"(?i)^(mrn|medical_record_number|medical_record|chart_number)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::Npi,
                pattern: Regex::new(
                    r"(?i)^(npi|provider_id|physician_id|provider_number)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::PatientName,
                pattern: Regex::new(
                    r"(?i)^(patient_name|patient_full_name|patient)$",
                )
                .unwrap(),
            },
            SignalPattern {
                signal: IdentitySignal::DateOfBirth,
                pattern: Regex::new(
                    r"(?i)^(date_of_birth|dob|birth_date|birthdate|birthday)$",
                )
                .unwrap(),
            },
        ]
    })
}

fn email_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^[^@\s]+@[^@\s]+\.[a-zA-Z]{2,}$").unwrap())
}

fn phone_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^[\d\s\-\+\(\)\.]{7,}$").unwrap())
}

fn timestamp_col_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| {
        Regex::new(r"(?i)(timestamp|created_at|updated_at|event_time|occurred_at|_at$|_time$|_date$)")
            .unwrap()
    })
}

fn fk_col_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"(?i)^.+_(id|key|fk)$").unwrap())
}

/// Matches values that contain corporate/organization suffixes.
fn company_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| {
        Regex::new(
            r"(?i)\b(inc\.?|incorporated|corp\.?|corporation|ltd\.?|limited|llc|llp|gmbh|ag|sa|pty|plc|co\.|group|holdings|partners|associates|enterprises|solutions|services|technologies|consulting|foundation|university|institute|bank)\b"
        ).unwrap()
    })
}

/// Matches SKU-like values: short alphanumeric codes with optional dash/underscore.
fn sku_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^[A-Za-z]{1,5}[-_]?\d{3,}[A-Za-z0-9]*$").unwrap())
}

/// Matches UPC/EAN barcodes: 12-13 digit numbers.
fn upc_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^\d{12,13}$").unwrap())
}

/// Matches monetary amounts. Requires either a currency symbol prefix OR a decimal with cents.
/// Plain integers like "42" or "2024" do NOT match (avoids false positives on age/quantity/year columns).
fn amount_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| {
        Regex::new(r"^([\$\u{00a3}\u{20ac}]\d[\d,]*\.?\d{0,2}|\d[\d,]*\.\d{2})$").unwrap()
    })
}

/// Matches ISO 4217 currency codes (3 uppercase letters like USD, EUR, GBP).
fn currency_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^[A-Z]{3}$").unwrap())
}

/// Matches US EIN tax IDs: XX-XXXXXXX format.
fn tax_id_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^\d{2}-\d{7}$").unwrap())
}

/// Matches bare domain names (not emails): alphanumeric-dot-tld.
fn domain_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*\.[a-z]{2,}$").unwrap())
}

/// Matches NPI numbers: exactly 10 digits.
fn npi_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^\d{10}$").unwrap())
}

/// Matches MRN (Medical Record Numbers): requires MRN/MR prefix or dash separator
/// to avoid false-positives on plain 6-10 digit numbers (zip codes, IDs, etc.).
fn mrn_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"(?i)^(MRN|MR)[-# ]?\d{6,10}$").unwrap())
}

/// Matches date-of-birth-like values: YYYY-MM-DD or MM/DD/YYYY with year in 1920-2020.
fn dob_value_re() -> &'static Regex {
    static RE: OnceLock<Regex> = OnceLock::new();
    RE.get_or_init(|| Regex::new(r"^(19[2-9]\d|20[01]\d|2020)[-/](0[1-9]|1[0-2])[-/](0[1-9]|[12]\d|3[01])$|^(0[1-9]|1[0-2])/(0[1-9]|[12]\d|3[01])/(19[2-9]\d|20[01]\d|2020)$").unwrap())
}

// ---------------------------------------------------------------------------
// Signal detection
// ---------------------------------------------------------------------------

/// Detect an identity signal from a column name alone.
pub fn detect_signal(col_name: &str) -> Option<IdentitySignal> {
    for sp in signal_patterns() {
        if sp.pattern.is_match(col_name) {
            return Some(sp.signal.clone());
        }
    }
    None
}

/// Detect an identity signal by inspecting sampled values. Values are the
/// ground truth - column names are unreliable hints. Returns `Some` if a
/// sufficient fraction of values match a signal pattern.
pub fn detect_signal_from_values(values: &[&str]) -> Option<IdentitySignal> {
    if values.is_empty() {
        return None;
    }
    let n = values.len() as f64;

    // Email: > 50% look like email addresses
    let email_hits = values
        .iter()
        .filter(|v| email_value_re().is_match(v))
        .count();
    if (email_hits as f64) / n > 0.5 {
        return Some(IdentitySignal::Email);
    }

    // --- Specific numeric patterns BEFORE phone (phone regex is broad) ---

    // UPC/EAN: > 30% are 12-13 digit barcodes
    let upc_hits = values.iter().filter(|v| upc_value_re().is_match(v)).count();
    if (upc_hits as f64) / n > 0.3 {
        return Some(IdentitySignal::Upc);
    }

    // NPI: > 30% are exactly 10-digit numbers
    let npi_hits = values.iter().filter(|v| npi_value_re().is_match(v)).count();
    if (npi_hits as f64) / n > 0.3 {
        return Some(IdentitySignal::Npi);
    }

    // Tax ID (EIN): > 30% match XX-XXXXXXX pattern
    let tax_hits = values.iter().filter(|v| tax_id_value_re().is_match(v)).count();
    if (tax_hits as f64) / n > 0.3 {
        return Some(IdentitySignal::TaxId);
    }

    // MRN: > 30% match MRN prefix + digits pattern
    let mrn_hits = values.iter().filter(|v| mrn_value_re().is_match(v)).count();
    if (mrn_hits as f64) / n > 0.3 {
        return Some(IdentitySignal::Mrn);
    }

    // Date of Birth: > 30% match date patterns in birth-year range
    let dob_hits = values.iter().filter(|v| dob_value_re().is_match(v)).count();
    if (dob_hits as f64) / n > 0.3 {
        return Some(IdentitySignal::DateOfBirth);
    }

    // Phone: > 50% look like phone numbers (checked AFTER specific digit patterns
    // because the phone regex is broad and matches UPCs, NPIs, tax IDs, dates, etc.)
    let phone_hits = values
        .iter()
        .filter(|v| phone_value_re().is_match(v))
        .count();
    if (phone_hits as f64) / n > 0.5 {
        return Some(IdentitySignal::Phone);
    }

    // --- Non-numeric patterns ---

    // SKU: > 30% match short alphanumeric code pattern
    let sku_hits = values.iter().filter(|v| sku_value_re().is_match(v)).count();
    if (sku_hits as f64) / n > 0.3 {
        return Some(IdentitySignal::Sku);
    }

    // Domain: > 40% match bare domain pattern (not emails)
    let domain_hits = values
        .iter()
        .filter(|v| domain_value_re().is_match(&v.to_lowercase()) && !v.contains('@'))
        .count();
    if (domain_hits as f64) / n > 0.4 {
        return Some(IdentitySignal::Domain);
    }

    // Currency codes: > 50% match 3-letter ISO codes
    let currency_hits = values.iter().filter(|v| currency_value_re().is_match(v)).count();
    if (currency_hits as f64) / n > 0.5 {
        return Some(IdentitySignal::Currency);
    }

    // Amount: > 50% match monetary amount patterns
    let amount_hits = values.iter().filter(|v| amount_value_re().is_match(v)).count();
    if (amount_hits as f64) / n > 0.5 {
        return Some(IdentitySignal::Amount);
    }

    // Company name: > 20% contain corporate suffixes (Ltd, Inc, Corp, LLC, etc.)
    // Lower threshold because not all companies have suffixes but a meaningful
    // fraction is a strong signal this column holds org names, not person names.
    let company_hits = values
        .iter()
        .filter(|v| company_value_re().is_match(v))
        .count();
    if (company_hits as f64) / n > 0.2 {
        return Some(IdentitySignal::CompanyName);
    }

    None
}

/// Returns true if the column name looks like a timestamp field.
pub fn is_timestamp_column(col_name: &str) -> bool {
    timestamp_col_re().is_match(col_name)
}

/// Returns true if the column name looks like a foreign key.
pub fn is_fk_column(col_name: &str) -> bool {
    fk_col_re().is_match(col_name)
}

// ---------------------------------------------------------------------------
// Field defaults (port of Python _FIELD_DEFAULTS, lines 479-522)
// ---------------------------------------------------------------------------

/// Default configuration for a detected identity signal.
#[derive(Debug, Clone)]
pub struct FieldDefault {
    pub comparator: FsComparatorType,
    pub weight: f64,
    pub m_probability: f64,
    pub u_probability: f64,
    pub normalizer: &'static str,
    pub rule_comparator: CompiledRuleType,
}

/// Per-entity signal catalog: defines which fields, defaults, blocking templates,
/// and field ordering to use when building an IdentityPlan for a given entity type.
pub struct SignalCatalog {
    pub entity_type: EntityType,
    pub field_defaults: HashMap<&'static str, FieldDefault>,
    pub blocking_templates: Vec<Vec<&'static str>>,
    pub field_order: Vec<&'static str>,
}

// Helper to build an exact-match FieldDefault.
fn exact_default(
    field: &'static str,
    weight: f64,
    m: f64,
    u: f64,
    normalizer: &'static str,
) -> FieldDefault {
    FieldDefault {
        comparator: FsComparatorType::Exact,
        weight,
        m_probability: m,
        u_probability: u,
        normalizer,
        rule_comparator: CompiledRuleType::Exact {
            field: field.into(),
            weight,
            case_insensitive: true,
            normalize: true,
            normalizer: Some(normalizer.into()),
        },
    }
}

// Helper to build a similarity-match FieldDefault.
fn similarity_default(
    field: &'static str,
    weight: f64,
    m: f64,
    u: f64,
    threshold: f64,
    normalizer: &'static str,
) -> FieldDefault {
    FieldDefault {
        comparator: FsComparatorType::JaroWinkler,
        weight,
        m_probability: m,
        u_probability: u,
        normalizer,
        rule_comparator: CompiledRuleType::Similarity {
            field: field.into(),
            algorithm: SimilarityAlgorithm::JaroWinkler,
            threshold,
            weight,
            normalizer: Some(normalizer.into()),
        },
    }
}

fn person_catalog() -> SignalCatalog {
    let mut m = HashMap::new();
    m.insert("email", exact_default("email", 2.0, 0.95, 0.001, "email"));
    m.insert("phone", exact_default("phone", 2.0, 0.95, 0.001, "phone"));
    m.insert("first_name", similarity_default("first_name", 1.0, 0.85, 0.05, 0.85, "nickname"));
    m.insert("last_name", similarity_default("last_name", 1.0, 0.88, 0.02, 0.85, "name"));
    m.insert("full_name", similarity_default("full_name", 1.0, 0.80, 0.03, 0.80, "name"));
    m.insert("company_name", similarity_default("company_name", 1.0, 0.80, 0.02, 0.80, "generic"));
    SignalCatalog {
        entity_type: EntityType::Person,
        field_defaults: m,
        blocking_templates: vec![
            vec!["email"],
            vec!["phone"],
            vec!["last_name", "first_name"],
            vec!["company_name", "last_name"],
        ],
        field_order: vec!["email", "phone", "first_name", "last_name", "full_name", "company_name"],
    }
}

fn company_catalog() -> SignalCatalog {
    let mut m = HashMap::new();
    m.insert("domain", exact_default("domain", 2.0, 0.92, 0.001, "generic"));
    m.insert("tax_id", exact_default("tax_id", 2.0, 0.98, 0.0001, "generic"));
    m.insert("company_name", similarity_default("company_name", 1.5, 0.80, 0.02, 0.80, "generic"));
    m.insert("industry", similarity_default("industry", 0.5, 0.60, 0.10, 0.90, "generic"));
    m.insert("email", exact_default("email", 1.5, 0.90, 0.002, "email"));
    m.insert("phone", exact_default("phone", 1.0, 0.80, 0.008, "phone"));
    SignalCatalog {
        entity_type: EntityType::Company,
        field_defaults: m,
        blocking_templates: vec![
            vec!["domain"],
            vec!["tax_id"],
            vec!["company_name"],
            vec!["company_name", "industry"],
        ],
        field_order: vec!["domain", "tax_id", "company_name", "industry", "email", "phone"],
    }
}

fn product_catalog() -> SignalCatalog {
    let mut m = HashMap::new();
    m.insert("sku", exact_default("sku", 2.5, 0.95, 0.0005, "generic"));
    m.insert("upc", exact_default("upc", 3.0, 0.99, 0.0001, "generic"));
    m.insert("product_name", similarity_default("product_name", 1.0, 0.75, 0.03, 0.80, "generic"));
    m.insert("brand", similarity_default("brand", 0.8, 0.70, 0.05, 0.85, "generic"));
    m.insert("category", similarity_default("category", 0.5, 0.60, 0.10, 0.90, "generic"));
    SignalCatalog {
        entity_type: EntityType::Product,
        field_defaults: m,
        blocking_templates: vec![
            vec!["sku"],
            vec!["upc"],
            vec!["brand", "product_name"],
            vec!["category", "product_name"],
        ],
        field_order: vec!["sku", "upc", "product_name", "brand", "category"],
    }
}

fn transaction_catalog() -> SignalCatalog {
    let mut m = HashMap::new();
    m.insert("transaction_id", exact_default("transaction_id", 3.0, 0.99, 0.0001, "generic"));
    m.insert("amount", exact_default("amount", 1.5, 0.80, 0.01, "generic"));
    m.insert("currency", exact_default("currency", 0.3, 0.95, 0.20, "generic"));
    SignalCatalog {
        entity_type: EntityType::Transaction,
        field_defaults: m,
        blocking_templates: vec![
            vec!["transaction_id"],
            vec!["amount", "currency"],
        ],
        field_order: vec!["transaction_id", "amount", "currency"],
    }
}

fn healthcare_catalog() -> SignalCatalog {
    let mut m = HashMap::new();
    m.insert("mrn", exact_default("mrn", 2.5, 0.97, 0.0002, "generic"));
    m.insert("npi", exact_default("npi", 2.0, 0.95, 0.0005, "generic"));
    m.insert("patient_name", similarity_default("patient_name", 1.0, 0.80, 0.03, 0.80, "name"));
    m.insert("date_of_birth", exact_default("date_of_birth", 1.5, 0.90, 0.003, "generic"));
    m.insert("email", exact_default("email", 1.5, 0.90, 0.002, "email"));
    m.insert("phone", exact_default("phone", 1.0, 0.80, 0.008, "phone"));
    SignalCatalog {
        entity_type: EntityType::Healthcare,
        field_defaults: m,
        blocking_templates: vec![
            vec!["mrn"],
            vec!["npi"],
            vec!["patient_name", "date_of_birth"],
        ],
        field_order: vec!["mrn", "npi", "patient_name", "date_of_birth", "email", "phone"],
    }
}

/// Get the signal catalog for an entity type. Custom types fall back to Person catalog.
pub fn catalog_for_entity(entity_type: &EntityType) -> SignalCatalog {
    match entity_type {
        EntityType::Person => person_catalog(),
        EntityType::Company => company_catalog(),
        EntityType::Product => product_catalog(),
        EntityType::Transaction => transaction_catalog(),
        EntityType::Healthcare => healthcare_catalog(),
        EntityType::Custom(_) => person_catalog(),
    }
}

/// Preferred ordering when building the plan (Person default for backward compat).
const FIELD_ORDER: &[&str] = &[
    "email",
    "phone",
    "first_name",
    "last_name",
    "full_name",
    "company_name",
];

// ---------------------------------------------------------------------------
// Data profile types
// ---------------------------------------------------------------------------

/// Profile of a single column in the data.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ColumnProfile {
    pub name: String,
    pub signal: Option<IdentitySignal>,
    pub distinct_count: usize,
    pub null_count: usize,
    pub total_count: usize,
    pub sample_values: Vec<String>,
}

impl ColumnProfile {
    pub fn uniqueness(&self) -> f64 {
        if self.total_count == 0 {
            return 0.0;
        }
        self.distinct_count as f64 / self.total_count as f64
    }

    pub fn null_rate(&self) -> f64 {
        if self.total_count == 0 {
            return 0.0;
        }
        self.null_count as f64 / self.total_count as f64
    }
}

/// Profile of all SF data for a tenant.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataProfile {
    pub tenant_id: Uuid,
    pub columns: Vec<ColumnProfile>,
    pub total_records: usize,
    pub source_names: Vec<String>,
    /// Entity type inferred from the detected signals. `None` if profiling was
    /// done with the legacy `profile_from_rows()` path and inference hasn't run.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub inferred_entity_type: Option<EntityType>,
}

impl DataProfile {
    /// Returns detected identity signals, in FIELD_ORDER (Person default).
    /// For entity-aware ordering, use `detected_signals_for_entity`.
    pub fn detected_signals(&self) -> Vec<(&ColumnProfile, &IdentitySignal)> {
        self.detected_signals_ordered(FIELD_ORDER)
    }

    /// Returns detected identity signals, ordered by the given entity type's catalog.
    pub fn detected_signals_for_entity(&self, entity_type: &EntityType) -> Vec<(&ColumnProfile, &IdentitySignal)> {
        let catalog = catalog_for_entity(entity_type);
        self.detected_signals_ordered(&catalog.field_order)
    }

    /// Internal: return signals ordered by provided field order.
    fn detected_signals_ordered(&self, field_order: &[&str]) -> Vec<(&ColumnProfile, &IdentitySignal)> {
        let mut results = Vec::new();
        for &field_name in field_order {
            for col in &self.columns {
                if let Some(ref sig) = col.signal {
                    if sig.canonical_name() == field_name && col.null_rate() < 0.9 {
                        results.push((col, sig));
                        break;
                    }
                }
            }
        }
        results
    }
}

// ---------------------------------------------------------------------------
// Data profiling (queries external_entities)
// ---------------------------------------------------------------------------

/// Build a DataProfile from pre-fetched rows of (raw_data JSON, source_name).
///
/// This is the shared profiling logic used by both `profile_sf_data` (DB query path)
/// and the worker pipeline (which already has the rows from a PgConnection).
pub fn profile_from_rows(
    tenant_id: Uuid,
    rows: &[(serde_json::Value, String)],
) -> DataProfile {
    profile_from_rows_with_entity(tenant_id, rows, None)
}

/// Build a DataProfile with optional entity type hint.
///
/// If `entity_type` is `Some`, uses that entity type's catalog for signal detection.
/// If `None`, profiles with all detectors and infers the entity type from the signals found.
pub fn profile_from_rows_with_entity(
    tenant_id: Uuid,
    rows: &[(serde_json::Value, String)],
    entity_type: Option<EntityType>,
) -> DataProfile {
    if rows.is_empty() {
        return DataProfile {
            tenant_id,
            columns: Vec::new(),
            total_records: 0,
            source_names: Vec::new(),
            inferred_entity_type: entity_type,
        };
    }

    let total_records = rows.len();
    let mut source_names: HashSet<String> = HashSet::new();

    // Collect all keys and their values
    let mut key_values: HashMap<String, Vec<Option<String>>> = HashMap::new();

    for (raw_data, source_name) in rows {
        source_names.insert(source_name.clone());
        if let Some(obj) = raw_data.as_object() {
            for (key, val) in obj {
                let entry = key_values.entry(key.clone()).or_default();
                match val.as_str() {
                    Some(s) if !s.is_empty() => entry.push(Some(s.to_string())),
                    _ => entry.push(None),
                }
            }
        }
    }

    // Build column profiles
    let mut columns = Vec::new();
    for (key, values) in &key_values {
        // Skip timestamp and FK columns, but NOT if they match a known signal pattern.
        // Many new signals have `_id` suffixes (transaction_id, tax_id, provider_id)
        // and `_date` suffixes (birth_date) that would be incorrectly filtered out.
        let has_signal_pattern = detect_signal(key).is_some();
        if !has_signal_pattern && (is_timestamp_column(key) || is_fk_column(key)) {
            continue;
        }

        let non_null: Vec<&str> = values
            .iter()
            .filter_map(|v| v.as_deref())
            .collect();
        let null_count = values.len() - non_null.len();

        let mut distinct: HashSet<&str> = HashSet::new();
        for v in &non_null {
            distinct.insert(v);
        }

        // Detect signal: values are authoritative, name is a fallback hint.
        // Values override name when they disagree on name-vs-company.
        let name_hint = detect_signal(key);
        let value_signal = detect_signal_from_values(&non_null);
        let signal = match (&name_hint, &value_signal) {
            // Values detected company but name said person name -> values win
            (Some(IdentitySignal::FullName), Some(IdentitySignal::CompanyName)) => value_signal,
            (Some(IdentitySignal::FirstName), Some(IdentitySignal::CompanyName)) => value_signal,
            (Some(IdentitySignal::LastName), Some(IdentitySignal::CompanyName)) => value_signal,
            // Values detected something specific -> values win
            (None, Some(_)) => value_signal,
            // Name hint exists, no value override -> name hint
            (Some(_), _) => name_hint,
            // Nothing detected
            _ => None,
        };

        let sample_values: Vec<String> = non_null
            .iter()
            .take(5)
            .map(|s| s.to_string())
            .collect();

        columns.push(ColumnProfile {
            name: key.clone(),
            signal,
            distinct_count: distinct.len(),
            null_count,
            total_count: values.len(),
            sample_values,
        });
    }

    let mut profile = DataProfile {
        tenant_id,
        columns,
        total_records,
        source_names: source_names.into_iter().collect(),
        inferred_entity_type: entity_type.clone(),
    };

    // If no entity type was provided, infer it from the detected signals
    if entity_type.is_none() {
        profile.inferred_entity_type = Some(infer_entity_type(&profile));
    }

    profile
}

/// Infer the entity type from detected signals in a profile.
///
/// Scores each entity type by how well the detected signals match its expected
/// signal set. The entity type with the highest score wins. Falls back to Person.
pub fn infer_entity_type(profile: &DataProfile) -> EntityType {
    let detected: HashSet<String> = profile
        .columns
        .iter()
        .filter_map(|col| col.signal.as_ref().map(|s| s.canonical_name().to_string()))
        .collect();

    let has = |name: &str| detected.contains(name);

    // Score each entity type based on detected signals
    let mut scores: Vec<(EntityType, f64)> = Vec::new();

    // Person: has (email OR phone) AND (first_name OR last_name OR full_name)
    let person_id = has("email") || has("phone");
    let person_name = has("first_name") || has("last_name") || has("full_name");
    let person_score = if person_id && person_name {
        0.9
    } else if person_id || person_name {
        0.5
    } else {
        0.3 // default fallback base
    };
    scores.push((EntityType::Person, person_score));

    // Company: has (domain OR tax_id) AND company_name
    let company_id = has("domain") || has("tax_id");
    let company_name = has("company_name");
    let company_score = if company_id && company_name {
        0.9
    } else if company_id {
        0.7
    } else if company_name && has("industry") {
        0.6
    } else {
        0.0
    };
    scores.push((EntityType::Company, company_score));

    // Product: has (sku OR upc) AND (product_name OR brand)
    let product_id = has("sku") || has("upc");
    let product_desc = has("product_name") || has("brand");
    let product_score = if product_id && product_desc {
        0.9
    } else if product_id {
        0.8
    } else if product_desc && has("category") {
        0.5
    } else {
        0.0
    };
    scores.push((EntityType::Product, product_score));

    // Transaction: has transaction_id OR (amount AND currency)
    let txn_id = has("transaction_id");
    let txn_fields = has("amount") && has("currency");
    let txn_score = if txn_id {
        0.85
    } else if txn_fields {
        0.7
    } else if has("amount") {
        0.3
    } else {
        0.0
    };
    scores.push((EntityType::Transaction, txn_score));

    // Healthcare: has (mrn OR npi) AND (patient_name OR date_of_birth)
    let hc_id = has("mrn") || has("npi");
    let hc_demo = has("patient_name") || has("date_of_birth");
    let hc_score = if hc_id && hc_demo {
        0.9
    } else if hc_id {
        0.7
    } else {
        0.0
    };
    scores.push((EntityType::Healthcare, hc_score));

    // Pick the highest scoring entity type
    scores.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
    scores.into_iter().next().map(|(et, _)| et).unwrap_or(EntityType::Person)
}

#[cfg(feature = "server")]
/// Profile SF data in external_entities for a tenant.
///
/// Scans raw_data JSONB keys, computes uniqueness and null rates,
/// detects identity signals by column name and value sampling.
pub async fn profile_sf_data(
    pool: &sqlx::PgPool,
    tenant_id: Uuid,
) -> anyhow::Result<DataProfile> {
    // Sample up to 500 records from SF sources
    #[derive(sqlx::FromRow)]
    struct ProfileRow {
        raw_data: serde_json::Value,
        name: String,
    }

    let rows = sqlx::query_as!(
        ProfileRow,
        r#"
        SELECT ee.raw_data, ds.name
        FROM external_entities ee
        JOIN data_sources ds ON ee.data_source_id = ds.id
        WHERE ee.tenant_id = $1
          AND ds.name LIKE ANY(ARRAY['sf_%', 'pd_%', 'upload_%'])
          AND ee.raw_data IS NOT NULL
        LIMIT 500
        "#,
        tenant_id,
    )
    .fetch_all(pool)
    .await?;

    let tuple_rows: Vec<(serde_json::Value, String)> = rows.into_iter().map(|r| (r.raw_data, r.name)).collect();
    Ok(profile_from_rows(tenant_id, &tuple_rows))
}

// ---------------------------------------------------------------------------
// Blocking key inference (port of Python _infer_blocking_keys, lines 443-475)
// ---------------------------------------------------------------------------

/// Infer blocking keys from the detected signals in the profile.
///
/// Returns blocking keys as lists of canonical field names.
/// Uses the profile's inferred entity type if available, otherwise defaults to Person.
pub fn infer_blocking_keys(profile: &DataProfile) -> Vec<Vec<String>> {
    let entity_type = profile
        .inferred_entity_type
        .as_ref()
        .cloned()
        .unwrap_or(EntityType::Person);
    infer_blocking_keys_for_entity(profile, &entity_type)
}

/// Infer blocking keys using the entity type's catalog templates.
///
/// For each blocking template in the catalog, checks whether all required signals
/// are present in the profile. Only templates where all fields are detected are included.
pub fn infer_blocking_keys_for_entity(
    profile: &DataProfile,
    entity_type: &EntityType,
) -> Vec<Vec<String>> {
    let detected_signals: HashSet<String> = profile
        .columns
        .iter()
        .filter_map(|col| col.signal.as_ref().map(|s| s.canonical_name().to_string()))
        .collect();

    let catalog = catalog_for_entity(entity_type);

    catalog
        .blocking_templates
        .iter()
        .filter(|template| template.iter().all(|field| detected_signals.contains(*field)))
        .map(|template| template.iter().map(|s| s.to_string()).collect())
        .collect()
}

// ---------------------------------------------------------------------------
// Plan building (port of Python bootstrap_spec, lines 370-440)
// ---------------------------------------------------------------------------

/// Build a compiled `IdentityPlan` from a data profile.
///
/// Only includes fields actually detected in the data. Sets FS priors,
/// comparators, normalizers, and blocking keys based on signal type.
pub fn build_identity_plan(
    profile: &DataProfile,
    source_names: &[String],
    identity_version: &str,
) -> IdentityPlan {
    build_identity_plan_with_entity(profile, source_names, identity_version, "sf_person")
}

/// Build a compiled `IdentityPlan` from a data profile with a custom entity name.
///
/// Parses entity_name into an EntityType, then uses that entity type's signal catalog
/// for field defaults, blocking keys, and field ordering. Falls back to Person defaults
/// for unknown entity names (backward compatible).
pub fn build_identity_plan_with_entity(
    profile: &DataProfile,
    source_names: &[String],
    identity_version: &str,
    entity_name: &str,
) -> IdentityPlan {
    // Resolve entity type: prefer the profile's inferred type, otherwise parse the name
    let entity_type = profile
        .inferred_entity_type
        .clone()
        .unwrap_or_else(|| EntityType::from_str_loose(entity_name));

    let catalog = catalog_for_entity(&entity_type);
    let detected = profile.detected_signals_for_entity(&entity_type);

    // Build FS fields and match rules from detected signals using entity catalog defaults
    let mut fs_fields = Vec::new();
    let mut rules = Vec::new();
    let pii_fields = Vec::new();

    for (_col, signal) in &detected {
        let canonical = signal.canonical_name();
        if let Some(defaults) = catalog.field_defaults.get(canonical) {
            let m = defaults.m_probability;
            let u = defaults.u_probability;
            let w = defaults.weight;
            let w_agree = (m / u).ln() * w;
            let w_disagree = ((1.0 - m) / (1.0 - u)).ln() * w;

            fs_fields.push(CompiledFsField {
                name: canonical.to_string(),
                comparator: defaults.comparator.clone(),
                weight: w,
                m_probability: m,
                u_probability: u,
                w_agree,
                w_disagree,
                normalizer: Some(defaults.normalizer.to_string()),
            });

            rules.push(CompiledRule {
                name: canonical.to_string(),
                rule_type: defaults.rule_comparator.clone(),
            });

        }
    }

    // Compute FS plan bounds
    let max_composite: f64 = fs_fields.iter().map(|f| f.w_agree).sum();
    let min_composite: f64 = fs_fields.iter().map(|f| f.w_disagree).sum();

    let fs_plan = CompiledFellegiSunterPlan {
        fields: fs_fields,
        match_threshold: 10.0,
        possible_threshold: 4.0,
        non_match_threshold: -5.0,
        merge_threshold: None,
        max_composite,
        min_composite,
        em_training: Some(CompiledEmTraining {
            max_iterations: 25,
            convergence_threshold: 0.001,
            estimate_u: Some("random_sampling".to_string()),
            max_random_pairs: 1_000_000,
            prior_weight: 0.7,
        }),
    };

    // Build blocking keys using entity-aware inference
    let blocking_keys = infer_blocking_keys_for_entity(profile, &entity_type);
    let compiled_blocking_keys: Vec<BlockingKey> = blocking_keys
        .into_iter()
        .map(|fields| {
            let transformations = fields
                .iter()
                .map(|f| match f.as_str() {
                    "email" => BlockingTransformation::NormalizeEmail,
                    "phone" => BlockingTransformation::NormalizePhone,
                    _ => BlockingTransformation::Lowercase,
                })
                .collect();
            BlockingKey {
                fields,
                transformations,
            }
        })
        .collect();

    // Build sources
    let sources: Vec<ResolvedSource> = source_names
        .iter()
        .map(|name| {
            // Map canonical field name -> original column name from the data
            let field_mappings: HashMap<String, String> = detected
                .iter()
                .map(|(col, sig)| {
                    (sig.canonical_name().to_string(), col.name.clone())
                })
                .collect();

            ResolvedSource {
                name: name.clone(),
                adapter: "database".to_string(),
                location: "external_entities".to_string(),
                id_field: "external_id".to_string(),
                field_mappings,
                temporal: None,
                freshness: None,
                schema: None,
                sampling: None,
                tags: Vec::new(),
                pii_fields: pii_fields.clone(),
                reliability: 1.0,
            }
        })
        .collect();

    // Compute plan hash
    let hash_input = format!(
        "{}:{}:{}",
        identity_version,
        source_names.join(","),
        detected
            .iter()
            .map(|(_, s)| s.canonical_name())
            .collect::<Vec<_>>()
            .join(",")
    );
    let mut hasher = Sha256::new();
    hasher.update(hash_input.as_bytes());
    let plan_hash = hex::encode(hasher.finalize());

    // Determine allow_single_field based on entity type
    let allow_single_field = match entity_type {
        EntityType::Person => vec!["email".to_string()],
        EntityType::Company => vec!["domain".to_string(), "tax_id".to_string()],
        EntityType::Product => vec!["sku".to_string(), "upc".to_string()],
        EntityType::Transaction => vec!["transaction_id".to_string()],
        EntityType::Healthcare => vec!["mrn".to_string(), "npi".to_string()],
        EntityType::Custom(_) => vec!["email".to_string()],
    };

    // Use the resolved entity type's name for the plan entity label.
    // This ensures the label matches the actual catalog used (blocking keys,
    // field defaults, etc.) rather than the caller's potentially stale name.
    let resolved_entity_name = entity_type.to_entity_name();

    IdentityPlan {
        entity: resolved_entity_name.to_string(),
        plan_hash,
        identity_version: identity_version.to_string(),
        sources,
        blocking: CompiledBlocking {
            strategy: BlockingStrategy::Composite,
            keys: compiled_blocking_keys,
            fallback_sample_rate: None,
            lsh_bands: None,
            lsh_rows: None,
            neighborhood_window: None,
            sort_fields: Vec::new(),
        },
        match_graph: MatchGraph {
            rules,
            leaf_count: detected.len(),
        },
        survivorship: SurvivorshipPolicy {
            default_strategy: SurvivorshipStrategy::MostComplete,
            field_policies: HashMap::new(),
        },
        decision: DecisionConfig {
            match_threshold: 0.9,
            review_threshold: Some(0.7),
            reject_threshold: None,
            conflict_strategy: ConflictStrategy::PreferHighConfidence,
            review_webhook: None,
            scoring_method: ScoringMethod::FellegiSunter,
            ml_ensemble_config: None,
            custom_scoring_config: None,
            fellegi_sunter: Some(fs_plan),
            tie_breaking: Vec::new(),
            clustering: Some(CompiledClusteringConfig {
                method: crate::ir::ClusteringMethod::Graph,
                min_edge_weight: 0.6,
                min_cluster_coherence: 0.6,
                max_cluster_size: 100,
            }),
            min_total_weight: 0.0,
            allow_single_field,
        },
        reference_identifiers: Vec::new(),
        relations: Vec::new(),
        exclusions: Vec::new(),
        compliance: None,
        hierarchy: None,
        audit: Some(CompiledAudit {
            log_all_comparisons: false,
            log_decisions: true,
            log_conflicts: true,
        }),
        metadata: Some(PlanMetadata {
            name: Some("Bootstrapped from data profiling".to_string()),
            description: Some(format!(
                "Auto-generated {} plan with {} detected fields",
                entity_type.display_name(),
                detected.len()
            )),
            owner: None,
            tags: vec!["bootstrapped".to_string(), entity_type.display_name().to_string()],
        }),
        governance: None,
    }
}

/// Build a placeholder plan with default 5-field schema for initial setup
/// (before any data has been synced). Metadata marks it as `placeholder`.
pub fn build_placeholder_plan(
    source_names: &[String],
    identity_version: &str,
) -> IdentityPlan {
    // Create a synthetic profile with all standard signals
    let columns = vec![
        ColumnProfile {
            name: "email".into(),
            signal: Some(IdentitySignal::Email),
            distinct_count: 0,
            null_count: 0,
            total_count: 1,
            sample_values: Vec::new(),
        },
        ColumnProfile {
            name: "phone".into(),
            signal: Some(IdentitySignal::Phone),
            distinct_count: 0,
            null_count: 0,
            total_count: 1,
            sample_values: Vec::new(),
        },
        ColumnProfile {
            name: "first_name".into(),
            signal: Some(IdentitySignal::FirstName),
            distinct_count: 0,
            null_count: 0,
            total_count: 1,
            sample_values: Vec::new(),
        },
        ColumnProfile {
            name: "last_name".into(),
            signal: Some(IdentitySignal::LastName),
            distinct_count: 0,
            null_count: 0,
            total_count: 1,
            sample_values: Vec::new(),
        },
        ColumnProfile {
            name: "company_name".into(),
            signal: Some(IdentitySignal::CompanyName),
            distinct_count: 0,
            null_count: 0,
            total_count: 1,
            sample_values: Vec::new(),
        },
    ];

    let profile = DataProfile {
        tenant_id: Uuid::nil(),
        columns,
        total_records: 0,
        source_names: source_names.to_vec(),
        inferred_entity_type: Some(EntityType::Person),
    };

    let mut plan = build_identity_plan(&profile, source_names, identity_version);

    // Mark as placeholder in metadata
    if let Some(ref mut meta) = plan.metadata {
        meta.tags.push("placeholder".to_string());
        meta.description = Some("Placeholder plan - will be rebuilt after first sync".to_string());
    }

    plan
}

// ---------------------------------------------------------------------------
// Sensitivity mapping (moved from crm.rs)
// ---------------------------------------------------------------------------

/// Map a [0.0, 1.0] sensitivity slider to Fellegi-Sunter log-likelihood
/// thresholds via piecewise linear interpolation.
///
/// Anchor points:
///  - 0.3 (Conservative): match=14.0, possible=8.0
///  - 0.5 (Balanced):     match=10.0, possible=3.0
///  - 0.7 (Moderate):     match=6.0,  possible=1.5
///  - 0.9 (Aggressive):   match=2.0,  possible=0.0
pub fn sensitivity_to_fs_thresholds(sensitivity: f64) -> (f64, f64) {
    let s = sensitivity.clamp(0.0, 1.0);

    let anchors: [(f64, f64, f64); 4] = [
        (0.3, 14.0, 8.0),
        (0.5, 10.0, 3.0),
        (0.7, 6.0, 1.5),
        (0.9, 2.0, 0.0),
    ];

    if s <= anchors[0].0 {
        return (anchors[0].1, anchors[0].2);
    }
    if s >= anchors[3].0 {
        return (anchors[3].1, anchors[3].2);
    }

    for i in 0..anchors.len() - 1 {
        let (s0, m0, p0) = anchors[i];
        let (s1, m1, p1) = anchors[i + 1];
        if s >= s0 && s <= s1 {
            let t = (s - s0) / (s1 - s0);
            let match_t = m0 + t * (m1 - m0);
            let possible_t = p0 + t * (p1 - p0);
            return (match_t, possible_t);
        }
    }

    (10.0, 3.0)
}

/// Apply a sensitivity value to an existing plan, adjusting FS thresholds.
pub fn apply_sensitivity_to_plan(plan: &mut IdentityPlan, sensitivity: f64) {
    let (match_t, possible_t) = sensitivity_to_fs_thresholds(sensitivity);
    if let Some(ref mut fs) = plan.decision.fellegi_sunter {
        fs.match_threshold = match_t;
        fs.possible_threshold = possible_t;
    }
}

// ---------------------------------------------------------------------------
// Bridge field detection (for Transitive Intelligence)
// ---------------------------------------------------------------------------

/// Detect which columns are good candidates for relationship bridge fields.
///
/// A bridge field is one that links entities across merge clusters - not a
/// primary key (uniqueness ~1.0) and not a constant (uniqueness ~0.0).
/// Fields with uniqueness ratio between 0.01 and 0.8 are selected.
///
/// Returns sorted Vec<String> for determinism. Does NOT hardcode field names;
/// discovers from data statistics.
pub fn detect_bridge_fields(profiles: &[ColumnProfile]) -> Vec<String> {
    let mut fields: Vec<String> = profiles
        .iter()
        .filter(|col| {
            let u = col.uniqueness();
            // Not PK-like (too unique) and not constant (too common)
            (0.01..=0.8).contains(&u)
                // Skip columns with very high null rates
                && col.null_rate() < 0.9
                // Only consider columns that are not timestamp/FK columns
                && !is_timestamp_column(&col.name)
                && !is_fk_column(&col.name)
        })
        .map(|col| col.name.clone())
        .collect();

    fields.sort();
    fields
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_detect_signal_email() {
        assert_eq!(detect_signal("email"), Some(IdentitySignal::Email));
        assert_eq!(detect_signal("Email"), Some(IdentitySignal::Email));
        assert_eq!(detect_signal("email_address"), Some(IdentitySignal::Email));
        assert_eq!(detect_signal("user_email"), Some(IdentitySignal::Email));
        assert_eq!(detect_signal("contact_email"), Some(IdentitySignal::Email));
    }

    #[test]
    fn test_detect_signal_phone() {
        assert_eq!(detect_signal("phone"), Some(IdentitySignal::Phone));
        assert_eq!(detect_signal("Phone_Number"), Some(IdentitySignal::Phone));
        assert_eq!(detect_signal("mobile"), Some(IdentitySignal::Phone));
        assert_eq!(detect_signal("cell"), Some(IdentitySignal::Phone));
        assert_eq!(detect_signal("tel"), Some(IdentitySignal::Phone));
    }

    #[test]
    fn test_detect_signal_names() {
        assert_eq!(detect_signal("first_name"), Some(IdentitySignal::FirstName));
        assert_eq!(detect_signal("fname"), Some(IdentitySignal::FirstName));
        assert_eq!(detect_signal("last_name"), Some(IdentitySignal::LastName));
        assert_eq!(detect_signal("surname"), Some(IdentitySignal::LastName));
        assert_eq!(detect_signal("full_name"), Some(IdentitySignal::FullName));
        assert_eq!(detect_signal("display_name"), Some(IdentitySignal::FullName));
    }

    #[test]
    fn test_detect_signal_company() {
        assert_eq!(detect_signal("company"), Some(IdentitySignal::CompanyName));
        assert_eq!(detect_signal("company_name"), Some(IdentitySignal::CompanyName));
        assert_eq!(detect_signal("account_name"), Some(IdentitySignal::CompanyName));
        assert_eq!(detect_signal("organization"), Some(IdentitySignal::CompanyName));
        assert_eq!(detect_signal("employer"), Some(IdentitySignal::CompanyName));
    }

    #[test]
    fn test_detect_signal_none() {
        assert_eq!(detect_signal("foo_bar"), None);
        assert_eq!(detect_signal("status"), None);
        assert_eq!(detect_signal("created_at"), None);
    }

    #[test]
    fn test_detect_signal_from_values_email() {
        let vals = vec!["alice@example.com", "bob@test.org", "charlie@foo.net"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Email));
    }

    #[test]
    fn test_detect_signal_from_values_phone() {
        let vals = vec!["+1-555-123-4567", "(555) 987-6543", "555.111.2222"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Phone));
    }

    #[test]
    fn test_detect_signal_from_values_company() {
        // Corporate suffixes are strong signal for company names
        let vals = vec![
            "Grand Hotels & Resorts Ltd",
            "Edge Communications",
            "United Oil & Gas Corp.",
            "Acme Inc.",
            "sForce",
        ];
        assert_eq!(
            detect_signal_from_values(&vals),
            Some(IdentitySignal::CompanyName)
        );
    }

    #[test]
    fn test_detect_signal_from_values_company_mixed() {
        // Even a minority with corporate suffixes triggers detection (> 20%)
        let vals = vec![
            "Acme LLC",
            "John Smith",
            "Jane Doe",
            "Some Company Inc.",
            "Alice Johnson",
        ];
        // 2/5 = 40% have corporate suffixes -> CompanyName
        assert_eq!(
            detect_signal_from_values(&vals),
            Some(IdentitySignal::CompanyName)
        );
    }

    #[test]
    fn test_detect_signal_from_values_person_names() {
        // Person names should NOT match company detection
        let vals = vec!["John Smith", "Jane Doe", "Alice Johnson", "Bob Williams"];
        assert_eq!(detect_signal_from_values(&vals), None);
    }

    #[test]
    fn test_values_override_name_hint_for_company() {
        // Column named "name" (hints FullName) but values contain corporate suffixes
        // -> values should win and return CompanyName
        let name_hint = detect_signal("name");
        assert_eq!(name_hint, Some(IdentitySignal::FullName));

        let vals = vec![
            "Grand Hotels & Resorts Ltd",
            "Edge Communications Inc.",
            "United Oil Corp.",
        ];
        let value_signal = detect_signal_from_values(&vals);
        assert_eq!(value_signal, Some(IdentitySignal::CompanyName));

        // The merge logic should pick values over name hint
        let signal = match (&name_hint, &value_signal) {
            (Some(IdentitySignal::FullName), Some(IdentitySignal::CompanyName)) => &value_signal,
            _ => &name_hint,
        };
        assert_eq!(*signal, Some(IdentitySignal::CompanyName));
    }

    #[test]
    fn test_detect_signal_from_values_none() {
        let vals = vec!["hello", "world", "foo"];
        assert_eq!(detect_signal_from_values(&vals), None);
    }

    #[test]
    fn test_detect_signal_from_values_empty() {
        assert_eq!(detect_signal_from_values(&[]), None);
    }

    #[test]
    fn test_is_timestamp_column() {
        assert!(is_timestamp_column("created_at"));
        assert!(is_timestamp_column("updated_at"));
        assert!(is_timestamp_column("event_time"));
        assert!(!is_timestamp_column("email"));
        assert!(!is_timestamp_column("name"));
    }

    #[test]
    fn test_is_fk_column() {
        assert!(is_fk_column("account_id"));
        assert!(is_fk_column("user_key"));
        assert!(is_fk_column("parent_fk"));
        assert!(!is_fk_column("email"));
        assert!(!is_fk_column("name"));
    }

    #[test]
    fn test_infer_blocking_keys_all_signals() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "email".into(),
                    signal: Some(IdentitySignal::Email),
                    distinct_count: 100,
                    null_count: 0,
                    total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "phone".into(),
                    signal: Some(IdentitySignal::Phone),
                    distinct_count: 90,
                    null_count: 10,
                    total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "first_name".into(),
                    signal: Some(IdentitySignal::FirstName),
                    distinct_count: 50,
                    null_count: 0,
                    total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "last_name".into(),
                    signal: Some(IdentitySignal::LastName),
                    distinct_count: 60,
                    null_count: 0,
                    total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "company_name".into(),
                    signal: Some(IdentitySignal::CompanyName),
                    distinct_count: 40,
                    null_count: 0,
                    total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["sf_contact".into()],
            inferred_entity_type: Some(EntityType::Person),
        };

        let keys = infer_blocking_keys(&profile);
        assert_eq!(keys.len(), 4);
        assert_eq!(keys[0], vec!["email"]);
        assert_eq!(keys[1], vec!["phone"]);
        assert_eq!(keys[2], vec!["last_name", "first_name"]);
        assert_eq!(keys[3], vec!["company_name", "last_name"]);
    }

    #[test]
    fn test_infer_blocking_keys_email_only() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![ColumnProfile {
                name: "email".into(),
                signal: Some(IdentitySignal::Email),
                distinct_count: 100,
                null_count: 0,
                total_count: 100,
                sample_values: Vec::new(),
            }],
            total_records: 100,
            source_names: vec!["sf_contact".into()],
            inferred_entity_type: Some(EntityType::Person),
        };

        let keys = infer_blocking_keys(&profile);
        assert_eq!(keys.len(), 1);
        assert_eq!(keys[0], vec!["email"]);
    }

    #[test]
    fn test_build_identity_plan_includes_detected_fields() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "email".into(),
                    signal: Some(IdentitySignal::Email),
                    distinct_count: 100,
                    null_count: 0,
                    total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "last_name".into(),
                    signal: Some(IdentitySignal::LastName),
                    distinct_count: 60,
                    null_count: 0,
                    total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["sf_contact".into()],
            inferred_entity_type: Some(EntityType::Person),
        };

        let plan = build_identity_plan(
            &profile,
            &["sf_contact".to_string()],
            "sf_prism_v3",
        );

        assert_eq!(plan.identity_version, "sf_prism_v3");
        // Entity name resolved from inferred_entity_type (Person -> "person"),
        // not the caller's "sf_person" string
        assert_eq!(plan.entity, "person");
        assert_eq!(plan.sources.len(), 1);
        assert_eq!(plan.sources[0].name, "sf_contact");

        // FS plan should have 2 fields
        let fs = plan.decision.fellegi_sunter.as_ref().unwrap();
        assert_eq!(fs.fields.len(), 2);
        assert_eq!(fs.fields[0].name, "email");
        assert_eq!(fs.fields[1].name, "last_name");

        // Match rules
        assert_eq!(plan.match_graph.rules.len(), 2);
    }

    #[test]
    fn test_build_placeholder_plan() {
        let plan = build_placeholder_plan(
            &["sf_contact".to_string(), "sf_lead".to_string()],
            "sf_prism_v3",
        );

        assert_eq!(plan.sources.len(), 2);
        let fs = plan.decision.fellegi_sunter.as_ref().unwrap();
        assert_eq!(fs.fields.len(), 5); // All 5 default signals
        assert!(plan
            .metadata
            .as_ref()
            .unwrap()
            .tags
            .contains(&"placeholder".to_string()));
    }

    #[test]
    fn test_sensitivity_to_fs_thresholds() {
        let (m, p) = sensitivity_to_fs_thresholds(0.5);
        assert!((m - 10.0).abs() < 0.01);
        assert!((p - 3.0).abs() < 0.01);

        let (m, p) = sensitivity_to_fs_thresholds(0.3);
        assert!((m - 14.0).abs() < 0.01);
        assert!((p - 8.0).abs() < 0.01);

        let (m, p) = sensitivity_to_fs_thresholds(0.9);
        assert!((m - 2.0).abs() < 0.01);
        assert!((p - 0.0).abs() < 0.01);

        // Interpolated
        let (m, _p) = sensitivity_to_fs_thresholds(0.4);
        assert!(m > 10.0 && m < 14.0);
    }

    #[test]
    fn test_column_profile_metrics() {
        let col = ColumnProfile {
            name: "email".into(),
            signal: Some(IdentitySignal::Email),
            distinct_count: 80,
            null_count: 20,
            total_count: 100,
            sample_values: Vec::new(),
        };
        assert!((col.uniqueness() - 0.8).abs() < 0.01);
        assert!((col.null_rate() - 0.2).abs() < 0.01);
    }

    #[test]
    fn test_detect_bridge_fields_basic() {
        let profiles = vec![
            ColumnProfile {
                name: "company_name".into(),
                signal: Some(IdentitySignal::CompanyName),
                distinct_count: 40,
                null_count: 5,
                total_count: 100,
                sample_values: Vec::new(),
            },
            ColumnProfile {
                name: "domain".into(),
                signal: None,
                distinct_count: 30,
                null_count: 10,
                total_count: 100,
                sample_values: Vec::new(),
            },
            // PK-like (uniqueness = 1.0) - should be excluded
            ColumnProfile {
                name: "record_id".into(),
                signal: None,
                distinct_count: 100,
                null_count: 0,
                total_count: 100,
                sample_values: Vec::new(),
            },
            // Constant (uniqueness = 0.0) - should be excluded
            ColumnProfile {
                name: "status".into(),
                signal: None,
                distinct_count: 0,
                null_count: 0,
                total_count: 100,
                sample_values: Vec::new(),
            },
        ];

        let fields = detect_bridge_fields(&profiles);
        assert!(fields.contains(&"company_name".to_string()));
        assert!(fields.contains(&"domain".to_string()));
        assert!(!fields.contains(&"record_id".to_string()));
        assert!(!fields.contains(&"status".to_string()));
    }

    #[test]
    fn test_detect_bridge_fields_excludes_timestamps_and_fks() {
        let profiles = vec![
            ColumnProfile {
                name: "created_at".into(),
                signal: None,
                distinct_count: 50,
                null_count: 0,
                total_count: 100,
                sample_values: Vec::new(),
            },
            ColumnProfile {
                name: "account_id".into(),
                signal: None,
                distinct_count: 50,
                null_count: 0,
                total_count: 100,
                sample_values: Vec::new(),
            },
            ColumnProfile {
                name: "industry".into(),
                signal: None,
                distinct_count: 20,
                null_count: 5,
                total_count: 100,
                sample_values: Vec::new(),
            },
        ];

        let fields = detect_bridge_fields(&profiles);
        assert!(!fields.contains(&"created_at".to_string()));
        assert!(!fields.contains(&"account_id".to_string()));
        assert!(fields.contains(&"industry".to_string()));
    }

    #[test]
    fn test_detect_bridge_fields_sorted_output() {
        let profiles = vec![
            ColumnProfile {
                name: "zebra".into(),
                signal: None,
                distinct_count: 30,
                null_count: 0,
                total_count: 100,
                sample_values: Vec::new(),
            },
            ColumnProfile {
                name: "apple".into(),
                signal: None,
                distinct_count: 30,
                null_count: 0,
                total_count: 100,
                sample_values: Vec::new(),
            },
        ];

        let fields = detect_bridge_fields(&profiles);
        assert_eq!(fields, vec!["apple".to_string(), "zebra".to_string()]);
    }

    // -----------------------------------------------------------------------
    // New signal detection tests
    // -----------------------------------------------------------------------

    #[test]
    fn test_detect_signal_product_columns() {
        assert_eq!(detect_signal("sku"), Some(IdentitySignal::Sku));
        assert_eq!(detect_signal("item_code"), Some(IdentitySignal::Sku));
        assert_eq!(detect_signal("upc"), Some(IdentitySignal::Upc));
        assert_eq!(detect_signal("ean"), Some(IdentitySignal::Upc));
        assert_eq!(detect_signal("barcode"), Some(IdentitySignal::Upc));
        assert_eq!(detect_signal("product_name"), Some(IdentitySignal::ProductName));
        assert_eq!(detect_signal("brand"), Some(IdentitySignal::Brand));
        assert_eq!(detect_signal("category"), Some(IdentitySignal::Category));
    }

    #[test]
    fn test_detect_signal_company_columns() {
        assert_eq!(detect_signal("domain"), Some(IdentitySignal::Domain));
        assert_eq!(detect_signal("tax_id"), Some(IdentitySignal::TaxId));
        assert_eq!(detect_signal("ein"), Some(IdentitySignal::TaxId));
        assert_eq!(detect_signal("industry"), Some(IdentitySignal::Industry));
    }

    #[test]
    fn test_detect_signal_transaction_columns() {
        assert_eq!(detect_signal("transaction_id"), Some(IdentitySignal::TransactionId));
        assert_eq!(detect_signal("order_id"), Some(IdentitySignal::TransactionId));
        assert_eq!(detect_signal("amount"), Some(IdentitySignal::Amount));
        assert_eq!(detect_signal("currency"), Some(IdentitySignal::Currency));
    }

    #[test]
    fn test_detect_signal_healthcare_columns() {
        assert_eq!(detect_signal("mrn"), Some(IdentitySignal::Mrn));
        assert_eq!(detect_signal("npi"), Some(IdentitySignal::Npi));
        assert_eq!(detect_signal("patient_name"), Some(IdentitySignal::PatientName));
        assert_eq!(detect_signal("date_of_birth"), Some(IdentitySignal::DateOfBirth));
        assert_eq!(detect_signal("dob"), Some(IdentitySignal::DateOfBirth));
    }

    #[test]
    fn test_detect_signal_from_values_sku() {
        let vals = vec!["AB-1234", "CD-5678", "EF-9012", "GH-3456"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Sku));
    }

    #[test]
    fn test_detect_signal_from_values_upc() {
        let vals = vec!["012345678901", "123456789012", "234567890123"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Upc));
    }

    #[test]
    fn test_detect_signal_from_values_tax_id() {
        let vals = vec!["12-3456789", "98-7654321", "45-6789012", "11-2233445"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::TaxId));
    }

    #[test]
    fn test_detect_signal_from_values_domain() {
        let vals = vec!["acme.com", "example.org", "bigcorp.io", "startup.dev", "test.co"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Domain));
    }

    #[test]
    fn test_detect_signal_from_values_npi() {
        let vals = vec!["1234567890", "0987654321", "1122334455"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Npi));
    }

    #[test]
    fn test_detect_signal_from_values_mrn() {
        let vals = vec!["MRN123456", "MRN789012", "MR-345678", "MRN901234"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Mrn));
    }

    #[test]
    fn test_mrn_regex_rejects_bare_digits() {
        // Plain 6-10 digit numbers should NOT match MRN (too many false positives)
        let vals = vec!["123456", "789012", "345678", "901234"];
        assert_ne!(detect_signal_from_values(&vals), Some(IdentitySignal::Mrn));
    }

    #[test]
    fn test_detect_signal_from_values_dob() {
        let vals = vec!["1985-06-15", "1990-01-20", "1978-12-03", "2001-03-10"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::DateOfBirth));
    }

    #[test]
    fn test_detect_signal_from_values_currency() {
        let vals = vec!["USD", "EUR", "GBP", "JPY"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Currency));
    }

    #[test]
    fn test_detect_signal_from_values_amount() {
        let vals = vec!["$19.99", "$249.00", "$5.50", "$1,299.99"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Amount));
    }

    #[test]
    fn test_detect_signal_from_values_amount_without_symbol() {
        // Decimal amounts without currency symbol should still match
        let vals = vec!["19.99", "249.00", "5.50", "1299.99"];
        assert_eq!(detect_signal_from_values(&vals), Some(IdentitySignal::Amount));
    }

    #[test]
    fn test_plain_integers_not_detected_as_amount() {
        // Plain integers (age, quantity, year) must NOT be detected as Amount
        let vals = vec!["42", "100", "2024", "7"];
        assert_ne!(detect_signal_from_values(&vals), Some(IdentitySignal::Amount));
    }

    // -----------------------------------------------------------------------
    // Entity type inference tests
    // -----------------------------------------------------------------------

    #[test]
    fn test_entity_type_from_str_loose() {
        assert_eq!(EntityType::from_str_loose("person"), EntityType::Person);
        assert_eq!(EntityType::from_str_loose("sf_person"), EntityType::Person);
        assert_eq!(EntityType::from_str_loose("company"), EntityType::Company);
        assert_eq!(EntityType::from_str_loose("product"), EntityType::Product);
        assert_eq!(EntityType::from_str_loose("transaction"), EntityType::Transaction);
        assert_eq!(EntityType::from_str_loose("healthcare"), EntityType::Healthcare);
        assert_eq!(EntityType::from_str_loose("patient"), EntityType::Healthcare);
        assert_eq!(EntityType::from_str_loose("custom_thing"), EntityType::Custom("custom_thing".to_string()));
    }

    #[test]
    fn test_infer_entity_type_person() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "email".into(),
                    signal: Some(IdentitySignal::Email),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "first_name".into(),
                    signal: Some(IdentitySignal::FirstName),
                    distinct_count: 50, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "last_name".into(),
                    signal: Some(IdentitySignal::LastName),
                    distinct_count: 60, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["crm".into()],
            inferred_entity_type: None,
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Person);
    }

    #[test]
    fn test_infer_entity_type_product() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "sku".into(),
                    signal: Some(IdentitySignal::Sku),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "product_name".into(),
                    signal: Some(IdentitySignal::ProductName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "brand".into(),
                    signal: Some(IdentitySignal::Brand),
                    distinct_count: 20, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["shopify".into()],
            inferred_entity_type: None,
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Product);
    }

    #[test]
    fn test_infer_entity_type_company() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "domain".into(),
                    signal: Some(IdentitySignal::Domain),
                    distinct_count: 90, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "company_name".into(),
                    signal: Some(IdentitySignal::CompanyName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["clearbit".into()],
            inferred_entity_type: None,
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Company);
    }

    #[test]
    fn test_infer_entity_type_healthcare() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "mrn".into(),
                    signal: Some(IdentitySignal::Mrn),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "patient_name".into(),
                    signal: Some(IdentitySignal::PatientName),
                    distinct_count: 90, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["ehr".into()],
            inferred_entity_type: None,
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Healthcare);
    }

    #[test]
    fn test_infer_entity_type_transaction() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "transaction_id".into(),
                    signal: Some(IdentitySignal::TransactionId),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "amount".into(),
                    signal: Some(IdentitySignal::Amount),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["stripe".into()],
            inferred_entity_type: None,
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Transaction);
    }

    #[test]
    fn test_infer_entity_type_defaults_to_person() {
        // No signals detected at all - should fall back to Person
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "foo".into(),
                    signal: None,
                    distinct_count: 50, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["unknown".into()],
            inferred_entity_type: None,
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Person);
    }

    // -----------------------------------------------------------------------
    // Entity-specific blocking key tests
    // -----------------------------------------------------------------------

    #[test]
    fn test_blocking_keys_product() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "sku".into(),
                    signal: Some(IdentitySignal::Sku),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "upc".into(),
                    signal: Some(IdentitySignal::Upc),
                    distinct_count: 90, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "product_name".into(),
                    signal: Some(IdentitySignal::ProductName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "brand".into(),
                    signal: Some(IdentitySignal::Brand),
                    distinct_count: 20, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "category".into(),
                    signal: Some(IdentitySignal::Category),
                    distinct_count: 10, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["shopify".into()],
            inferred_entity_type: Some(EntityType::Product),
        };

        let keys = infer_blocking_keys(&profile);
        assert_eq!(keys.len(), 4);
        assert_eq!(keys[0], vec!["sku"]);
        assert_eq!(keys[1], vec!["upc"]);
        assert_eq!(keys[2], vec!["brand", "product_name"]);
        assert_eq!(keys[3], vec!["category", "product_name"]);
    }

    #[test]
    fn test_blocking_keys_company() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "domain".into(),
                    signal: Some(IdentitySignal::Domain),
                    distinct_count: 90, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "company_name".into(),
                    signal: Some(IdentitySignal::CompanyName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "industry".into(),
                    signal: Some(IdentitySignal::Industry),
                    distinct_count: 15, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["clearbit".into()],
            inferred_entity_type: Some(EntityType::Company),
        };

        let keys = infer_blocking_keys(&profile);
        assert_eq!(keys.len(), 3);
        assert_eq!(keys[0], vec!["domain"]);
        assert_eq!(keys[1], vec!["company_name"]);
        assert_eq!(keys[2], vec!["company_name", "industry"]);
    }

    // -----------------------------------------------------------------------
    // Entity-aware plan building tests
    // -----------------------------------------------------------------------

    #[test]
    fn test_build_product_plan() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "sku".into(),
                    signal: Some(IdentitySignal::Sku),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "product_name".into(),
                    signal: Some(IdentitySignal::ProductName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["shopify".into()],
            inferred_entity_type: Some(EntityType::Product),
        };

        let plan = build_identity_plan_with_entity(
            &profile,
            &["shopify".to_string()],
            "v1",
            "product",
        );

        // Should use product catalog defaults
        let fs = plan.decision.fellegi_sunter.as_ref().unwrap();
        assert_eq!(fs.fields.len(), 2);
        assert_eq!(fs.fields[0].name, "sku");
        assert_eq!(fs.fields[1].name, "product_name");
        // SKU weight should be 2.5 (from product catalog)
        assert!((fs.fields[0].weight - 2.5).abs() < 0.01);
        // allow_single_field should include sku and upc
        assert!(plan.decision.allow_single_field.contains(&"sku".to_string()));
        assert!(plan.decision.allow_single_field.contains(&"upc".to_string()));
    }

    #[test]
    fn test_backward_compat_person_profiling() {
        // profile_from_rows without entity hint should produce Person results
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"email": "a@b.com", "first_name": "Alice", "last_name": "Smith"}), "crm".into()),
            (serde_json::json!({"email": "c@d.com", "first_name": "Bob", "last_name": "Jones"}), "crm".into()),
            (serde_json::json!({"email": "e@f.com", "first_name": "Carol", "last_name": "White"}), "crm".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        assert_eq!(profile.inferred_entity_type, Some(EntityType::Person));

        // Signals should include email, first_name, last_name
        let signals: Vec<String> = profile.detected_signals()
            .iter()
            .map(|(_, sig)| sig.canonical_name().to_string())
            .collect();
        assert!(signals.contains(&"email".to_string()));
        assert!(signals.contains(&"first_name".to_string()));
        assert!(signals.contains(&"last_name".to_string()));
    }

    #[test]
    fn test_profile_from_rows_with_product_data() {
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"sku": "AB-1234", "product_name": "Widget Pro", "brand": "Acme"}), "shopify".into()),
            (serde_json::json!({"sku": "CD-5678", "product_name": "Gadget Plus", "brand": "TechCo"}), "shopify".into()),
            (serde_json::json!({"sku": "EF-9012", "product_name": "Doohickey", "brand": "Acme"}), "warehouse".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        assert_eq!(profile.inferred_entity_type, Some(EntityType::Product));
    }

    #[test]
    fn test_profile_from_rows_with_entity_hint() {
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"domain": "acme.com", "company_name": "Acme Inc."}), "crm".into()),
        ];

        let profile = profile_from_rows_with_entity(
            Uuid::nil(),
            &rows,
            Some(EntityType::Company),
        );
        // Should use the provided hint, not infer
        assert_eq!(profile.inferred_entity_type, Some(EntityType::Company));
    }

    #[test]
    fn test_entity_type_display_name() {
        assert_eq!(EntityType::Person.display_name(), "person");
        assert_eq!(EntityType::Company.display_name(), "company");
        assert_eq!(EntityType::Product.display_name(), "product");
        assert_eq!(EntityType::Transaction.display_name(), "transaction");
        assert_eq!(EntityType::Healthcare.display_name(), "healthcare");
        assert_eq!(EntityType::Custom("widgets".into()).display_name(), "widgets");
    }

    #[test]
    fn test_entity_type_to_entity_name() {
        assert_eq!(EntityType::Person.to_entity_name(), "person");
        assert_eq!(EntityType::Product.to_entity_name(), "product");
        assert_eq!(EntityType::Custom("my_entity".into()).to_entity_name(), "my_entity");
    }

    #[test]
    fn test_fk_filter_does_not_kill_signal_columns() {
        // transaction_id, tax_id, order_id all end in _id but are signal columns.
        // The FK filter should NOT skip them.
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"transaction_id": "TXN-001", "amount": "$19.99", "currency": "USD"}), "stripe".into()),
            (serde_json::json!({"transaction_id": "TXN-002", "amount": "$49.00", "currency": "EUR"}), "stripe".into()),
            (serde_json::json!({"transaction_id": "TXN-003", "amount": "$5.50", "currency": "USD"}), "stripe".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        let col_names: Vec<&str> = profile.columns.iter().map(|c| c.name.as_str()).collect();
        // transaction_id must NOT be filtered out
        assert!(col_names.contains(&"transaction_id"), "transaction_id was filtered out by FK regex");
    }

    #[test]
    fn test_timestamp_filter_does_not_kill_birth_date() {
        // birth_date ends in _date but is a signal column, not a timestamp
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"mrn": "MRN123456", "birth_date": "1985-06-15"}), "ehr".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        let col_names: Vec<&str> = profile.columns.iter().map(|c| c.name.as_str()).collect();
        assert!(col_names.contains(&"birth_date"), "birth_date was filtered out by timestamp regex");
    }

    #[test]
    fn test_plan_entity_matches_inferred_type() {
        // When profiler infers Product, the plan entity should say "product" not the caller's name
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "sku".into(),
                    signal: Some(IdentitySignal::Sku),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
                ColumnProfile {
                    name: "product_name".into(),
                    signal: Some(IdentitySignal::ProductName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                },
            ],
            total_records: 100,
            source_names: vec!["shopify".into()],
            inferred_entity_type: Some(EntityType::Product),
        };

        // Caller passes "person" but profile says Product - plan should use "product"
        let plan = build_identity_plan_with_entity(
            &profile,
            &["shopify".to_string()],
            "v1",
            "person",
        );
        assert_eq!(plan.entity, "product");
    }

    #[test]
    fn test_non_signal_fk_columns_still_filtered() {
        // Regular FK columns like account_id should still be filtered
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"email": "a@b.com", "account_id": "ACC-123", "user_key": "UK-1"}), "crm".into()),
            (serde_json::json!({"email": "c@d.com", "account_id": "ACC-456", "user_key": "UK-2"}), "crm".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        let col_names: Vec<&str> = profile.columns.iter().map(|c| c.name.as_str()).collect();
        assert!(!col_names.contains(&"account_id"), "account_id should be filtered as FK");
        assert!(!col_names.contains(&"user_key"), "user_key should be filtered as FK");
        assert!(col_names.contains(&"email"), "email should not be filtered");
    }
}
