//! Data profile types and profiling logic.

use super::catalogs::{catalog_for_entity, FIELD_ORDER};
use super::signals::{
    detect_signal, detect_signal_from_values, is_fk_column, is_timestamp_column, EntityType,
    IdentitySignal,
};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use uuid::Uuid;

// ---------------------------------------------------------------------------
// Data profile types
// ---------------------------------------------------------------------------

/// Profile of a single column in the data.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ColumnProfile {
    pub name: String,
    pub signal: Option<IdentitySignal>,
    pub distinct_count: usize,
    pub null_count: usize,
    pub total_count: usize,
    pub sample_values: Vec<String>,
    /// Cross-source value overlap (max pairwise Jaccard similarity).
    /// Only computed for identifier signals (Sku, Mpn, Gtin, Upc, Asin).
    /// `None` if single source or not an identifier signal.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cross_source_overlap: Option<f64>,
}

impl ColumnProfile {
    pub fn uniqueness(&self) -> f64 {
        if self.total_count == 0 {
            return 0.0;
        }
        self.distinct_count as f64 / self.total_count as f64
    }

    pub fn null_rate(&self) -> f64 {
        if self.total_count == 0 {
            return 0.0;
        }
        self.null_count as f64 / self.total_count as f64
    }
}

/// Profile of all SF data for a tenant.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataProfile {
    pub tenant_id: Uuid,
    pub columns: Vec<ColumnProfile>,
    pub total_records: usize,
    pub source_names: Vec<String>,
    /// Entity type inferred from the detected signals. `None` if profiling was
    /// done with the legacy `profile_from_rows()` path and inference hasn't run.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub inferred_entity_type: Option<EntityType>,
    /// Column names present in each source. Used to build per-source field
    /// mappings so that different sources can have different column names for
    /// the same canonical signal (e.g. `item_name` vs `product_name`).
    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
    pub source_columns: HashMap<String, HashSet<String>>,
}

impl DataProfile {
    /// Returns detected identity signals, in FIELD_ORDER (Person default).
    /// For entity-aware ordering, use `detected_signals_for_entity`.
    pub fn detected_signals(&self) -> Vec<(&ColumnProfile, &IdentitySignal)> {
        self.detected_signals_ordered(FIELD_ORDER)
    }

    /// Returns detected identity signals, ordered by the given entity type's catalog.
    pub fn detected_signals_for_entity(&self, entity_type: &EntityType) -> Vec<(&ColumnProfile, &IdentitySignal)> {
        let catalog = catalog_for_entity(entity_type);
        self.detected_signals_ordered(&catalog.field_order)
    }

    /// Internal: return signals ordered by provided field order.
    ///
    /// When multiple columns share the same canonical signal (e.g. "brand" and
    /// "maker" both detected as Brand), prefer the column whose name exactly
    /// matches the canonical name. This avoids non-determinism from HashMap
    /// iteration order in `self.columns`.
    fn detected_signals_ordered(&self, field_order: &[&str]) -> Vec<(&ColumnProfile, &IdentitySignal)> {
        let mut results = Vec::new();
        for &field_name in field_order {
            let mut best: Option<(&ColumnProfile, &IdentitySignal)> = None;
            for col in &self.columns {
                if let Some(ref sig) = col.signal {
                    if sig.canonical_name() == field_name && col.null_rate() < 0.9 {
                        if col.name == field_name {
                            // Exact name match - use immediately
                            best = Some((col, sig));
                            break;
                        } else if best.is_none() {
                            // First non-exact match - keep as fallback
                            best = Some((col, sig));
                        }
                    }
                }
            }
            if let Some(entry) = best {
                results.push(entry);
            }
        }
        results
    }
}

// ---------------------------------------------------------------------------
// Data profiling (queries external_entities)
// ---------------------------------------------------------------------------

/// Build a DataProfile from pre-fetched rows of (raw_data JSON, source_name).
///
/// This is the shared profiling logic used by both `profile_sf_data` (DB query path)
/// and the worker pipeline (which already has the rows from a PgConnection).
pub fn profile_from_rows(
    tenant_id: Uuid,
    rows: &[(serde_json::Value, String)],
) -> DataProfile {
    profile_from_rows_with_entity(tenant_id, rows, None)
}

/// Build a DataProfile with optional entity type hint.
///
/// If `entity_type` is `Some`, uses that entity type's catalog for signal detection.
/// If `None`, profiles with all detectors and infers the entity type from the signals found.
pub fn profile_from_rows_with_entity(
    tenant_id: Uuid,
    rows: &[(serde_json::Value, String)],
    entity_type: Option<EntityType>,
) -> DataProfile {
    if rows.is_empty() {
        return DataProfile {
            tenant_id,
            columns: Vec::new(),
            total_records: 0,
            source_names: Vec::new(),
            inferred_entity_type: entity_type,
            source_columns: HashMap::new(),
        };
    }

    let total_records = rows.len();
    let mut source_names: HashSet<String> = HashSet::new();
    let mut source_columns: HashMap<String, HashSet<String>> = HashMap::new();

    // Collect all keys and their values, tracking which columns each source has
    let mut key_values: HashMap<String, Vec<Option<String>>> = HashMap::new();

    for (raw_data, source_name) in rows {
        source_names.insert(source_name.clone());
        if let Some(obj) = raw_data.as_object() {
            let src_cols = source_columns.entry(source_name.clone()).or_default();
            for (key, val) in obj {
                src_cols.insert(key.clone());
                let entry = key_values.entry(key.clone()).or_default();
                match val.as_str() {
                    Some(s) if !s.is_empty() => entry.push(Some(s.to_string())),
                    _ => entry.push(None),
                }
            }
        }
    }

    // Build column profiles
    let mut columns = Vec::new();
    for (key, values) in &key_values {
        // Skip timestamp and FK columns, but NOT if they match a known signal pattern.
        // Many new signals have `_id` suffixes (transaction_id, tax_id, provider_id)
        // and `_date` suffixes (birth_date) that would be incorrectly filtered out.
        let has_signal_pattern = detect_signal(key).is_some();
        if !has_signal_pattern && (is_timestamp_column(key) || is_fk_column(key)) {
            continue;
        }

        let non_null: Vec<&str> = values
            .iter()
            .filter_map(|v| v.as_deref())
            .collect();
        let null_count = values.len() - non_null.len();

        let mut distinct: HashSet<&str> = HashSet::new();
        for v in &non_null {
            distinct.insert(v);
        }

        // Detect signal: values are authoritative, name is a fallback hint.
        // Values override name when they disagree on name-vs-company.
        let name_hint = detect_signal(key);
        let value_signal = detect_signal_from_values(&non_null);
        let signal = match (&name_hint, &value_signal) {
            // Values detected company but name said person name -> values win
            (Some(IdentitySignal::FullName), Some(IdentitySignal::CompanyName)) => value_signal,
            (Some(IdentitySignal::FirstName), Some(IdentitySignal::CompanyName)) => value_signal,
            (Some(IdentitySignal::LastName), Some(IdentitySignal::CompanyName)) => value_signal,
            // SKU reclassification: "sku" column but values are actually barcodes or ASINs
            (Some(IdentitySignal::Sku), Some(IdentitySignal::Gtin)) => value_signal,
            (Some(IdentitySignal::Sku), Some(IdentitySignal::Upc)) => value_signal,
            (Some(IdentitySignal::Sku), Some(IdentitySignal::Asin)) => value_signal,
            // Values detected something specific -> values win
            (None, Some(_)) => value_signal,
            // Name hint exists, no value override -> name hint
            (Some(_), _) => name_hint,
            // Nothing detected
            _ => None,
        };

        let sample_values: Vec<String> = non_null
            .iter()
            .take(5)
            .map(|s| s.to_string())
            .collect();

        columns.push(ColumnProfile {
            name: key.clone(),
            signal,
            distinct_count: distinct.len(),
            null_count,
            total_count: values.len(),
            sample_values,
            cross_source_overlap: None,
        });
    }

    // Compute cross-source value overlap for identifier signals.
    // For each column with an identifier signal (Sku, Mpn, Gtin, Upc, Asin),
    // collect values per source and compute max pairwise Jaccard similarity.
    //
    // IMPORTANT: Group by SIGNAL canonical name, not column name. Two sources may
    // have different column names for the same signal (e.g., "sku" vs "supplier_sku"),
    // but we need to detect overlap of their VALUES regardless of column naming.
    if source_names.len() >= 2 {
        // Step 1: Build column_name -> signal_canonical lookup from already-detected signals
        let col_to_signal: HashMap<&str, &str> = columns.iter()
            .filter_map(|col| {
                let sig = col.signal.as_ref()?;
                let canonical = sig.canonical_name();
                let is_identifier = matches!(
                    sig,
                    IdentitySignal::Sku
                        | IdentitySignal::Mpn
                        | IdentitySignal::Gtin
                        | IdentitySignal::Upc
                        | IdentitySignal::Asin
                );
                if is_identifier { Some((col.name.as_str(), canonical)) } else { None }
            })
            .collect();

        // Step 2: Build per-source value sets grouped by SIGNAL: signal -> source -> values
        let mut signal_source_values: HashMap<&str, HashMap<String, HashSet<String>>> = HashMap::new();
        for (raw_data, source_name) in rows {
            if let Some(obj) = raw_data.as_object() {
                for (key, val) in obj {
                    if let Some(&signal_name) = col_to_signal.get(key.as_str()) {
                        if let Some(s) = val.as_str() {
                            if !s.is_empty() {
                                signal_source_values
                                    .entry(signal_name)
                                    .or_default()
                                    .entry(source_name.clone())
                                    .or_default()
                                    .insert(s.to_lowercase());
                            }
                        }
                    }
                }
            }
        }

        // Step 3: Compute overlap per signal canoncial name, then assign to matching columns
        let mut signal_overlaps: HashMap<&str, f64> = HashMap::new();
        for (&signal_name, source_vals) in &signal_source_values {
            if source_vals.len() >= 2 {
                let sources: Vec<&HashSet<String>> = source_vals.values().collect();
                let mut max_jaccard = 0.0_f64;
                for i in 0..sources.len() {
                    for j in (i + 1)..sources.len() {
                        let intersection = sources[i].intersection(sources[j]).count();
                        let union = sources[i].union(sources[j]).count();
                        if union > 0 {
                            let jaccard = intersection as f64 / union as f64;
                            if jaccard > max_jaccard {
                                max_jaccard = jaccard;
                            }
                        }
                    }
                }
                signal_overlaps.insert(signal_name, max_jaccard);
            }
        }

        // Assign overlap to all columns with the matching signal
        for col in &mut columns {
            if let Some(ref sig) = col.signal {
                if let Some(&overlap) = signal_overlaps.get(sig.canonical_name()) {
                    col.cross_source_overlap = Some(overlap);
                }
            }
        }
    }

    let mut profile = DataProfile {
        tenant_id,
        columns,
        total_records,
        source_names: source_names.into_iter().collect(),
        inferred_entity_type: entity_type.clone(),
        source_columns,
    };

    // If no entity type was provided, infer it from the detected signals
    if entity_type.is_none() {
        profile.inferred_entity_type = Some(infer_entity_type(&profile));
    }

    profile
}

/// Infer the entity type from detected signals in a profile.
///
/// Scores each entity type by how well the detected signals match its expected
/// signal set. The entity type with the highest score wins. Falls back to Person.
pub fn infer_entity_type(profile: &DataProfile) -> EntityType {
    let detected: HashSet<String> = profile
        .columns
        .iter()
        .filter_map(|col| col.signal.as_ref().map(|s| s.canonical_name().to_string()))
        .collect();

    let has = |name: &str| detected.contains(name);

    // Score each entity type based on detected signals
    let mut scores: Vec<(EntityType, f64)> = Vec::new();

    // Person: has (email OR phone) AND (first_name OR last_name OR full_name)
    let person_id = has("email") || has("phone");
    let person_name = has("first_name") || has("last_name") || has("full_name");
    let person_score = if person_id && person_name {
        0.9
    } else if person_id || person_name {
        0.5
    } else {
        0.3 // default fallback base
    };
    scores.push((EntityType::Person, person_score));

    // Company: has (domain OR tax_id) AND company_name
    let company_id = has("domain") || has("tax_id");
    let company_name = has("company_name");
    let company_score = if company_id && company_name {
        0.9
    } else if company_id {
        0.7
    } else if company_name && has("industry") {
        0.6
    } else {
        0.0
    };
    scores.push((EntityType::Company, company_score));

    // Product: has (sku OR upc OR gtin OR asin) AND (product_name OR brand)
    let product_id = has("sku") || has("upc") || has("gtin") || has("asin");
    let product_desc = has("product_name") || has("brand");
    let product_score = if product_id && product_desc {
        0.9
    } else if product_id {
        0.8
    } else if product_desc && has("category") {
        0.5
    } else {
        0.0
    };
    scores.push((EntityType::Product, product_score));

    // Transaction: has transaction_id OR (amount AND currency)
    let txn_id = has("transaction_id");
    let txn_fields = has("amount") && has("currency");
    let txn_score = if txn_id {
        0.85
    } else if txn_fields {
        0.7
    } else if has("amount") {
        0.3
    } else {
        0.0
    };
    scores.push((EntityType::Transaction, txn_score));

    // Healthcare: has (mrn OR npi) AND (patient_name OR date_of_birth)
    let hc_id = has("mrn") || has("npi");
    let hc_demo = has("patient_name") || has("date_of_birth");
    let hc_score = if hc_id && hc_demo {
        0.9
    } else if hc_id {
        0.7
    } else {
        0.0
    };
    scores.push((EntityType::Healthcare, hc_score));

    // Pick the highest scoring entity type
    scores.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
    scores.into_iter().next().map(|(et, _)| et).unwrap_or(EntityType::Person)
}

#[cfg(feature = "server")]
/// Profile SF data in external_entities for a tenant.
///
/// Scans raw_data JSONB keys, computes uniqueness and null rates,
/// detects identity signals by column name and value sampling.
pub async fn profile_sf_data(
    pool: &sqlx::PgPool,
    tenant_id: Uuid,
) -> anyhow::Result<DataProfile> {
    // Sample up to 500 records from SF sources
    #[derive(sqlx::FromRow)]
    struct ProfileRow {
        raw_data: serde_json::Value,
        name: String,
    }

    let rows = sqlx::query_as!(
        ProfileRow,
        r#"
        SELECT ee.raw_data, ds.name
        FROM external_entities ee
        JOIN data_sources ds ON ee.data_source_id = ds.id
        WHERE ee.tenant_id = $1
          AND ds.name LIKE ANY(ARRAY['sf_%', 'pd_%', 'upload_%'])
          AND ee.raw_data IS NOT NULL
        LIMIT 500
        "#,
        tenant_id,
    )
    .fetch_all(pool)
    .await?;

    let tuple_rows: Vec<(serde_json::Value, String)> = rows.into_iter().map(|r| (r.raw_data, r.name)).collect();
    Ok(profile_from_rows(tenant_id, &tuple_rows))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_column_profile_metrics() {
        let col = ColumnProfile {
            name: "email".into(),
            signal: Some(IdentitySignal::Email),
            distinct_count: 80,
            null_count: 20,
            total_count: 100,
            sample_values: Vec::new(),
            cross_source_overlap: None,
        };
        assert!((col.uniqueness() - 0.8).abs() < 0.01);
        assert!((col.null_rate() - 0.2).abs() < 0.01);
    }

    #[test]
    fn test_infer_entity_type_person() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "email".into(),
                    signal: Some(IdentitySignal::Email),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
                ColumnProfile {
                    name: "first_name".into(),
                    signal: Some(IdentitySignal::FirstName),
                    distinct_count: 50, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
                ColumnProfile {
                    name: "last_name".into(),
                    signal: Some(IdentitySignal::LastName),
                    distinct_count: 60, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
            ],
            total_records: 100,
            source_names: vec!["crm".into()],
            inferred_entity_type: None,
            source_columns: HashMap::new(),
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Person);
    }

    #[test]
    fn test_infer_entity_type_product() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "sku".into(),
                    signal: Some(IdentitySignal::Sku),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
                ColumnProfile {
                    name: "product_name".into(),
                    signal: Some(IdentitySignal::ProductName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
                ColumnProfile {
                    name: "brand".into(),
                    signal: Some(IdentitySignal::Brand),
                    distinct_count: 20, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
            ],
            total_records: 100,
            source_names: vec!["shopify".into()],
            inferred_entity_type: None,
            source_columns: HashMap::new(),
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Product);
    }

    #[test]
    fn test_infer_entity_type_company() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "domain".into(),
                    signal: Some(IdentitySignal::Domain),
                    distinct_count: 90, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
                ColumnProfile {
                    name: "company_name".into(),
                    signal: Some(IdentitySignal::CompanyName),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
            ],
            total_records: 100,
            source_names: vec!["clearbit".into()],
            inferred_entity_type: None,
            source_columns: HashMap::new(),
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Company);
    }

    #[test]
    fn test_infer_entity_type_healthcare() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "mrn".into(),
                    signal: Some(IdentitySignal::Mrn),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
                ColumnProfile {
                    name: "patient_name".into(),
                    signal: Some(IdentitySignal::PatientName),
                    distinct_count: 90, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
            ],
            total_records: 100,
            source_names: vec!["ehr".into()],
            inferred_entity_type: None,
            source_columns: HashMap::new(),
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Healthcare);
    }

    #[test]
    fn test_infer_entity_type_transaction() {
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "transaction_id".into(),
                    signal: Some(IdentitySignal::TransactionId),
                    distinct_count: 100, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
                ColumnProfile {
                    name: "amount".into(),
                    signal: Some(IdentitySignal::Amount),
                    distinct_count: 80, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
            ],
            total_records: 100,
            source_names: vec!["stripe".into()],
            inferred_entity_type: None,
            source_columns: HashMap::new(),
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Transaction);
    }

    #[test]
    fn test_infer_entity_type_defaults_to_person() {
        // No signals detected at all - should fall back to Person
        let profile = DataProfile {
            tenant_id: Uuid::nil(),
            columns: vec![
                ColumnProfile {
                    name: "foo".into(),
                    signal: None,
                    distinct_count: 50, null_count: 0, total_count: 100,
                    sample_values: Vec::new(),
                    cross_source_overlap: None,
                },
            ],
            total_records: 100,
            source_names: vec!["unknown".into()],
            inferred_entity_type: None,
            source_columns: HashMap::new(),
        };
        assert_eq!(infer_entity_type(&profile), EntityType::Person);
    }

    #[test]
    fn test_backward_compat_person_profiling() {
        // profile_from_rows without entity hint should produce Person results
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"email": "a@b.com", "first_name": "Alice", "last_name": "Smith"}), "crm".into()),
            (serde_json::json!({"email": "c@d.com", "first_name": "Bob", "last_name": "Jones"}), "crm".into()),
            (serde_json::json!({"email": "e@f.com", "first_name": "Carol", "last_name": "White"}), "crm".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        assert_eq!(profile.inferred_entity_type, Some(EntityType::Person));

        // Signals should include email, first_name, last_name
        let signals: Vec<String> = profile.detected_signals()
            .iter()
            .map(|(_, sig)| sig.canonical_name().to_string())
            .collect();
        assert!(signals.contains(&"email".to_string()));
        assert!(signals.contains(&"first_name".to_string()));
        assert!(signals.contains(&"last_name".to_string()));
    }

    #[test]
    fn test_profile_from_rows_with_product_data() {
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"sku": "AB-1234", "product_name": "Widget Pro", "brand": "Acme"}), "shopify".into()),
            (serde_json::json!({"sku": "CD-5678", "product_name": "Gadget Plus", "brand": "TechCo"}), "shopify".into()),
            (serde_json::json!({"sku": "EF-9012", "product_name": "Doohickey", "brand": "Acme"}), "warehouse".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        assert_eq!(profile.inferred_entity_type, Some(EntityType::Product));
    }

    #[test]
    fn test_profile_from_rows_with_entity_hint() {
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"domain": "acme.com", "company_name": "Acme Inc."}), "crm".into()),
        ];

        let profile = profile_from_rows_with_entity(
            Uuid::nil(),
            &rows,
            Some(EntityType::Company),
        );
        // Should use the provided hint, not infer
        assert_eq!(profile.inferred_entity_type, Some(EntityType::Company));
    }

    #[test]
    fn test_fk_filter_does_not_kill_signal_columns() {
        // transaction_id, tax_id, order_id all end in _id but are signal columns.
        // The FK filter should NOT skip them.
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"transaction_id": "TXN-001", "amount": "$19.99", "currency": "USD"}), "stripe".into()),
            (serde_json::json!({"transaction_id": "TXN-002", "amount": "$49.00", "currency": "EUR"}), "stripe".into()),
            (serde_json::json!({"transaction_id": "TXN-003", "amount": "$5.50", "currency": "USD"}), "stripe".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        let col_names: Vec<&str> = profile.columns.iter().map(|c| c.name.as_str()).collect();
        // transaction_id must NOT be filtered out
        assert!(col_names.contains(&"transaction_id"), "transaction_id was filtered out by FK regex");
    }

    #[test]
    fn test_timestamp_filter_does_not_kill_birth_date() {
        // birth_date ends in _date but is a signal column, not a timestamp
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"mrn": "MRN123456", "birth_date": "1985-06-15"}), "ehr".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        let col_names: Vec<&str> = profile.columns.iter().map(|c| c.name.as_str()).collect();
        assert!(col_names.contains(&"birth_date"), "birth_date was filtered out by timestamp regex");
    }

    #[test]
    fn test_non_signal_fk_columns_still_filtered() {
        // Regular FK columns like account_id should still be filtered
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"email": "a@b.com", "account_id": "ACC-123", "user_key": "UK-1"}), "crm".into()),
            (serde_json::json!({"email": "c@d.com", "account_id": "ACC-456", "user_key": "UK-2"}), "crm".into()),
        ];

        let profile = profile_from_rows(Uuid::nil(), &rows);
        let col_names: Vec<&str> = profile.columns.iter().map(|c| c.name.as_str()).collect();
        assert!(!col_names.contains(&"account_id"), "account_id should be filtered as FK");
        assert!(!col_names.contains(&"user_key"), "user_key should be filtered as FK");
        assert!(col_names.contains(&"email"), "email should not be filtered");
    }

    #[test]
    fn test_cross_source_overlap_detected() {
        // Two sources share the same SKU values -> overlap should be > 0
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"sku": "555088-062", "brand": "Nike"}), "jd_sports".into()),
            (serde_json::json!({"sku": "BY3549", "brand": "Adidas"}), "jd_sports".into()),
            (serde_json::json!({"sku": "555088-062", "brand": "Nike"}), "wholesale".into()),
            (serde_json::json!({"sku": "CW2288-001", "brand": "Nike"}), "wholesale".into()),
        ];

        let profile = profile_from_rows_with_entity(Uuid::nil(), &rows, Some(EntityType::Product));
        let sku_col = profile.columns.iter().find(|c| c.name == "sku").unwrap();
        let overlap = sku_col.cross_source_overlap.unwrap();
        assert!(overlap > 0.0, "Expected overlap > 0, got {}", overlap);
    }

    #[test]
    fn test_cross_source_overlap_not_detected() {
        // Two sources have completely different internal SKU values -> overlap = 0
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"sku": "STR-APP-001", "brand": "Apple"}), "store".into()),
            (serde_json::json!({"sku": "STR-APP-002", "brand": "Apple"}), "store".into()),
            (serde_json::json!({"sku": "RE29571", "brand": "Apple"}), "supplier".into()),
            (serde_json::json!({"sku": "RE29572", "brand": "Apple"}), "supplier".into()),
        ];

        let profile = profile_from_rows_with_entity(Uuid::nil(), &rows, Some(EntityType::Product));
        let sku_col = profile.columns.iter().find(|c| c.name == "sku").unwrap();
        let overlap = sku_col.cross_source_overlap.unwrap();
        assert!((overlap - 0.0).abs() < 0.001, "Expected overlap ~ 0, got {}", overlap);
    }

    #[test]
    fn test_cross_source_overlap_different_column_names() {
        // Two sources use different column names for the same SKU signal.
        // Source A has "sku", source B has "supplier_sku". They share values,
        // so overlap should still be detected by grouping on signal, not column name.
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"sku": "555088-062", "brand": "Nike"}), "retailer".into()),
            (serde_json::json!({"sku": "BY3549", "brand": "Adidas"}), "retailer".into()),
            (serde_json::json!({"sku": "CW2288-001", "brand": "Nike"}), "retailer".into()),
            (serde_json::json!({"supplier_sku": "555088-062", "brand": "Nike"}), "wholesale".into()),
            (serde_json::json!({"supplier_sku": "BY3549", "brand": "Adidas"}), "wholesale".into()),
            (serde_json::json!({"supplier_sku": "AQ1090-003", "brand": "Nike"}), "wholesale".into()),
        ];

        let profile = profile_from_rows_with_entity(Uuid::nil(), &rows, Some(EntityType::Product));

        // Both columns should have the Sku signal detected
        let sku_col = profile.columns.iter().find(|c| c.name == "sku").unwrap();
        let supplier_col = profile.columns.iter().find(|c| c.name == "supplier_sku").unwrap();
        assert_eq!(sku_col.signal, Some(IdentitySignal::Sku));
        assert_eq!(supplier_col.signal, Some(IdentitySignal::Sku));

        // Both should have overlap detected (same signal, shared values across sources)
        let sku_overlap = sku_col.cross_source_overlap.unwrap();
        let supplier_overlap = supplier_col.cross_source_overlap.unwrap();
        assert!(sku_overlap > 0.0, "Expected sku overlap > 0, got {}", sku_overlap);
        assert!(supplier_overlap > 0.0, "Expected supplier_sku overlap > 0, got {}", supplier_overlap);
        // Both should have the same overlap value (same signal)
        assert!((sku_overlap - supplier_overlap).abs() < 0.001,
            "Expected same overlap for both columns, got {} vs {}", sku_overlap, supplier_overlap);
    }

    #[test]
    fn test_cross_source_overlap_single_source_skipped() {
        // Single source should not compute overlap (returns None)
        let rows: Vec<(serde_json::Value, String)> = vec![
            (serde_json::json!({"sku": "555088-062", "brand": "Nike"}), "single_source".into()),
            (serde_json::json!({"sku": "BY3549", "brand": "Adidas"}), "single_source".into()),
        ];

        let profile = profile_from_rows_with_entity(Uuid::nil(), &rows, Some(EntityType::Product));
        let sku_col = profile.columns.iter().find(|c| c.name == "sku").unwrap();
        assert!(sku_col.cross_source_overlap.is_none(),
            "Single source should not compute overlap");
    }

    #[test]
    fn test_cross_source_overlap_three_sources_max_pairwise() {
        // Three sources: A and C share values, B has unique values.
        // Max pairwise Jaccard should be between A and C.
        let rows: Vec<(serde_json::Value, String)> = vec![
            // Source A
            (serde_json::json!({"sku": "MFG-001", "brand": "X"}), "source_a".into()),
            (serde_json::json!({"sku": "MFG-002", "brand": "X"}), "source_a".into()),
            // Source B (completely unique)
            (serde_json::json!({"sku": "INT-AAA", "brand": "X"}), "source_b".into()),
            (serde_json::json!({"sku": "INT-BBB", "brand": "X"}), "source_b".into()),
            // Source C (shares with A)
            (serde_json::json!({"sku": "MFG-001", "brand": "X"}), "source_c".into()),
            (serde_json::json!({"sku": "MFG-003", "brand": "X"}), "source_c".into()),
        ];

        let profile = profile_from_rows_with_entity(Uuid::nil(), &rows, Some(EntityType::Product));
        let sku_col = profile.columns.iter().find(|c| c.name == "sku").unwrap();
        let overlap = sku_col.cross_source_overlap.unwrap();
        // A={mfg-001, mfg-002}, C={mfg-001, mfg-003} -> intersection=1, union=3 -> Jaccard=0.333
        // A vs B: 0 overlap. B vs C: 0 overlap.
        // Max pairwise = A vs C = 0.333
        assert!(overlap > 0.3 && overlap < 0.4,
            "Expected ~0.333 Jaccard between A and C, got {}", overlap);
    }
}
