from __future__ import annotations
from typing import Any, Dict, List, Literal, Tuple, Optional
from collections import defaultdict, OrderedDict
from datetime import datetime
import json
import os
import time
import hashlib
import logging
from difflib import SequenceMatcher

import pandas as pd
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text

from .base_intelligent_service import BaseIntelligentService
from .dedupe_service_intelligent import (
    IntelligentDedupeService,
)  # reuse your persistence
from .domain_family import (
    get_registry,
    CONFIDENCE_THRESHOLD_ROUTING,
)
from .salesforce_gateway import SalesforceGateway
from ..utils.logging_utils import _s

# Import shared normalization utilities
from .utils.match_normalization import (
    normalize_score_column,
    normalize_id_columns,
    extract_id_from_dict,
)

log = logging.getLogger(__name__)

ObjectType = Literal["Lead", "Contact", "Account"]

# Common free email domains that shouldn't drive matches
COMMON_FREE = {
    "gmail.com",
    "googlemail.com",
    "yahoo.com",
    "outlook.com",
    "hotmail.com",
    "icloud.com",
    "aol.com",
    "mail.com",
    "protonmail.com",
}

# Coverage thresholds
MIN_COVERAGE_MAPPING = 0.10
MIN_COVERAGE_BLOCKING = 0.25
DOMAIN_WEIGHT_REDUCTION = 0.7


# TTL Cache for Intelligence configs
class _TTLCache:
    def __init__(self, ttl_s=900, max_entries=200):
        self.ttl_s = ttl_s
        self.max = max_entries
        self._store = OrderedDict()

    def get(self, key):
        item = self._store.get(key)
        if not item:
            return None
        exp, val = item
        if exp < time.time():
            self._store.pop(key, None)
            return None
        self._store.move_to_end(key)
        return val

    def set(self, key, val):
        now = time.time()
        self._store[key] = (now + self.ttl_s, val)
        self._store.move_to_end(key)
        if len(self._store) > self.max:
            self._store.popitem(last=False)


_INTEL_CACHE = _TTLCache(ttl_s=900, max_entries=200)

# Core fields that should always exist
CORE_FIELDS: Dict[ObjectType, List[str]] = {
    "Lead": [
        "Id",
        "FirstName",
        "LastName",
        "Company",
        "Email",
        "Phone",
        "MobilePhone",
        "CreatedDate",
        "IsConverted",
        "IsDeleted",
    ],
    "Contact": [
        "Id",
        "FirstName",
        "LastName",
        "AccountId",
        "Email",
        "Phone",
        "MobilePhone",
        "CreatedDate",
        "IsDeleted",
    ],
    "Account": ["Id", "Name", "Website", "Phone", "CreatedDate", "IsDeleted"],
}

# Optional custom fields - will be checked before use
OPTIONAL_FIELDS: Dict[ObjectType, List[str]] = {
    "Lead": ["LinkedIn__c", "External_Id__c", "Vendor_Lead_Id__c"],
    "Contact": ["LinkedIn__c", "External_Id__c"],
    "Account": ["LinkedIn_Company__c", "EIN__c", "ABN__c"],
}


def _where_clause(obj: ObjectType) -> str:
    w = "IsDeleted = false"
    if obj == "Lead":
        w += " AND IsConverted = false"
    return w


def norm_email(v: str | None) -> Optional[str]:
    """Normalize email with gmail dot removal and + alias handling"""
    if not v:
        return None
    v = v.strip().lower()
    if "@" not in v:
        return None
    local, domain = v.split("@", 1)
    # Remove + aliases
    local = local.split("+", 1)[0]
    # Gmail/googlemail dot removal
    if domain in {"gmail.com", "googlemail.com"}:
        local = local.replace(".", "")
    return f"{local}@{domain}"


def email_domain(v: str | None) -> Optional[str]:
    """Extract domain from email"""
    if not v or "@" not in v:
        return None
    return v.split("@", 1)[1].lower()


def norm_phone(v: str | None, region: str = "US") -> Optional[str]:
    """
    Normalize phone to digits-only format (not strict E.164).
    Returns digits only (e.g., "14155551234" for US numbers).

    Args:
        v: Raw phone string
        region: ISO country code for parsing hints

    Returns:
        Digits-only phone string (10-11 digits for US)
    """
    if not v:
        return None

    # Try to use the proper phone utils if available
    try:
        from ...preprocessing.rules.phone_utils import parse_phone

        parsed = parse_phone(v, region_hint=region)
        if parsed and parsed.get("e164"):
            # Return digits only (remove leading +)
            return parsed["e164"].lstrip("+")
    except ImportError:
        pass

    # Fallback to simple digit extraction
    digits = "".join(ch for ch in v if ch.isdigit())
    if len(digits) >= 10:
        # Keep last 10-11 digits (supports leading country code)
        normalized = digits[-11:] if len(digits) > 10 else digits[-10:]
        # Add US country code if 10 digits
        if len(normalized) == 10 and region == "US":
            return "1" + normalized
        return normalized
    return None


def _domain(v: str | None) -> str:
    v = (v or "").strip().lower()
    if not v:
        return ""
    return v.split("://", 1)[-1].split("/", 1)[0].lstrip("www.")


def same_linkedin(a: Optional[str], b: Optional[str]) -> bool:
    """Check if two LinkedIn URLs/handles are the same"""
    if not a or not b:
        return False

    def clean(x: str) -> str:
        x = x.strip().lower()
        x = x.replace("https://", "").replace("http://", "").replace("www.", "")
        x = (
            x.replace("linkedin.com/in/", "")
            .replace("linkedin.com/pub/", "")
            .replace("linkedin.com/company/", "")
        )
        return x.strip("/")

    return clean(a) == clean(b)


def is_mobile_like(rec: Dict[str, Any], num: Optional[str]) -> bool:
    """Heuristic: mobile field present OR 10-11 digits not obviously a main line/ext."""
    if not num:
        return False
    # Prefer MobilePhone field for Lead/Contact
    if rec.get("MobilePhone"):
        mobile_norm = norm_phone(rec.get("MobilePhone"))
        if mobile_norm and mobile_norm == num:
            return True
    # Reject if has extension notation
    phone_raw = rec.get("Phone") or ""
    if "x" in phone_raw.lower() or "ext" in phone_raw.lower():
        return False
    # Simple shape checks
    digits = "".join(ch for ch in num if ch.isdigit())
    if len(digits) < 10:
        return False
    # Patterns that suggest office phones (000, 111 endings often indicate main lines)
    if digits[-3:] in ("000", "111", "100"):
        return False
    return True


def phone_is_strong(
    rec_a: Dict[str, Any],
    rec_b: Dict[str, Any],
    pa: Optional[str],
    pb: Optional[str],
    phone_freq: Dict[str, int],
) -> bool:
    """Phone is a strong signal only if rare and mobile-like."""
    if not (pa and pb and pa == pb):
        return False
    # Rare (≤2 occurrences) + mobile-ish on at least one side
    if phone_freq.get(pa, 0) <= 2 and (
        is_mobile_like(rec_a, pa) or is_mobile_like(rec_b, pb)
    ):
        return True
    return False


def strong_identity(
    a: Dict[str, Any],
    b: Dict[str, Any],
    obj: ObjectType,
    phone_freq: Dict[str, int] = None,
) -> Tuple[bool, str]:
    """
    Check if two records share a strong identity anchor.
    Returns (has_strong_identity, signal_type)

    STRICT RULES:
    - Lead: Must have same email OR same phone (company alone is NOT enough)
    - Contact: Must have same email OR same phone OR (same AccountId + similar name)
    - Account: Must have same website/domain (not in free email list) OR registration number
    """

    if obj == "Lead":
        # Email is always a strong anchor
        ea = norm_email(a.get("Email"))
        eb = norm_email(b.get("Email"))
        if ea and eb and ea == eb:
            return True, "email"

        # Phone is strong only if rare and mobile-like
        pa = norm_phone(a.get("Phone") or a.get("MobilePhone"))
        pb = norm_phone(b.get("Phone") or b.get("MobilePhone"))
        if phone_freq and phone_is_strong(a, b, pa, pb, phone_freq):
            return True, "phone_strong"

        # LinkedIn match (if field exists)
        if "LinkedIn__c" in a and "LinkedIn__c" in b:
            if same_linkedin(a.get("LinkedIn__c"), b.get("LinkedIn__c")):
                return True, "linkedin"

        # External ID matches (if fields exist)
        for field in ("External_Id__c", "Vendor_Lead_Id__c"):
            if field in a and field in b:
                if a.get(field) and b.get(field) and a[field] == b[field]:
                    return True, "external_id"

        # NO company-only or weak phone matches for Leads!
        return False, ""

    elif obj == "Contact":
        # Email is always strong
        ea = norm_email(a.get("Email"))
        eb = norm_email(b.get("Email"))
        if ea and eb and ea == eb:
            return True, "email"

        # Phone is strong only if rare and mobile-like
        pa = norm_phone(a.get("Phone") or a.get("MobilePhone"))
        pb = norm_phone(b.get("Phone") or b.get("MobilePhone"))
        if phone_freq and phone_is_strong(a, b, pa, pb, phone_freq):
            return True, "phone_strong"

        # Same AccountId + very similar name (tighter threshold)
        if (
            a.get("AccountId")
            and b.get("AccountId")
            and a["AccountId"] == b["AccountId"]
        ):
            name_a = f"{a.get('FirstName', '')} {a.get('LastName', '')}".strip().lower()
            name_b = f"{b.get('FirstName', '')} {b.get('LastName', '')}".strip().lower()
            if (
                name_a
                and name_b
                and SequenceMatcher(None, name_a, name_b).ratio() >= 0.92
            ):
                return True, "account_name"

        # LinkedIn match (if field exists)
        if "LinkedIn__c" in a and "LinkedIn__c" in b:
            if same_linkedin(a.get("LinkedIn__c"), b.get("LinkedIn__c")):
                return True, "linkedin"

        # External ID match
        if "External_Id__c" in a and "External_Id__c" in b:
            if a.get("External_Id__c") and a["External_Id__c"] == b.get(
                "External_Id__c"
            ):
                return True, "external_id"

        return False, ""

    elif obj == "Account":
        # Domain + modest name similarity
        wa = _domain(a.get("Website"))
        wb = _domain(b.get("Website"))
        if wa and wb and wa == wb and wa not in COMMON_FREE:
            name_a = (a.get("Name") or "").lower()
            name_b = (b.get("Name") or "").lower()
            if SequenceMatcher(None, name_a, name_b).ratio() >= 0.70:
                return True, "website"

        # Registration ID match (strong anchor)
        for field in ("EIN__c", "ABN__c"):
            if field in a and field in b:
                if a.get(field) and b.get(field) and a[field] == b[field]:
                    return True, "registration"

        # Phone can help only with high name similarity
        if phone_freq:  # Only if we have frequency data
            pa = norm_phone(a.get("Phone"))
            pb = norm_phone(b.get("Phone"))
            if pa and pb and pa == pb:
                name_a = (a.get("Name") or "").lower()
                name_b = (b.get("Name") or "").lower()
                if SequenceMatcher(None, name_a, name_b).ratio() >= 0.80:
                    return True, "phone_name"

        return False, ""

    return False, ""


def hard_negative(
    a: Dict[str, Any], b: Dict[str, Any], obj: ObjectType, domain_freq: Dict[str, int]
) -> bool:
    """
    Check if two records should NOT be merged despite similarities.
    Enhanced to prevent company-only matches for Leads.
    """
    ea = norm_email(a.get("Email"))
    eb = norm_email(b.get("Email"))
    pa = norm_phone(a.get("Phone") or a.get("MobilePhone"))
    pb = norm_phone(b.get("Phone") or b.get("MobilePhone"))

    # Different emails AND different phones = not dupes (for Leads/Contacts)
    if obj in ("Lead", "Contact"):
        if ea and eb and ea != eb:
            # Different emails - probably different people
            if pa and pb and pa != pb:
                # Different phones too - definitely different people
                return True
            elif not pa and not pb:
                # No phones to compare, different emails = different people
                name_a = (
                    f"{a.get('FirstName', '')} {a.get('LastName', '')}".strip().lower()
                )
                name_b = (
                    f"{b.get('FirstName', '')} {b.get('LastName', '')}".strip().lower()
                )
                if SequenceMatcher(None, name_a, name_b).ratio() < 0.90:
                    return True

    # For Leads: Same company but different contact info = different people
    if obj == "Lead":
        company_a = (a.get("Company") or "").strip().lower()
        company_b = (b.get("Company") or "").strip().lower()
        if company_a and company_b and company_a == company_b:
            # Same company - need strong identity match
            if (ea and eb and ea != eb) or (pa and pb and pa != pb):
                # Different email or phone at same company = different people
                return True

    # Don't merge solely on common/massive domains
    da = email_domain(ea)
    db = email_domain(eb)
    if da and db and da == db:
        # If domain is free email or appears too frequently, need other signals
        if da in COMMON_FREE or domain_freq.get(da, 0) > 100:
            # Check if they have ANY other strong signal
            has_strong, signal = strong_identity(a, b, obj)
            if not has_strong or signal == "domain":
                return True

    return False


def _mk_exact_detail(tag: str, score: float = 1.0) -> Dict[str, Any]:
    # Unified detail entry (0..1 score)
    return {
        "source_column": tag,
        "reference_column": tag,
        "score": float(score),  # already 0..1
        "weight": 1.0,
        "algorithm": "Exact",
    }


def _norm01(x: float) -> float:
    try:
        x = float(x)
        return x / 100.0 if x > 1.0 else max(0.0, min(1.0, x))
    except Exception:
        return 0.0


def _normalize_details(details: Any) -> List[Dict[str, Any]]:
    """
    Normalize engine details to the unified schema and 0..1 scores.
    Accepts:
      - list[{ source_column|source_field, reference_column|ref_field, score (0..1 or 0..100), weight, algorithm }]
    """
    out: List[Dict[str, Any]] = []
    if not isinstance(details, list):
        return out
    for d in details:
        if not isinstance(d, dict):
            continue
        src = (
            d.get("source_column")
            or d.get("source_field")
            or d.get("source")
            or d.get("field")
            or "?"
        )
        ref = (
            d.get("reference_column")
            or d.get("ref_field")
            or d.get("reference")
            or d.get("reference_field")
            or src
        )
        alg = d.get("algorithm") or d.get("algo_used") or d.get("algo") or "Unknown"
        wt = float(d.get("weight", 1.0))
        sc = _norm01(d.get("score", 0.0))
        out.append(
            {
                "source_column": src,
                "reference_column": ref,
                "algorithm": alg,
                "weight": wt,
                "score": sc,
            }
        )
    return out


def _reasons_from_details(details: List[Dict[str, Any]]) -> str:
    """Compact reasons: { 'Email->Email': 100.0, ... } (values as % for display)."""
    if not details:
        return "{}"
    items: Dict[str, float] = {}
    for f in details:
        src = f.get("source_column") or "?"
        ref = f.get("reference_column") or src
        key = f"{src}->{ref}"  # ASCII arrow
        val = round(_norm01(f.get("score", 0.0)) * 100.0, 1)
        items[key] = max(items.get(key, 0.0), val)  # keep max per pair
    return json.dumps(items)


def _tier_from_score(score: float, thresholds: Dict[str, float] | None) -> str:
    """Use L2A rule: auto_link ⇒ CERTAIN; max(suggest,0.80) ⇒ LIKELY; else POSSIBLE."""
    suggest = _norm01((thresholds or {}).get("suggest", 0.60))
    auto = _norm01((thresholds or {}).get("auto_link", 0.95))
    if score >= auto:
        return "CERTAIN"
    if score >= max(suggest, 0.80):
        return "LIKELY"
    return "POSSIBLE"


def _convert_adaptive_blocking(blocking: dict, df: pd.DataFrame, obj: str) -> dict:
    """Convert adaptive blocking to concrete strategy based on object type and data"""
    import logging

    log = logging.getLogger(__name__)

    # If it's not adaptive or already has keys, return as-is
    if blocking.get("strategy") != "adaptive" or blocking.get("keys"):
        return blocking

    # Handle the case where Intelligence returns {'enabled': True, 'strategy': 'adaptive'}
    if blocking.get("enabled") or blocking.get("strategy") == "adaptive":
        log.info(f"Converting adaptive blocking strategy for {obj}")

        # Choose column based on object type
        column = None
        if obj == "Account" and "Name" in df.columns:
            column = "Name"
        elif obj == "Lead" and "Company" in df.columns:
            column = "Company"
        elif obj == "Contact" and "LastName" in df.columns:
            column = "LastName"
        else:
            # Find first text column with good coverage
            for col in df.columns:
                if col in ["Id", "id", "CreatedDate", "LastModifiedDate", "IsDeleted"]:
                    continue  # Skip system fields
                if df[col].notna().mean() > 0.8 and df[col].dtype == "object":
                    column = col
                    log.info(
                        f"Selected '{col}' for adaptive blocking (coverage: {df[col].notna().mean():.1%})"
                    )
                    break

        if column:
            log.info(f"Adaptive blocking converted to {column}-based blocking")
            return {
                "apply_blocking": True,
                "keys": [{"source_column": column, "key_length": 3}],
            }
        else:
            # No suitable column, disable blocking
            log.warning("No suitable column for adaptive blocking, disabling blocking")
            return {"apply_blocking": False, "keys": []}

    # Return original if not adaptive
    return blocking


def _normalize_intel_config(cfg, default_threshold: float = 0.75) -> dict:
    """
    Normalizes intelligence config to canonical dict shape.
    Returns dict with:
      - field_mappings: list[dict]
      - threshold: float
      - blocking: dict (apply_blocking: bool, keys: list[dict])
    """
    # Handle list case (just mappings)
    if isinstance(cfg, list):
        return {
            "field_mappings": cfg,
            "threshold": default_threshold,
            "blocking": {"apply_blocking": False, "keys": []},
        }

    # Handle dict case
    if not isinstance(cfg, dict):
        return {
            "field_mappings": [],
            "threshold": default_threshold,
            "blocking": {"apply_blocking": False, "keys": []},
        }

    # Normalize field mappings key
    mappings = cfg.get("field_mappings") or cfg.get("mappings") or []

    # Normalize mapping keys: algorithm -> preferred_algo
    normalized_mappings = []
    for m in mappings:
        nm = m.copy()
        # Normalize algorithm key
        if "algorithm" in nm and "preferred_algo" not in nm:
            nm["preferred_algo"] = nm.pop("algorithm")
        # Normalize column names
        if "source" in nm and "source_column" not in nm:
            nm["source_column"] = nm.pop("source")
        if "reference" in nm and "reference_column" not in nm:
            nm["reference_column"] = nm.pop("reference")
        normalized_mappings.append(nm)

    # Normalize threshold
    threshold = (
        cfg.get("threshold")
        or cfg.get("thresholds", {}).get("suggest")
        or default_threshold
    )
    threshold = float(threshold) if threshold else default_threshold

    # Normalize blocking
    blocking = cfg.get("blocking", {})
    if isinstance(blocking, dict):
        # Ensure keys is a list and normalize column -> source_column
        if "column" in blocking and "keys" not in blocking:
            blocking["keys"] = [{"source_column": blocking.pop("column")}]
        elif "keys" in blocking:
            normalized_keys = []
            for key in blocking.get("keys", []):
                if isinstance(key, dict):
                    nk = key.copy()
                    if "column" in nk and "source_column" not in nk:
                        nk["source_column"] = nk.pop("column")
                    normalized_keys.append(nk)
            blocking["keys"] = normalized_keys
        else:
            blocking["keys"] = []

        # Safety check: ensure source_column is never None
        if blocking.get("apply_blocking") and blocking.get("keys"):
            valid_keys = []
            for key in blocking["keys"]:
                if key.get("source_column") is not None:
                    valid_keys.append(key)

            if not valid_keys:
                # No valid keys found, disable blocking
                blocking["apply_blocking"] = False
                blocking["keys"] = []
            else:
                blocking["keys"] = valid_keys
    else:
        blocking = {"apply_blocking": False, "keys": []}

    return {
        "field_mappings": normalized_mappings,
        "threshold": threshold,
        "blocking": blocking,
    }


class IntelligentDedupeOrchestrator(BaseIntelligentService):
    """
    Phase-2 orchestrator: Intelligence -> Engine. Gracefully falls back to exact matching
    (email/phone/website) when intelligence/engine is unavailable.
    Enhanced with config normalization and schema-driven baseline.
    """

    def __init__(self, db: AsyncSession, account_id: str):
        super().__init__(db, account_id)
        self.persist = IntelligentDedupeService(db, account_id)

    def _schema_sig(self, df: pd.DataFrame, extra_flags: dict) -> str:
        """Create cache key from schema and flags"""
        schema = sorted(df.columns.tolist())

        # CRITICAL: Include registry version to invalidate cache when domain mappings update
        try:
            registry_version = get_registry()._last_load
        except:
            registry_version = 0

        payload = {"cols": schema, "registry_version": registry_version, **extra_flags}

        # Sort keys for consistent hashing
        return hashlib.md5(json.dumps(payload, sort_keys=True).encode()).hexdigest()[
            :16
        ]

    def _validate_mappings(self, df: pd.DataFrame, mappings: List[dict]) -> List[dict]:
        """Validate mappings have columns and coverage"""
        valid = []
        for m in mappings:
            src = m.get("source_column") or m.get("source")
            ref = m.get("reference_column") or m.get("reference") or src

            if src not in df.columns or ref not in df.columns:
                log.warning(f"Dropping mapping with missing column: {src}->{ref}")
                continue

            src_cov = df[src].notna().mean()
            ref_cov = df[ref].notna().mean()

            if src_cov < MIN_COVERAGE_MAPPING or ref_cov < MIN_COVERAGE_MAPPING:
                log.info(
                    f"Dropping mapping {src}->{ref} (coverage {src_cov:.1%}/{ref_cov:.1%})"
                )
                continue

            valid.append({**m, "source_column": src, "reference_column": ref})

        return valid

    def _validate_engine_config(self, cfg: dict) -> None:
        """Validate config before sending to engine - raises ValueError if invalid

        This is a safety net that will catch any bugs in the normalization logic itself.
        While normalization should have already fixed these issues, this ensures we
        never send invalid configs to the engine even if normalization has a bug.
        """
        # Check field names in mappings
        for mapping in cfg.get("field_mappings", []):
            if "algorithm" in mapping:
                raise ValueError(
                    f"Found 'algorithm' instead of 'preferred_algo' in mapping: {mapping}"
                )
            if "source" in mapping and "source_column" not in mapping:
                raise ValueError(
                    f"Found 'source' instead of 'source_column' in mapping: {mapping}"
                )
            if not mapping.get("preferred_algo"):
                raise ValueError(f"Missing preferred_algo in mapping: {mapping}")

        # Check blocking configuration
        blocking = cfg.get("blocking", {})
        if blocking.get("column"):
            raise ValueError("Found 'column' instead of 'source_column' in blocking")

        for key in blocking.get("keys", []):
            if isinstance(key, dict) and "column" in key:
                raise ValueError(f"Found 'column' in blocking key: {key}")

    def _local_baseline(self, df: pd.DataFrame, obj: ObjectType) -> dict:
        """Conservative fallback when Intelligence unavailable"""
        maps = []

        # Always have Name/Company
        if obj == "Lead" and "Company" in df.columns:
            maps.append(
                {
                    "source_column": "Company",
                    "reference_column": "Company",
                    "preferred_algo": "WRatio",
                    "weight": 1.0,
                }
            )
        elif obj == "Contact":
            if "LastName" in df.columns:
                maps.append(
                    {
                        "source_column": "LastName",
                        "reference_column": "LastName",
                        "preferred_algo": "JaroWinkler",
                        "weight": 0.7,
                    }
                )
        elif obj == "Account" and "Name" in df.columns:
            # Use multiple algorithms for Name to catch different variations
            maps.extend(
                [
                    {
                        "source_column": "Name",
                        "reference_column": "Name",
                        "preferred_algo": "WRatio",
                        "weight": 0.6,
                    },
                    {
                        "source_column": "Name",
                        "reference_column": "Name",
                        "preferred_algo": "TokenSetRatio",
                        "weight": 0.3,
                    },
                    {
                        "source_column": "Name",
                        "reference_column": "Name",
                        "preferred_algo": "JaroWinkler",
                        "weight": 0.2,
                    },
                ]
            )

            # Add Website if available for Account matching
            if "Website" in df.columns and df["Website"].notna().mean() >= 0.2:
                maps.append(
                    {
                        "source_column": "Website",
                        "reference_column": "Website",
                        "preferred_algo": "Domain",
                        "weight": 0.5,
                    }
                )

            # Add Phone if available
            if "Phone" in df.columns and df["Phone"].notna().mean() >= 0.2:
                maps.append(
                    {
                        "source_column": "Phone",
                        "reference_column": "Phone",
                        "preferred_algo": "Exact",
                        "weight": 0.4,
                    }
                )

            # Add BillingStreet if available
            if (
                "BillingStreet" in df.columns
                and df["BillingStreet"].notna().mean() >= 0.2
            ):
                maps.append(
                    {
                        "source_column": "BillingStreet",
                        "reference_column": "BillingStreet",
                        "preferred_algo": "JaroWinkler",
                        "weight": 0.3,
                    }
                )

        # Add email if available
        if "Email" in df.columns and df["Email"].notna().mean() >= MIN_COVERAGE_MAPPING:
            maps.append(
                {
                    "source_column": "Email",
                    "reference_column": "Email",
                    "preferred_algo": "Exact",
                    "weight": 1.0,
                }
            )

        # Add domain if good coverage
        if (
            "__website_domain" in df.columns
            and df["__website_domain"].notna().mean() >= MIN_COVERAGE_BLOCKING
        ):
            maps.append(
                {
                    "source_column": "__website_domain",
                    "reference_column": "__website_domain",
                    "preferred_algo": "Domain",
                    "weight": 0.8,
                }
            )
        elif (
            "__derived_domain" in df.columns
            and df["__derived_domain"].notna().mean() >= MIN_COVERAGE_BLOCKING
        ):
            maps.append(
                {
                    "source_column": "__derived_domain",
                    "reference_column": "__derived_domain",
                    "preferred_algo": "Domain",
                    "weight": 0.56,  # Reduced weight for derived
                }
            )

        # Blocking strategy
        blocking = {"apply_blocking": False}
        if (
            "__website_domain" in df.columns
            and df["__website_domain"].notna().mean() >= MIN_COVERAGE_BLOCKING
        ):
            blocking = {
                "apply_blocking": True,
                "keys": [{"source_column": "__website_domain", "key_length": 3}],
            }
        elif obj == "Lead" and "Company" in df.columns:
            blocking = {
                "apply_blocking": True,
                "keys": [{"source_column": "Company", "key_length": 3}],
            }
        elif obj == "Account" and "Name" in df.columns:
            blocking = {
                "apply_blocking": True,
                "keys": [{"source_column": "Name", "key_length": 3}],
            }
        else:
            # Final fallback - disable blocking rather than invalid config
            blocking = {"apply_blocking": False, "keys": []}

        # Lower threshold for baseline - be more aggressive in finding duplicates
        threshold = 0.65 if obj == "Account" else 0.70

        return {"field_mappings": maps, "threshold": threshold, "blocking": blocking}

    def _convert_adaptive_blocking(
        self, blocking: dict, df: pd.DataFrame, obj: str
    ) -> dict:
        """Convert adaptive blocking to concrete strategy based on object type and data"""
        # If it's not adaptive or already has keys, return as-is
        if blocking.get("strategy") != "adaptive" or blocking.get("keys"):
            return blocking

        # Handle the case where Intelligence returns {'enabled': True, 'strategy': 'adaptive'}
        if blocking.get("enabled") or blocking.get("strategy") == "adaptive":
            log.info(f"Converting adaptive blocking strategy for {obj}")

            # Choose column based on object type
            column = None
            if obj == "Account" and "Name" in df.columns:
                column = "Name"
            elif obj == "Lead" and "Company" in df.columns:
                column = "Company"
            elif obj == "Contact" and "LastName" in df.columns:
                column = "LastName"
            else:
                # Find first text column with good coverage
                for col in df.columns:
                    if col in [
                        "Id",
                        "id",
                        "CreatedDate",
                        "LastModifiedDate",
                        "IsDeleted",
                    ]:
                        continue  # Skip system fields
                    if df[col].notna().mean() > 0.8 and df[col].dtype == "object":
                        column = col
                        log.info(
                            f"Selected '{col}' for adaptive blocking (coverage: {df[col].notna().mean():.1%})"
                        )
                        break

            if column:
                log.info(f"Adaptive blocking converted to {column}-based blocking")
                return {
                    "apply_blocking": True,
                    "keys": [{"source_column": column, "key_length": 3}],
                }
            else:
                # No suitable column, disable blocking
                log.warning(
                    "No suitable column for adaptive blocking, disabling blocking"
                )
                return {"apply_blocking": False, "keys": []}

        # Return original if not adaptive
        return blocking

    async def get_available_fields(
        self,
        gateway: SalesforceGateway,
        obj: ObjectType,
        *,
        tenant_id: str | None = None,
    ) -> List[str]:
        """Get the list of fields available for this object type."""
        if hasattr(self, f"_cached_fields_{obj}"):
            return getattr(self, f"_cached_fields_{obj}")

        fields = list(CORE_FIELDS[obj])

        # Try to describe the object to see what custom fields exist
        try:
            if tenant_id is not None:
                try:
                    describe_result = await gateway.describe(tenant_id, obj)
                except TypeError:
                    describe_result = await gateway.describe(obj)
            else:
                describe_result = await gateway.describe(obj)
            available_field_names = {
                f["name"] for f in describe_result.get("fields", [])
            }

            # Add optional fields that actually exist
            for field in OPTIONAL_FIELDS.get(obj, []):
                if field in available_field_names:
                    fields.append(field)
        except Exception as e:
            log.debug(f"Could not describe {obj}, using core fields only: {e}")

        # Cache for this session
        setattr(self, f"_cached_fields_{obj}", fields)
        return fields

    async def fetch_records(
        self,
        gateway: SalesforceGateway,
        obj: ObjectType,
        *,
        limit: int | None = 5000,
        tenant_id: str | None = None,
    ) -> List[Dict[str, Any]]:
        fields = await self.get_available_fields(gateway, obj, tenant_id=tenant_id)
        soql = f"SELECT {', '.join(fields)} FROM {obj} WHERE {_where_clause(obj)}"
        if limit:
            soql += f" LIMIT {int(limit)}"
        if tenant_id is not None:
            try:
                data = await gateway.soql(tenant_id, soql)
            except TypeError:
                data = await gateway.soql(soql)
        else:
            data = await gateway.soql(soql)
        return data.get("records", [])

    def _as_dataframe(
        self, obj: ObjectType, recs: List[Dict[str, Any]]
    ) -> pd.DataFrame:
        if not recs:
            # Return empty DataFrame with whatever columns we have
            return pd.DataFrame(columns=["Id"])

        df = pd.DataFrame(recs)

        # Enhanced domain extraction and family enrichment
        from .domain_utils import extract_and_enrich_domains

        # Extract and enrich domains for all objects
        df, family_coverage = extract_and_enrich_domains(df)
        if family_coverage > 0:
            log.info(f"Domain family coverage for {obj}: {family_coverage:.1f}%")

        # Domain enhancement for dedupe
        feature_domain = os.getenv("FEATURE_DOMAIN_ENHANCED", "false").lower() == "true"

        if obj in ("Lead", "Contact"):
            if "Email" in df.columns:
                df["Email_norm"] = df["Email"].map(norm_email)
            if "Phone" in df.columns:
                df["Phone_norm"] = df["Phone"].map(norm_phone)
            if "MobilePhone" in df.columns:
                df["Mobile_norm"] = df["MobilePhone"].map(norm_phone)

            # Domain enhancement for Lead/Contact with Company field
            if feature_domain and "Company" in df.columns:
                try:
                    reg = get_registry()

                    def derive_company_domain(company):
                        if not company:
                            return None
                        try:
                            suggestions = reg.suggest_domains(company)
                            if (
                                suggestions
                                and suggestions[0][1] >= CONFIDENCE_THRESHOLD_ROUTING
                            ):
                                return suggestions[0][0]
                        except Exception:
                            pass
                        return None

                    df["__derived_domain"] = df["Company"].apply(derive_company_domain)
                    coverage = df["__derived_domain"].notna().mean()
                    if coverage >= 0.10:
                        log.info(
                            f"DomainEnhanced(Dedupe): {coverage:.1%} coverage for derived domains"
                        )
                except Exception as e:
                    log.warning(f"Domain enhancement failed for dedupe: {e}")

        else:  # Account
            if "Website" in df.columns:
                df["Website_domain"] = df["Website"].map(_domain)
            if "Phone" in df.columns:
                df["Phone_norm"] = df["Phone"].map(norm_phone)

            # Domain enhancement for Account with Name field
            if feature_domain and "Name" in df.columns:
                try:
                    reg = get_registry()

                    def derive_account_domain(name):
                        if not name:
                            return None
                        try:
                            suggestions = reg.suggest_domains(name)
                            if (
                                suggestions
                                and suggestions[0][1] >= CONFIDENCE_THRESHOLD_ROUTING
                            ):
                                return suggestions[0][0]
                        except Exception:
                            pass
                        return None

                    df["__derived_domain"] = df["Name"].apply(derive_account_domain)
                    coverage = df["__derived_domain"].notna().mean()
                    if coverage >= 0.10:
                        log.info(
                            f"DomainEnhanced(Dedupe-Account): {coverage:.1%} coverage for derived domains"
                        )
                except Exception as e:
                    log.warning(f"Domain enhancement failed for dedupe accounts: {e}")

        return df

    def _phase1_exact_pairs(
        self, obj: ObjectType, recs: List[Dict[str, Any]]
    ) -> List[Dict[str, Any]]:
        """Enhanced exact matching with strong identity gates and phone frequency analysis"""

        # Build frequency maps
        domain_freq: Dict[str, int] = defaultdict(int)
        phone_freq: Dict[str, int] = defaultdict(int)
        company_freq: Dict[str, int] = defaultdict(int)
        rec_by_id = {}

        # First pass: build frequency maps
        for r in recs:
            rec_by_id[r["Id"]] = r

            # Email domain frequency
            em = norm_email(r.get("Email"))
            if em:
                dom = email_domain(em)
                if dom:
                    domain_freq[dom] += 1

            # Phone frequency (both Phone and MobilePhone)
            ph = norm_phone(r.get("Phone") or r.get("MobilePhone"))
            if ph:
                phone_freq[ph] += 1

            # Company frequency for Leads
            if obj == "Lead":
                company = (r.get("Company") or "").strip().lower()
                if company:
                    company_freq[company] += 1

        # Build blocking groups - ONLY on strong identifiers
        groups: Dict[str, List[str]] = defaultdict(list)
        for r in recs:
            rid = r["Id"]

            if obj == "Lead":
                # Group by email, rare/mobile phones, LinkedIn, external IDs
                # NEVER group by company!
                em = norm_email(r.get("Email"))
                ph = norm_phone(r.get("Phone") or r.get("MobilePhone"))
                linkedin = (
                    r.get("LinkedIn__c", "").strip() if "LinkedIn__c" in r else ""
                )
                ext_id = (
                    r.get("External_Id__c", "").strip() if "External_Id__c" in r else ""
                )
                vendor_id = (
                    r.get("Vendor_Lead_Id__c", "").strip()
                    if "Vendor_Lead_Id__c" in r
                    else ""
                )

                if em:
                    groups[f"em:{em}"].append(rid)

                # Only group by phone if it's rare and mobile-like
                if ph and phone_freq.get(ph, 0) <= 2 and is_mobile_like(r, ph):
                    groups[f"ph:{ph}"].append(rid)

                if linkedin:
                    groups[f"li:{linkedin.lower()}"].append(rid)
                if ext_id:
                    groups[f"ext:{ext_id}"].append(rid)
                if vendor_id:
                    groups[f"ven:{vendor_id}"].append(rid)

            elif obj == "Contact":
                em = norm_email(r.get("Email"))
                ph = norm_phone(r.get("Phone") or r.get("MobilePhone"))
                linkedin = (
                    r.get("LinkedIn__c", "").strip() if "LinkedIn__c" in r else ""
                )
                ext_id = (
                    r.get("External_Id__c", "").strip() if "External_Id__c" in r else ""
                )
                account_id = r.get("AccountId")

                if em:
                    groups[f"em:{em}"].append(rid)

                # Only group by phone if it's rare and mobile-like
                if ph and phone_freq.get(ph, 0) <= 2 and is_mobile_like(r, ph):
                    groups[f"ph:{ph}"].append(rid)

                if linkedin:
                    groups[f"li:{linkedin.lower()}"].append(rid)
                if ext_id:
                    groups[f"ext:{ext_id}"].append(rid)

                # Group by AccountId + LastName for potential name variations
                if account_id and r.get("LastName"):
                    last_name = r["LastName"].strip().lower()
                    if last_name:
                        groups[f"acct_name:{account_id}:{last_name}"].append(rid)

            else:  # Account
                web = _domain(r.get("Website"))
                ph = norm_phone(r.get("Phone"))
                ein = r.get("EIN__c", "").strip() if "EIN__c" in r else ""
                abn = r.get("ABN__c", "").strip() if "ABN__c" in r else ""

                if web and web not in COMMON_FREE:
                    groups[f"web:{web}"].append(rid)
                if ph:
                    groups[f"ph:{ph}"].append(rid)
                if ein:
                    groups[f"ein:{ein}"].append(rid)
                if abn:
                    groups[f"abn:{abn}"].append(rid)

        pairs: List[Dict[str, Any]] = []
        seen_pairs = set()  # Avoid duplicates

        for k, ids in groups.items():
            if len(ids) < 2:
                continue

            # Determine signal type from key prefix
            if k.startswith("em:"):
                signal = "email"
            elif k.startswith("ph:"):
                signal = "phone"
            elif k.startswith("li:"):
                signal = "linkedin"
            elif k.startswith("ext:"):
                signal = "external_id"
            elif k.startswith("ven:"):
                signal = "vendor_id"
            elif k.startswith("web:"):
                signal = "website"
            elif k.startswith("ein:") or k.startswith("abn:"):
                signal = "registration"
            else:
                signal = "unknown"

            for i in range(len(ids)):
                for j in range(i + 1, len(ids)):
                    pair_key = tuple(sorted([ids[i], ids[j]]))
                    if pair_key in seen_pairs:
                        continue

                    rec_a = rec_by_id[ids[i]]
                    rec_b = rec_by_id[ids[j]]

                    # Apply strong identity gate with phone frequency
                    has_strong, sig_type = strong_identity(
                        rec_a, rec_b, obj, phone_freq
                    )
                    if not has_strong:
                        continue

                    # Apply hard negative filter
                    if hard_negative(rec_a, rec_b, obj, domain_freq, phone_freq):
                        continue

                    seen_pairs.add(pair_key)

                    # Cluster guardrail: Skip if signal is weak phone (frequent office number)
                    if (
                        signal == "phone"
                        and phone_freq
                        and phone_freq.get(k.split(":", 1)[1], 0) > 2
                    ):
                        log.debug(f"Skipping frequent phone cluster: {k}")
                        continue

                    # Build details with the actual matching signal
                    tag = (
                        sig_type.replace("_", " ").title()
                        if sig_type
                        else signal.replace("_", " ").title()
                    )
                    details = [_mk_exact_detail(tag, 1.0)]

                    # Determine if this is a true anchor
                    is_true_anchor = sig_type in (
                        "email",
                        "phone_strong",
                        "linkedin",
                        "external_id",
                        "website",
                        "registration",
                    )

                    # Add cluster guardrail info
                    cluster_guardrail = {}
                    if not is_true_anchor:
                        cluster_guardrail["cluster_guardrail_reason"] = "no_true_anchor"

                    pairs.append(
                        {
                            "rec_id_1": ids[
                                i
                            ],  # Changed from left_id to match DB schema
                            "rec_id_2": ids[
                                j
                            ],  # Changed from right_id to match DB schema
                            "score": 1.0,
                            "tier": "CERTAIN" if is_true_anchor else "LIKELY",
                            "reasons": _reasons_from_details(details),
                            "field_score_details": json.dumps(details),
                            "explanation": "",
                            "sig": sig_type,  # Tag with the signal type for filtering
                            **cluster_guardrail,
                        }
                    )

        return pairs

    async def run_on_salesforce(
        self,
        gateway: SalesforceGateway,
        obj: ObjectType,
        *,
        limit: int | None = 5000,
        mode: Literal["intelligent", "exact", "incremental", "rebuild"] = "intelligent",
        threshold_override: float | None = None,
        use_multi_algo: bool | None = None,
        save_integration=None,
        tenant_id: str | None = None,
    ) -> Tuple[int, int, Dict[str, Any]]:
        """
        Returns: (fetched_count, saved_suggestions, meta)
          meta = {
            "used": bool, "multi_algo_fields": int,
            "domain_verified_pct": int, "present_only_scoring": bool
          }
        """
        recs = await self.fetch_records(
            gateway, obj, limit=limit, tenant_id=tenant_id
        )
        if not recs:
            return (
                0,
                0,
                {
                    "used": False,
                    "multi_algo_fields": 0,
                    "domain_verified_pct": 0,
                    "present_only_scoring": True,
                },
            )

        # Handle different modes
        if mode in ["exact", "rebuild", "incremental"]:
            if mode == "exact" or mode in ["rebuild", "incremental"]:
                # For now, rebuild and incremental both use exact matching
                pairs = self._phase1_exact_pairs(obj, recs)
                saved = await self.persist.save_suggestions(obj, pairs)
                return (
                    len(recs),
                    saved,
                    {
                        "used": False,
                        "multi_algo_fields": 0,
                        "domain_verified_pct": 0,
                        "present_only_scoring": True,
                    },
                )

        # Build phone frequency for intelligent path too
        phone_freq: Dict[str, int] = defaultdict(int)
        for r in recs:
            ph = norm_phone(r.get("Phone") or r.get("MobilePhone"))
            if ph:
                phone_freq[ph] += 1

        # Timestamp for staging commit (use naive UTC to match DB schema)
        # Use local time to match database convention
        start_ts = datetime.now()
        saved = 0  # Initialize before try block
        staged = False  # Track whether we've actually saved staging records

        # Intelligent path with graceful fallback
        intelligence_mode = (
            "intelligence"  # Track which mode was used for observability
        )
        try:
            df = self._as_dataframe(obj, recs)

            # Create cache key
            feature_domain = (
                os.getenv("FEATURE_DOMAIN_ENHANCED", "false").lower() == "true"
            )
            flags = {
                "use_multi_algo": bool(use_multi_algo),
                "domain_enhanced": feature_domain,
                "object_type": obj,
            }
            cache_key = (self.account_id, "dedupe", self._schema_sig(df, flags))

            # Try cache first
            cached_cfg = _INTEL_CACHE.get(cache_key)
            cfg = cached_cfg
            was_cached = cfg is not None

            # Call Intelligence if not cached
            if cfg is None:
                try:
                    # Debug logging before sending to Intelligence
                    log.info(
                        f"Sending to Intelligence: columns={df.columns.tolist()}, rows={len(df)}"
                    )
                    null_counts = df.isnull().sum()
                    non_null_pcts = ((len(df) - null_counts) / len(df) * 100).round(1)
                    log.info(f"Field coverage: {non_null_pcts.to_dict()}")

                    cfg = await self.auto_configure(df, df, goal="dedupe")

                    if cfg and cfg.get("field_mappings"):
                        _INTEL_CACHE.set(cache_key, cfg)
                        log.info("Intelligence config cached for future use")
                        log.info(
                            f"Intelligence returned {len(cfg.get('field_mappings', []))} field mappings"
                        )
                        log.info(f"Intelligence blocking config: {cfg.get('blocking')}")

                        # Log details about blocking if present
                        if cfg.get("blocking", {}).get("apply_blocking"):
                            blocking_keys = cfg.get("blocking", {}).get("keys", [])
                            for idx, key in enumerate(blocking_keys):
                                col = key.get("source_column") or key.get("column")
                                log.info(
                                    f"  Blocking key {idx+1}: column='{col}', key_length={key.get('key_length', 'default')}"
                                )
                    else:
                        log.warning("Intelligence returned empty config")
                        cfg = None

                except Exception as e:
                    log.warning(f"Intelligence unavailable ({e}); using local baseline")
                    cfg = None
            else:
                log.info("Using cached Intelligence config")

            # Use local baseline if Intelligence failed
            if cfg is None:
                cfg = self._local_baseline(df, obj)
                intelligence_mode = "local_baseline"

            # Normalize config to handle any shape from intelligence
            cfg = _normalize_intel_config(cfg, default_threshold=0.75)

            # Convert adaptive blocking if needed
            if cfg.get("blocking") and isinstance(cfg["blocking"], dict):
                blocking = cfg["blocking"]
                if blocking.get("strategy") == "adaptive" and blocking.get("enabled"):
                    log.info("Converting adaptive blocking to concrete strategy")
                    cfg["blocking"] = self._convert_adaptive_blocking(blocking, df, obj)
                    log.info(f"Converted blocking: {cfg['blocking']}")

            # Remove system fields that hurt dedupe scoring
            system_fields = [
                "Id",
                "id",
                "CreatedDate",
                "LastModifiedDate",
                "IsDeleted",
                "attributes",
            ]
            original_mappings = cfg.get("field_mappings", [])
            cfg["field_mappings"] = [
                m
                for m in original_mappings
                if m.get("source") not in system_fields
                and m.get("reference") not in system_fields
            ]
            if len(original_mappings) != len(cfg["field_mappings"]):
                log.info(
                    f"Filtered system fields: {len(original_mappings)} -> {len(cfg['field_mappings'])} mappings"
                )
                filtered_fields = [
                    m.get("source")
                    for m in original_mappings
                    if m.get("source") in system_fields
                ]
                log.info(f"Removed fields: {filtered_fields}")

            # Override threshold for dedupe (temporary fix for testing)
            original_threshold = cfg.get("threshold", 0.8)
            if intelligence_mode == "intelligence" and obj == "Account":
                cfg["threshold"] = 0.65
                log.warning(
                    f"TEMPORARY: Lowering threshold from {original_threshold} to 0.65 for Account dedupe"
                )

            # Validate and filter mappings
            original_count = len(cfg.get("field_mappings", []))
            cfg["field_mappings"] = self._validate_mappings(
                df, cfg.get("field_mappings", [])
            )
            if original_count != len(cfg["field_mappings"]):
                log.info(
                    f"Validated mappings: {original_count} -> {len(cfg['field_mappings'])}"
                )

            # If no valid mappings after validation, fall back to baseline
            if not cfg.get("field_mappings"):
                log.warning("No valid mappings after validation; using local baseline")
                cfg = self._local_baseline(df, obj)
                cfg = _normalize_intel_config(cfg, default_threshold=0.70)
                intelligence_mode = "local_baseline_validation_fallback"

            # Expand mappings for ensemble if requested
            if use_multi_algo:
                expanded_mappings = []
                ensemble_algos = {
                    "WRatio": 0.60,
                    "JaroWinkler": 0.30,
                    "TokenSetRatio": 0.10,
                }

                for mapping in cfg.get("field_mappings", []):
                    algo = mapping.get("preferred_algo") or mapping.get(
                        "algorithm", "WRatio"
                    )
                    # Keep exact/domain algorithms as-is
                    if algo.lower() in ("exact", "domain"):
                        expanded_mappings.append(mapping)
                    else:
                        # Expand text algorithms to ensemble
                        base_weight = float(mapping.get("weight", 0.5))
                        for ensemble_algo, fraction in ensemble_algos.items():
                            expanded_mappings.append(
                                {
                                    **mapping,
                                    "preferred_algo": ensemble_algo,
                                    "weight": round(base_weight * fraction, 4),
                                }
                            )

                cfg["field_mappings"] = expanded_mappings
                log.info(
                    f"Expanded to {len(expanded_mappings)} ensemble mappings from {len(cfg.get('field_mappings', []))} base mappings"
                )

            # CRITICAL: Required logging for each dedupe run
            blocking_field = None
            if cfg.get("blocking", {}).get("apply_blocking"):
                keys = cfg.get("blocking", {}).get("keys", [])
                if keys and isinstance(keys[0], dict):
                    blocking_field = keys[0].get("source_column", "unknown")

            domain_coverage = 0.0
            if "__derived_domain" in df.columns:
                domain_coverage = df["__derived_domain"].notna().mean()
            elif "__website_domain" in df.columns:
                domain_coverage = df["__website_domain"].notna().mean()

            # Required telemetry log
            log.info(
                f"DEDUPE_RUN: mode={intelligence_mode} "
                f"maps={len(cfg.get('field_mappings', []))} "
                f"threshold={cfg.get('threshold', 0.75):.2f} "
                f"ensemble={'on' if use_multi_algo else 'off'} "
                f"blocking={blocking_field or 'none'} "
                f"coverage={domain_coverage:.1%} "
                f"cache={'hit' if was_cached else 'miss'} "
                f"rows={len(df)}"
            )

            # Non-breaking engine hints
            cfg["return_field_details"] = True
            cfg["explain_scores"] = True
            cfg["collect_reasons"] = True

            # Dedupe-specific engine threshold
            thr = cfg.get("threshold", 0.75)
            eff = min(float(os.getenv("DEDUPE_ENGINE_THRESHOLD", str(thr))), float(thr))
            cfg["threshold"] = eff
            log.info(_s(f"Dedupe engine threshold={eff:.2f} (cfg:{thr:.2f})"))

            # CRITICAL: Validate config before sending to engine
            try:
                self._validate_engine_config(cfg)
            except ValueError as e:
                log.error(f"Invalid engine config: {e}")
                # Fall back to safe baseline
                cfg = self._local_baseline(df, obj)
                cfg = _normalize_intel_config(cfg, default_threshold=0.70)
                intelligence_mode = "baseline_validation_error"

            # Diagnostic: Check for known duplicate patterns
            binance_records = df[
                df["Name"].str.contains("Binance", na=False, case=False)
            ]
            if len(binance_records) > 1:
                log.info(
                    f"Found {len(binance_records)} Binance records that should match:"
                )
                for idx, row in binance_records.iterrows():
                    log.info(
                        f"  - {row.get('Name', 'Unknown')}: ID={row.get('Id', 'no-id')}"
                    )

            # Check what blocking will do
            blocking_cfg = cfg.get("blocking", {})
            if blocking_cfg.get("apply_blocking"):
                blocking_col = None
                if blocking_cfg.get("keys"):
                    blocking_col = blocking_cfg["keys"][0].get(
                        "source_column"
                    ) or blocking_cfg["keys"][0].get("column")
                if blocking_col and blocking_col in df.columns:
                    # Check if Binance records would be in same block
                    if len(binance_records) > 1:
                        block_values = binance_records[blocking_col].tolist()
                        log.info(
                            f"Binance records' {blocking_col} values: {block_values}"
                        )
                        if (
                            len(set(str(v)[:3] for v in block_values if pd.notna(v)))
                            > 1
                        ):
                            log.warning(
                                f"WARNING: Binance records have different {blocking_col} prefixes - won't be compared!"
                            )

                            # TEMPORARY OVERRIDE: Force Name-based blocking for testing
                            if obj == "Account" and "Name" in df.columns:
                                log.warning(
                                    "OVERRIDE: Switching to Name-based blocking to ensure comparisons"
                                )
                                cfg["blocking"] = {
                                    "apply_blocking": True,
                                    "keys": [
                                        {"source_column": "Name", "key_length": 3}
                                    ],
                                }
                                # Re-check with new blocking
                                name_blocks = (
                                    binance_records["Name"].str[:3].str.upper().tolist()
                                )
                                log.info(
                                    f"With Name blocking, Binance prefixes: {name_blocks}"
                                )

            # DIAGNOSTIC: Log the exact field mappings being sent to engine
            log.warning(
                f"DIAGNOSTIC - Field mappings being sent to engine ({len(cfg.get('field_mappings', []))} total):"
            )
            for i, mapping in enumerate(
                cfg.get("field_mappings", [])[:5]
            ):  # Show first 5
                src = mapping.get("source") or mapping.get("source_column")
                ref = mapping.get("reference") or mapping.get("reference_column")
                algo = mapping.get("algorithm") or mapping.get("preferred_algo")
                weight = mapping.get("weight")
                log.warning(f"  [{i}] {src} -> {ref}: algo={algo}, weight={weight}")
            if len(cfg.get("field_mappings", [])) > 5:
                log.warning(
                    f"  ... and {len(cfg.get('field_mappings', [])) - 5} more mappings"
                )

            # DIAGNOSTIC: Test actual scoring algorithms
            if len(binance_records) > 1:
                try:
                    from ..intelligence.similarity import fuzz

                    name1 = binance_records.iloc[0]["Name"]
                    name2 = binance_records.iloc[1]["Name"]
                    scores = {
                        "token_set_ratio": fuzz.token_set_ratio(name1, name2) / 100.0,
                        "wratio": fuzz.WRatio(name1, name2) / 100.0,
                        "ratio": fuzz.ratio(name1, name2) / 100.0,
                        "partial_ratio": fuzz.partial_ratio(name1, name2) / 100.0,
                    }
                    log.warning(
                        f"DIAGNOSTIC - Direct scoring test: '{name1}' vs '{name2}'"
                    )
                    log.warning(f"  Scores: {scores}")
                    log.warning("  TokenSetRatio should be 1.0 for subset matching!")
                except Exception as e:
                    log.error(f"Failed to test scoring: {e}")

            # DIAGNOSTIC: Check blocking distribution
            if (
                blocking_cfg.get("apply_blocking")
                and blocking_col
                and blocking_col in df.columns
            ):
                # Add block keys to dataframe for analysis
                df["__block_key"] = (
                    df[blocking_col].fillna("").astype(str).str[:3].str.upper()
                )
                block_sizes = df.groupby("__block_key").size()
                multi_record_blocks = block_sizes[block_sizes > 1]
                log.info("DIAGNOSTIC - Blocking analysis:")
                log.info(f"  Total blocks: {len(block_sizes)}")
                log.info(f"  Blocks with >1 record: {len(multi_record_blocks)}")
                if len(multi_record_blocks) > 0:
                    log.info(
                        f"  Multi-record block sizes: {multi_record_blocks.tolist()[:10]}"
                    )  # First 10
                    # Check specific Binance block
                    if len(binance_records) > 1:
                        binance_block = binance_records.iloc[0]["Name"][:3].upper()
                        if binance_block in block_sizes.index:
                            log.info(
                                f"  Binance block '{binance_block}' has {block_sizes[binance_block]} records"
                            )

            thresholds = cfg.get("thresholds") or {"suggest": thr}
            engine_res = await self.dedupe_with_engine(df, cfg)

            # Use shared utility for score normalization
            if (
                hasattr(engine_res, "matches")
                and engine_res.matches is not None
                and not engine_res.matches.empty
            ):
                matches = engine_res.matches

                # Normalize score column using shared utility
                normalize_score_column(matches, inplace=True)

                if "score" not in matches.columns:
                    log.error(
                        "Could not identify or normalize score column in engine matches!"
                    )
                    log.error(f"Available columns: {matches.columns.tolist()}")

                # Update engine_res.matches with the modified DataFrame
                engine_res.matches = matches

                # Use shared utility for ID normalization
                normalize_id_columns(matches, mode="dedupe", inplace=True)

                # Update again with normalized columns
                engine_res.matches = matches

            # DIAGNOSTIC: Log engine response details
            if engine_res:
                log.warning("DIAGNOSTIC - Engine response received:")
                log.warning(f"  Has matches attr: {hasattr(engine_res, 'matches')}")
                if hasattr(engine_res, "matches"):
                    matches = engine_res.matches
                    if matches is not None:
                        log.warning(
                            f"  Matches shape: {matches.shape if hasattr(matches, 'shape') else 'N/A'}"
                        )
                        if not matches.empty:
                            log.warning(f"  Column names: {matches.columns.tolist()}")
                            log.warning(
                                f"  First few scores: {matches['score'].head().tolist() if 'score' in matches.columns else 'No score column'}"
                            )
                            # Check if Binance pairs exist
                            if (
                                "source_name" in matches.columns
                                and "reference_name" in matches.columns
                            ):
                                binance_matches = matches[
                                    (
                                        matches["source_name"].str.contains(
                                            "Binance", na=False, case=False
                                        )
                                    )
                                    | (
                                        matches["reference_name"].str.contains(
                                            "Binance", na=False, case=False
                                        )
                                    )
                                ]
                                if not binance_matches.empty:
                                    log.warning(
                                        f"  Found {len(binance_matches)} Binance-related matches:"
                                    )
                                    for _, row in binance_matches.head(3).iterrows():
                                        log.warning(
                                            f"    {row.get('source_name')} <-> {row.get('reference_name')}: score={row.get('score')}"
                                        )
                    else:
                        log.warning("  Matches is None")
                if hasattr(engine_res, "stats"):
                    log.warning(f"  Engine stats: {engine_res.stats}")

            pairs: List[Dict[str, Any]] = []
            dom_hits = 0

            # Check if we have results and extract matches DataFrame
            has_matches = False
            matches_df = pd.DataFrame()

            if engine_res:
                if hasattr(engine_res, "matches"):
                    matches_df = engine_res.matches
                    if matches_df is not None and not matches_df.empty:
                        has_matches = True

            # Diagnostic: If no matches found, try with lower threshold
            if not has_matches:
                current_threshold = cfg.get("threshold", 0.70)
                log.warning(f"No duplicates found at threshold {current_threshold}")
                log.info(f"Config used: {len(cfg.get('field_mappings', []))} mappings")

                # Try diagnostic run with very low threshold
                diagnostic_cfg = cfg.copy()
                diagnostic_cfg["threshold"] = 0.40
                log.info(
                    "Running diagnostic with threshold 0.40 to find potential matches"
                )

                try:
                    diagnostic_res = await self.dedupe_with_engine(df, diagnostic_cfg)
                    if diagnostic_res and hasattr(diagnostic_res, "matches"):
                        diag_matches = diagnostic_res.matches
                        if diag_matches is not None and not diag_matches.empty:
                            log.warning(
                                f"Found {len(diag_matches)} potential duplicates at threshold 0.40"
                            )
                            # Log a few examples
                            for idx, row in diag_matches.head(3).iterrows():
                                score = row.get("score", 0.0)
                                id1 = row.get("id1") or row.get("rec_id_1")
                                id2 = row.get("id2") or row.get("rec_id_2")
                                log.info(
                                    f"  Potential match: {id1} <-> {id2}, score={score:.3f}"
                                )
                            log.warning(
                                "Consider lowering your threshold if these look like real duplicates"
                            )
                        else:
                            log.info(
                                "No duplicates found even at threshold 0.40 - records appear genuinely unique"
                            )
                    else:
                        log.info("Diagnostic run returned no results")
                except Exception as e:
                    log.error(f"Diagnostic run failed: {e}")

            for _, s in matches_df.iterrows():
                # Use shared utility to extract IDs
                id1, id2 = extract_id_from_dict(s.to_dict(), mode="dedupe")
                if not id1 or not id2 or id1 == id2:
                    continue
                score = float(s.get("score", 0.0))
                if threshold_override is not None and score < float(threshold_override):
                    continue
                # Details might be stored as a JSON string or list
                details_raw = s.get("details") or s.get("field_score_details") or []
                if isinstance(details_raw, str):
                    try:
                        details_raw = json.loads(details_raw)
                    except:
                        details_raw = []
                details = _normalize_details(details_raw)
                details_json = json.dumps(details)
                # Check for true anchors in details
                has_true_anchor = any(
                    d.get("algorithm", "").lower() in ("exact", "domain")
                    and _norm01(d.get("score", 0)) >= 0.95
                    and d.get("is_anchor", False)
                    for d in details
                )

                # simple domain-verified heuristic for meta
                if any(
                    (d.get("algorithm") or "").lower().find("domain") >= 0
                    and _norm01(d.get("score")) >= 0.9
                    for d in details
                ):
                    dom_hits += 1

                # Determine tier with guardrails
                base_tier = _tier_from_score(score, thresholds)
                if base_tier == "CERTAIN" and not has_true_anchor:
                    # Downgrade to LIKELY if no true anchor
                    tier = "LIKELY"
                    cluster_guardrail = {"cluster_guardrail_reason": "no_true_anchor"}
                else:
                    tier = base_tier
                    cluster_guardrail = {}

                pairs.append(
                    {
                        "rec_id_1": id1,  # Changed from left_id to match DB schema
                        "rec_id_2": id2,  # Changed from right_id to match DB schema
                        "score": score,
                        "tier": tier,
                        "reasons": _reasons_from_details(details),
                        "field_score_details": details_json,
                        "explanation": s.get("explanation", ""),
                        **cluster_guardrail,
                    }
                )

            # 1) Save as staging with consistent timestamp
            saved = await self.persist.save_suggestions(
                obj, pairs, status="staging", created_at=start_ts
            )
            staged = True  # Mark that we've saved staging records

            # ... rest of the run work (telemetry, etc) ...

            # ensemble meta
            used = os.getenv("ENSEMBLE_SIGNALS_ENABLED", "true").lower() in (
                "1",
                "true",
                "yes",
                "y",
                "on",
            )
            mfields = 0
            try:
                # count mappings that appear to be multi-algo if cfg encodes lists
                mfields = sum(
                    1
                    for m in (cfg.get("mappings") or [])
                    if isinstance(m.get("algorithm"), list)
                )
            except Exception:
                pass
            dom_pct = int(round(100.0 * (dom_hits / max(1, len(pairs)))))

            # Count tiers and guardrails
            certain_count = sum(1 for p in pairs if p.get("tier") == "CERTAIN")
            guardrail_count = sum(1 for p in pairs if p.get("cluster_guardrail_reason"))

            # Build telemetry
            # Safely get blocking column name for telemetry
            blocking_keys = cfg.get("blocking", {}).get("keys", [])
            blocking_col = "none"
            if blocking_keys and len(blocking_keys) > 0:
                # Try both 'source_column' and 'column' keys
                blocking_col = blocking_keys[0].get("source_column") or blocking_keys[
                    0
                ].get("column", "none")

            telemetry = {
                "object": obj,
                "rows": len(recs),
                "blocking": blocking_col,
                "thr": cfg.get("threshold", 0.75),
                "kept_mappings": [
                    m.get("source", "?") for m in cfg.get("field_mappings", [])
                ],
                "dropped_mappings": [],  # Would need to track during validation
                "pairs": len(pairs),
                "clusters": -1,  # Would need proper connected components
                "certain": certain_count,
                "guardrail_no_anchor": guardrail_count,
            }

            # Log telemetry
            log.info(_s(f"Dedupe telemetry: {json.dumps(telemetry)}"))

            # 2) Flip to pending ONLY for rows created in this run
            # Format timestamp as string to match database format
            start_ts_str = start_ts.isoformat(sep=" ", timespec="seconds")
            flipped = await self.db.execute(
                text("""
                UPDATE dedupe_suggestions
                   SET status = 'pending'
                 WHERE account_id = :aid
                   AND object_type = :obj
                   AND status = 'staging'
                   AND created_at >= :start_ts
            """),
                {"aid": str(self.account_id), "obj": obj, "start_ts": start_ts_str},
            )
            await self.db.commit()
            log.info(
                _s(
                    f"Staging->Pending flipped={flipped.rowcount} account={self.account_id} object={obj}"
                )
            )

            meta = {
                "used": bool(used) and len(pairs) > 0,
                "multi_algo_fields": int(mfields),
                "domain_verified_pct": int(dom_pct),
                "present_only_scoring": True,
                "telemetry": telemetry,
                "intelligence_mode": intelligence_mode,  # Track brain vs fallback for observability
            }
            return len(recs), saved, meta

        except Exception:
            # Only clean up staging if we actually saved staging records
            if staged:
                log.exception(_s("Run failed after staging; cleaning staging records"))
                await self.db.execute(
                    text("""
                    DELETE FROM dedupe_suggestions
                     WHERE account_id = :aid
                       AND object_type = :obj
                       AND status = 'staging'
                       AND created_at >= :start_ts
                """),
                    {"aid": str(self.account_id), "obj": obj, "start_ts": start_ts},
                )
                await self.db.commit()
            else:
                log.exception(_s("Run failed before staging; no cleanup needed"))
            raise

    def _has_values(self, df: pd.DataFrame, col: str) -> bool:
        """Check if column exists and has non-empty values"""
        if col not in df.columns:
            return False
        non_empty = df[col].dropna()
        if len(non_empty) == 0:
            return False
        # Check for non-blank strings
        if df[col].dtype == "object":
            non_blank = non_empty[non_empty.astype(str).str.strip() != ""]
            return len(non_blank) > 0
        return True

    def _build_dynamic_baseline(
        self, df: pd.DataFrame, df2: pd.DataFrame, obj: ObjectType
    ) -> dict:
        """
        Schema-agnostic baseline builder that discovers fields from actual data.
        For dedupe, df and df2 are the same.
        """
        mappings = []
        blocking = {"apply_blocking": False, "keys": []}
        threshold = 0.70

        # Normalize object type once
        kind = str(obj).split(".")[-1].lower()

        # 1. Discover email/domain fields
        email_col = None
        domain_col = None
        phone_col = None

        for col in df.columns:
            if not email_col and (
                "email" in col.lower()
                or df[col].astype(str).str.contains("@", na=False).sum() > len(df) * 0.1
            ):
                email_col = col
            if not domain_col and any(
                x in col.lower() for x in ["website", "domain", "url"]
            ):
                domain_col = col
            if not phone_col and "phone" in col.lower():
                phone_col = col

        # Create derived columns
        if email_col and self._has_values(df, email_col):
            df["__email_lc"] = df[email_col].astype(str).str.lower()
            df["__email_domain"] = df[email_col].apply(email_domain).fillna("")

        if domain_col and self._has_values(df, domain_col):
            df["__website_domain"] = df[domain_col].apply(_domain).fillna("")

        if phone_col and self._has_values(df, phone_col):
            # Get region from environment or default to US
            region = os.getenv("DEDUPE_PHONE_REGION", "US")
            df["__phone_e164"] = (
                df[phone_col].apply(lambda x: norm_phone(x, region=region)).fillna("")
            )

        # 2. Discover best text columns for name matching
        text_cols = []
        for col in df.columns:
            if col in [
                "Id",
                "CreatedDate",
                "ModifiedDate",
                "IsDeleted",
                "IsConverted",
            ] or col.startswith("__"):
                continue
            sample = df[col].dropna().astype(str).head(100)
            if len(sample) == 0:
                continue

            # Score based on fill rate, alphabetic content, reasonable length
            fill_rate = len(df[col].dropna()) / max(1, len(df))
            alpha_ratio = sum(any(c.isalpha() for c in str(v)) for v in sample) / max(
                1, len(sample)
            )
            avg_len = sample.str.len().mean()
            is_reasonable_len = 3 <= avg_len <= 100

            score = (
                fill_rate * 0.55
                + alpha_ratio * 0.35
                + (0.1 if is_reasonable_len else 0)
            )
            text_cols.append((col, score))

        # Pick best text column
        text_cols.sort(key=lambda x: x[1], reverse=True)
        best_text_col = text_cols[0][0] if text_cols else None

        # 3. Build mappings based on object type

        # Add DomainFamily mapping first if available (highest priority for all objects)
        if "DomainFamily" in df.columns and self._has_values(df, "DomainFamily"):
            mappings.append(
                {
                    "source": "DomainFamily",
                    "reference": "DomainFamily",
                    "algorithm": "Exact",
                    "weight": 1.0,
                    "is_anchor": True,
                    "is_domain_family": True,
                }
            )
            log.info(f"Added DomainFamily exact matching for {obj} dedupe")

        if kind == "lead":
            # For Leads, prioritize Company and Email
            if "Company" in df.columns and self._has_values(df, "Company"):
                mappings.append(
                    {
                        "source": "Company",
                        "reference": "Company",
                        "algorithm": "WRatio",
                        "weight": 0.8,
                    }
                )
            if email_col and self._has_values(df, email_col):
                mappings.append(
                    {
                        "source": email_col,
                        "reference": email_col,
                        "algorithm": "Exact",
                        "weight": 1.0,
                        "is_anchor": True,
                    }
                )
            if "__phone_e164" in df and self._has_values(df, "__phone_e164"):
                mappings.append(
                    {
                        "source": "__phone_e164",
                        "reference": "__phone_e164",
                        "algorithm": "Exact",
                        "weight": 0.6,
                    }
                )

        elif kind == "contact":
            # For Contacts, use name fields + email/phone anchors
            if "LastName" in df.columns and self._has_values(df, "LastName"):
                mappings.append(
                    {
                        "source": "LastName",
                        "reference": "LastName",
                        "algorithm": "JaroWinkler",
                        "weight": 0.7,
                    }
                )
            if "FirstName" in df.columns and self._has_values(df, "FirstName"):
                mappings.append(
                    {
                        "source": "FirstName",
                        "reference": "FirstName",
                        "algorithm": "JaroWinkler",
                        "weight": 0.5,
                    }
                )
            # CRITICAL: Add email/phone anchors for Contacts
            if email_col and self._has_values(df, email_col):
                mappings.append(
                    {
                        "source": email_col,
                        "reference": email_col,
                        "algorithm": "Exact",
                        "weight": 1.0,
                        "is_anchor": True,
                    }
                )
            if "__phone_e164" in df and self._has_values(df, "__phone_e164"):
                mappings.append(
                    {
                        "source": "__phone_e164",
                        "reference": "__phone_e164",
                        "algorithm": "Exact",
                        "weight": 0.6,
                    }
                )

        elif kind == "account":
            if "Name" in df.columns and self._has_values(df, "Name"):
                mappings.append(
                    {
                        "source": "Name",
                        "reference": "Name",
                        "algorithm": "WRatio",
                        "weight": 0.8,
                    }
                )
            if "__website_domain" in df and self._has_values(df, "__website_domain"):
                mappings.append(
                    {
                        "source": "__website_domain",
                        "reference": "__website_domain",
                        "algorithm": "Domain",
                        "weight": 0.9,
                        "is_anchor": True,
                    }
                )

        # Drop mappings with missing/empty columns
        usable = []
        for m in mappings:
            src = m.get("source")
            if self._has_values(df, src):
                usable.append(m)
            else:
                log.warning(_s(f"Dropping mapping with missing/empty column: {src}"))
        mappings = usable

        # Fallback if no mappings found
        if not mappings and best_text_col:
            mappings.append(
                {
                    "source": best_text_col,
                    "reference": best_text_col,
                    "algorithm": "WRatio",
                    "weight": 0.8,
                }
            )

        # 4. Dedupe-specific blocking (same-object repetition)
        if kind in ("lead", "contact") and "__email_lc" in df:
            # Check for repeated emails
            email_counts = df["__email_lc"].value_counts()
            rep = (email_counts > 1).sum()

            # Check if personal emails would explode buckets
            if "__email_domain" in df:
                personal_domains = {
                    "gmail.com",
                    "googlemail.com",
                    "yahoo.com",
                    "outlook.com",
                    "hotmail.com",
                    "icloud.com",
                    "aol.com",
                    "mail.com",
                    "protonmail.com",
                }
                domain_counts = df["__email_domain"].value_counts()

                # Check if any personal domain has too many records
                max_bucket_pct = float(os.getenv("DEDUPE_MAX_BUCKET_PCT", "0.05"))
                personal_exploded = any(
                    domain in personal_domains and count > len(df) * max_bucket_pct
                    for domain, count in domain_counts.items()
                )

                if personal_exploded:
                    log.warning(
                        _s(
                            "Personal email domains too large for blocking, using domain blocking instead"
                        )
                    )
                    # Use domain blocking but with lower threshold
                    rep = (domain_counts > 1).sum()
                    if rep > 0:
                        log.info(
                            _s(
                                f"Dynamic baseline: using domain blocking (repeats={rep})"
                            )
                        )
                        blocking = {
                            "apply_blocking": True,
                            "keys": [
                                {
                                    "column": "__email_domain",
                                    "variant": "raw",
                                    "key_length": 0,
                                }
                            ],
                        }
                        threshold = 0.70  # Lower threshold for domain blocking

            if not blocking.get("apply_blocking") and rep > 0:
                log.info(_s(f"Dynamic baseline: using email blocking (repeats={rep})"))
                blocking = {
                    "apply_blocking": True,
                    "keys": [
                        {"column": "__email_lc", "variant": "raw", "key_length": 0}
                    ],
                }
                threshold = 0.72

        elif kind == "account" and "__website_domain" in df:
            # Check for repeated domains
            rep = df["__website_domain"].value_counts().gt(1).sum()
            if rep > 0:
                log.info(_s(f"Dynamic baseline: using domain blocking (repeats={rep})"))
                blocking = {
                    "apply_blocking": True,
                    "keys": [
                        {
                            "column": "__website_domain",
                            "variant": "alnum",
                            "key_length": 3,
                        }
                    ],
                }
                threshold = 0.72

        # Name prefix fallback with auto-widen
        if not blocking["apply_blocking"] and best_text_col:
            # Try 8-char prefix
            df["__block_name"] = df[best_text_col].astype(str).str[:8].str.lower()
            rep = df["__block_name"].value_counts().gt(1).sum()

            # Prevent pathological buckets
            max_bucket_pct = float(os.getenv("DEDUPE_MAX_BUCKET_PCT", "0.05"))
            max_bucket_size = df["__block_name"].value_counts().max()
            if max_bucket_size > len(df) * max_bucket_pct:
                log.warning(
                    _s(
                        f"Name prefix bucket too large ({max_bucket_size}/{len(df)}), widening to 4 chars"
                    )
                )
                df["__block_name"] = df[best_text_col].astype(str).str[:4].str.lower()
                rep = df["__block_name"].value_counts().gt(1).sum()

            if rep > 0:
                log.info(
                    _s(
                        f"Dynamic baseline: using name prefix blocking k=8 (repeats={rep})"
                    )
                )
                blocking = {
                    "apply_blocking": True,
                    "keys": [
                        {"column": "__block_name", "variant": "alnum", "key_length": 8}
                    ],
                }
                threshold = 0.72
            else:
                # Widen to 4 chars
                df["__block_name"] = df[best_text_col].astype(str).str[:4].str.lower()
                rep = df["__block_name"].value_counts().gt(1).sum()
                if rep > 0:
                    log.info(
                        _s(
                            f"Dynamic baseline: using name prefix blocking k=4 (repeats={rep})"
                        )
                    )
                    blocking = {
                        "apply_blocking": True,
                        "keys": [
                            {
                                "column": "__block_name",
                                "variant": "alnum",
                                "key_length": 4,
                            }
                        ],
                    }
                    threshold = 0.72
                else:
                    log.info(
                        _s(
                            "Dynamic baseline: no effective blocking found, using lower threshold"
                        )
                    )
                    threshold = 0.66

        # Engine payload check
        payload_check = []
        for m in mappings:
            src = m.get("source")
            non_empty = (
                int(df[src].dropna().astype(str).str.strip().ne("").sum())
                if src in df.columns
                else 0
            )
            payload_check.append(
                {
                    "src": src,
                    "src_nonempty": non_empty,
                    "algo": m.get("algorithm", "Unknown"),
                }
            )
        log.info(_s(f"Engine payload check: {payload_check}"))

        log.info(
            _s(
                f"Dynamic baseline: mappings={len(mappings)} blocking={blocking} thr={threshold}"
            )
        )

        return {
            "field_mappings": mappings,
            "threshold": threshold,
            "blocking": blocking,
        }
