"""
Intelligent L2A Service - Uses BaseIntelligentService
======================================================
This replaces the old L2AMatcher with a fully compliant version
that ALWAYS uses intelligence APIs for configuration.
"""

from typing import List, Dict, Any, Optional
import pandas as pd
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from datetime import datetime
import logging
import os
import time
from pydantic import BaseModel
from enum import Enum
from dataclasses import is_dataclass, asdict

from .base_intelligent_service import BaseIntelligentService
from ..model_defs.l2a_models import L2APolicy, L2ASuggestion
from ...preprocessing.rules.domain_utils import registrable_domain
from ..models import Integration, IntegrationProvider
from .rate_limiter import SmartRateLimiter, RateLimitConfig
from .salesforce_gateway import SalesforceGateway
from ..utils.logging_utils import _s
from .. import settings

# Import shared normalization utilities
from .utils.match_normalization import (
    extract_id_from_dict,
)

logger = logging.getLogger(__name__)


# Metrics tracking for domain lookups
def _track_domain_lookup(domain: str, found: bool, context: str = "l2a"):
    """Track domain lookup for coverage metrics"""
    try:
        # Only track if domain families DB is available
        import duckdb
        import os

        db_path = os.getenv("DOMAIN_DB_PATH", "config/foundrygraph.duckdb")
        if os.path.exists(db_path):
            conn = duckdb.connect(db_path, read_only=False)
            # Create metrics table if not exists
            conn.execute("""
                CREATE TABLE IF NOT EXISTS lookup_metrics (
                    domain TEXT,
                    found BOOLEAN,
                    service TEXT,
                    context TEXT,
                    timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                )
            """)
            conn.execute(
                "INSERT INTO lookup_metrics (domain, found, service, context) VALUES (?, ?, 'l2a', ?)",
                [domain, found, context],
            )
            conn.close()
    except Exception:
        pass  # Silently fail - metrics are non-critical


# Writeback configuration models
class OverwriteMode(str, Enum):
    if_empty = "if_empty"
    always = "always"
    never = "never"


class WritebackCandidate(BaseModel):
    apiName: str
    label: str
    updateable: bool = True
    accessible: bool = True
    reason: Optional[str] = None


# Feature flags for compliance vs override modes
ALLOW_SERVICE_OVERRIDES = (
    os.getenv("ALLOW_SERVICE_OVERRIDES", "false").lower() == "true"
)
STRICT_INTELLIGENCE_COMPLIANCE = (
    os.getenv("STRICT_INTELLIGENCE_COMPLIANCE", "false").lower() == "true"
)

# --- B2C Domain Detection using Desktop's battle-tested infrastructure ---
try:
    from fmatch.core.b2c_bloom_filter import get_b2c_checker, is_b2c_domain as check_b2c

    _b2c_checker = None
    _b2c_load_time = None
    _b2c_domain_count = None
    _b2c_filter_version = None
    _b2c_bloom_bits = None
    _b2c_bloom_hashes = None

    def _init_b2c_checker():
        """Lazy-load B2C checker once per process"""
        global \
            _b2c_checker, \
            _b2c_load_time, \
            _b2c_domain_count, \
            _b2c_filter_version, \
            _b2c_bloom_bits, \
            _b2c_bloom_hashes
        if _b2c_checker is None:
            start_t = time.time()
            _b2c_checker = get_b2c_checker()
            _b2c_load_time = time.time() - start_t

            # Extract actual domain count from the internal filter or domain set
            if hasattr(_b2c_checker, "_filter") and _b2c_checker._filter:
                # Bloom filter path - get actual count from filter
                _b2c_domain_count = getattr(_b2c_checker._filter, "item_count", 0)
                _b2c_bloom_bits = getattr(_b2c_checker._filter, "size", 0)
                _b2c_bloom_hashes = getattr(_b2c_checker._filter, "hash_count", 0)
            elif hasattr(_b2c_checker, "_domain_set") and _b2c_checker._domain_set:
                # Set-based path - count domains directly
                _b2c_domain_count = len(_b2c_checker._domain_set)
                _b2c_bloom_bits = 0  # Not using bloom
                _b2c_bloom_hashes = 0
            else:
                # Fallback - try other attributes
                _b2c_domain_count = getattr(
                    _b2c_checker,
                    "domain_count",
                    len(getattr(_b2c_checker, "domains", [])),
                )
                _b2c_bloom_bits = getattr(
                    getattr(_b2c_checker, "bloom_filter", None), "size", None
                ) or getattr(_b2c_checker, "bloom_size_bits", 0)
                _b2c_bloom_hashes = getattr(
                    getattr(_b2c_checker, "bloom_filter", None), "hash_count", None
                ) or getattr(_b2c_checker, "hash_count", 0)

            # Dynamically extract version from resource path or checker attributes
            import re as _re

            _b2c_filter_version = getattr(_b2c_checker, "version", None)
            _b2c_resource_path = getattr(_b2c_checker, "resource_path", None)
            if not _b2c_filter_version and _b2c_resource_path:
                m = _re.search(r"b2c_domains_v(\d+)\.txt$", str(_b2c_resource_path))
                _b2c_filter_version = f"v{m.group(1)}" if m else "unknown"
            if not _b2c_filter_version:
                _b2c_filter_version = "v20250719"  # fallback if can't detect

            logger.info(
                "B2C Bloom filter loaded: %s, %d domains, load_time=%.3fs, "
                "size=%d bits (%.1f KB), k=%d hashes",
                _b2c_filter_version,
                _b2c_domain_count,
                _b2c_load_time,
                _b2c_bloom_bits,
                _b2c_bloom_bits / 8 / 1024 if _b2c_bloom_bits else 0,
                _b2c_bloom_hashes,
            )
        return _b2c_checker

    def is_b2c_domain(domain: str) -> bool:
        """Check if domain is B2C using desktop's infrastructure"""
        checker = _init_b2c_checker()
        return check_b2c(domain)

    B2C_AVAILABLE = True
    logger.info("Desktop B2C infrastructure available for L2A service")

except ImportError as e:
    logger.warning(
        "Desktop B2C infrastructure not available: %s. Falling back to simple set.", e
    )
    B2C_AVAILABLE = False

    # Fallback to simple set if desktop B2C not available
    COMMON_B2C = {
        "gmail.com",
        "yahoo.com",
        "hotmail.com",
        "outlook.com",
        "icloud.com",
        "aol.com",
        "proton.me",
        "protonmail.com",
        "live.com",
        "msn.com",
        "ymail.com",
        "mail.com",
    }

    def is_b2c_domain(domain: str) -> bool:
        """Fallback B2C check using simple set"""
        if not domain:
            return False
        domain = domain.lower().strip()
        return domain in COMMON_B2C


# --- Ephemeral block-key helpers (Option 2) ---
import re

ALNUM = re.compile(r"[^a-z0-9]")

# Use global registrable_domain utility (imported above)
_registrable_domain = registrable_domain


def _blockkey3(
    s: str, key_length: int = 3, filter_b2c: bool = True, b2c_check_fn=None
) -> str:
    """Generate blockkey with optional B2C filtering and configurable length"""
    # Use ASCII punycode for blocking keys to handle IDN domains properly
    d = _registrable_domain(s, key_ascii=True)
    if filter_b2c:
        # For B2C check, use Unicode version to match against free domain lists
        d_unicode = _registrable_domain(s, key_ascii=False)
        # Use custom B2C check function if provided (for whitelist support)
        if b2c_check_fn and b2c_check_fn(d_unicode):
            return ""  # skip B2C buckets
        elif not b2c_check_fn and is_b2c_domain(d_unicode):
            return ""  # skip B2C buckets using default
    key = ALNUM.sub("", d.lower())[:key_length]
    if not key:
        return ""  # Skip empty keys after ALNUM processing
    return key


def _first_nonempty(row, cols):
    """Get first non-empty value from columns"""
    for c in cols:
        if c in row and isinstance(row[c], str) and row[c].strip():
            return row[c]
    return ""


# Alias for compatibility
_pick = _first_nonempty
# ----------------------------------------------


class IntelligentL2AService(BaseIntelligentService):
    """
    L2A Service that ALWAYS uses intelligence APIs.
    No hardcoded field mappings, thresholds, or algorithms!
    """

    def _run_diagnostic_fuzzy(self, exact_suggestions, accounts, goal):
        """Run diagnostic fuzzy matching (QA only, does not affect decisions)"""
        if not settings.RUN_DIAGNOSTIC_FUZZY or not exact_suggestions:
            return []

        # Fix: exact_suggestions is a list of dicts, not objects with .lead attribute
        sample_lead_ids = []
        for suggestion in exact_suggestions[: settings.DIAGNOSTIC_FUZZY_SAMPLE]:
            if isinstance(suggestion, dict):
                # Extract lead_id from suggestion dictionary
                lead_id = suggestion.get("lead_id")
                if lead_id:
                    sample_lead_ids.append(lead_id)

        try:
            # Note: This is a stub for diagnostic fuzzy matching
            # In production, this would process the sampled lead IDs
            diag = []  # Placeholder for actual fuzzy matching
            logger.info(
                f"Diagnostic fuzzy: sampled {len(sample_lead_ids)} leads (stub - no processing)"
            )
            return diag
        except Exception as e:
            logger.warning(f"diagnostic fuzzy failed: {e}")
            return []

    def _coerce_pairs_from_result(self, result) -> List[Dict]:
        """
        Return pairs/matches as a list[dict], regardless of engine return type:
        - dict (keys: pairs|matches|pairs_df)
        - pydantic model (.model_dump())
        - dataclass (asdict)
        - attr object (attributes: pairs|matches|pairs_df)
        - pandas DataFrame
        """
        data = None
        pairs_obj = None

        # 1) dict-style
        if isinstance(result, dict):
            data = result
            # avoid boolean coercion of DataFrame — check keys sequentially
            if "pairs" in data:
                pairs_obj = data["pairs"]
            elif "matches" in data:
                pairs_obj = data["matches"]
            elif "pairs_df" in data:
                pairs_obj = data["pairs_df"]

        # 2) pydantic v2 model
        elif hasattr(result, "model_dump") and callable(getattr(result, "model_dump")):
            data = result.model_dump()
            if "pairs" in data:
                pairs_obj = data["pairs"]
            elif "matches" in data:
                pairs_obj = data["matches"]
            elif "pairs_df" in data:
                pairs_obj = data["pairs_df"]

        # 3) dataclass
        elif is_dataclass(result):
            data = asdict(result)
            if "pairs" in data:
                pairs_obj = data["pairs"]
            elif "matches" in data:
                pairs_obj = data["matches"]
            elif "pairs_df" in data:
                pairs_obj = data["pairs_df"]

        # 4) attribute-style object
        else:
            pairs_obj = getattr(result, "pairs", None)
            if pairs_obj is None:
                pairs_obj = getattr(result, "matches", None)
            if pairs_obj is None:
                pairs_obj = getattr(result, "pairs_df", None)

        # 5) Coerce to list[dict]
        if isinstance(pairs_obj, pd.DataFrame):
            return pairs_obj.to_dict(orient="records")
        if isinstance(pairs_obj, list):
            return pairs_obj
        if pairs_obj is None:
            return []
        # last-resort: try list()
        try:
            return list(pairs_obj)
        except TypeError:
            return []

    def __init__(self, db: AsyncSession, account_id: str):
        assert account_id, "IntelligentL2AService requires a non-empty account_id"
        # BaseIntelligentService expects (db, account_id)
        super().__init__(db, str(account_id))
        self.account_id = str(account_id)
        self.db = db
        self.limiter = SmartRateLimiter(
            RateLimitConfig(requests_per_second=5, concurrent_limit=10)
        )

    async def get_intelligent_config(
        self,
        source_df=None,
        target_df=None,
        goal: str = "balanced",
        # accept legacy names to stay backward-compatible
        source_data=None,
        target_data=None,
        **kwargs,
    ):
        """
        Build/obtain a matching config. Use Intelligence if available,
        otherwise return a pragmatic fallback so runs produce output.
        """
        # normalize arg names
        source_df = source_df if source_df is not None else source_data
        target_df = target_df if target_df is not None else target_data

        # (optional) defensive: convert lists/dicts to DataFrames
        try:
            import pandas as pd  # local import to avoid global dep at import time

            if source_df is not None and not hasattr(source_df, "columns"):
                source_df = pd.DataFrame(source_df)
            if target_df is not None and not hasattr(target_df, "columns"):
                target_df = pd.DataFrame(target_df)
        except Exception as _e:
            # don't fail here; fallback config below doesn't need DFs
            pass

        try:
            # Try to use real Intelligence client if available
            from ..api.intelligence_client import IntelligenceClient

            intel = getattr(self, "intel", None) or IntelligenceClient()
            config = await intel.auto_configure(source_df, target_df, goal=goal)

            # Check if config is empty
            if not config or not config.get("field_mappings"):
                logger.warning(
                    _s("Intelligence returned no mappings; using dynamic baseline")
                )
                # Log the input shape that Intelligence saw
                logger.info(
                    _s(
                        f"Auto-config input: lead_cols={len(source_df.columns)} acct_cols={len(target_df.columns)}"
                    )
                )
                # Use dynamic baseline
                if source_df is not None and target_df is not None:
                    config, _, _ = self._build_dynamic_baseline(source_df, target_df)
                    logger.info(
                        _s(
                            f"Dynamic baseline: mappings={len(config['field_mappings'])} "
                            f"blocking={config['blocking']} thr={config['threshold']:.2f}"
                        )
                    )
                    return self._normalize_intel_config(
                        config, default_threshold=config["threshold"]
                    )
                else:
                    # Fall through to legacy fallback
                    raise ValueError("Empty config from Intelligence and no DataFrames")

            logger.info(
                "Using Intelligence API config with %d mappings",
                len(config.get("field_mappings", [])),
            )
            # Normalize before returning
            return self._normalize_intel_config(config, default_threshold=0.75)
        except Exception as e:
            logger.warning(
                _s(f"Intelligence auto-configure failed: {e}; using dynamic baseline")
            )

        # ---- Dynamic baseline (data-aware, no assumptions) ----
        # Build adaptive config based on actual data
        if source_df is not None and target_df is not None:
            config, source_df_updated, target_df_updated = self._build_dynamic_baseline(
                source_df, target_df
            )
            logger.info(
                _s(
                    f"Dynamic baseline: mappings={len(config['field_mappings'])} "
                    f"blocking={config['blocking']} thr={config['threshold']:.2f}"
                )
            )
            # Note: Updated dataframes with derived columns are returned but not used here
            # They'll be re-derived in the main run() method
            return self._normalize_intel_config(
                config, default_threshold=config["threshold"]
            )

        # ---- Legacy fallback if no DataFrames available ----
        # This should rarely happen but keeps backward compatibility
        cols_src = (
            {c.lower() for c in source_df.columns} if source_df is not None else set()
        )
        cols_tgt = (
            {c.lower() for c in target_df.columns} if target_df is not None else set()
        )

        def has(*names):
            """Check if any of the column names exist in either dataframe"""
            return any(n.lower() in cols_src or n.lower() in cols_tgt for n in names)

        # Build blocking dynamically based on available columns
        blocking = []
        if has("Domain", "Website", "Cleaned_Domain__c", "Domain_Append"):
            blocking.append(
                {"left": "Domain", "right": "Domain_Append", "k": 3, "preproc": "alnum"}
            )
        if has("Company", "Name", "Account_Name", "Company_Name"):
            blocking.append(
                {"left": "Company", "right": "Name", "k": 3, "preproc": "alnum"}
            )

        # Build scoring dynamically
        scoring = {}
        if has("Company", "Name", "Account_Name", "Company_Name"):
            scoring["Company<->Name"] = {"algo": "token_set_ratio", "weight": 0.6}
        if has("Email", "Email_Address", "Work_Email"):
            scoring["Email<->Email"] = {"algo": "exact", "weight": 0.7}
        if has("Domain", "Website", "Cleaned_Domain__c", "Domain_Append"):
            scoring["Domain<->Domain"] = {"algo": "exact", "weight": 0.9}
        if has("Phone", "Phone_Number", "Work_Phone"):
            scoring["Phone<->Phone"] = {"algo": "phone", "weight": 0.5}

        # Build field_mappings from scoring (required for new architecture)
        field_mappings = []

        # Add DomainFamily mapping if available (let intelligence determine weight)
        if "DomainFamily" in cols_src and "DomainFamily" in cols_tgt:
            # Only add to scoring, let intelligence determine the optimal weight
            scoring["DomainFamily<->DomainFamily"] = {"algo": "exact"}
            logger.info("Added DomainFamily column for L2A intelligence to optimize")

        # Convert scoring dict to field_mappings list
        for field_pair, config in scoring.items():
            if "<->" in field_pair:
                left, right = field_pair.split("<->")
            else:
                left = right = field_pair

            # Skip if already added (like DomainFamily)
            if any(
                m["source"] == left and m["reference"] == right for m in field_mappings
            ):
                continue

            field_mappings.append(
                {
                    "source": left,
                    "reference": right,
                    "algorithm": config.get("algo", "token_set_ratio"),
                    "weight": config.get("weight", 1.0),
                }
            )

        # If no field mappings were built, add default ones
        if not field_mappings:
            logger.warning("No columns detected for mapping, using defaults")
            field_mappings = [
                {
                    "source": "Company",
                    "reference": "Name",
                    "algorithm": "token_set_ratio",
                    "weight": 0.6,
                },
                {
                    "source": "Email",
                    "reference": "Email",
                    "algorithm": "exact",
                    "weight": 0.7,
                },
                {
                    "source": "Domain",
                    "reference": "Website",
                    "algorithm": "domain",
                    "weight": 0.9,
                },
            ]

        # Log what we're using
        logger.info(
            f"Fallback config using {len(blocking)} blocking keys and {len(field_mappings)} field mappings"
        )
        logger.debug(f"Source columns: {list(cols_src)[:10]}")
        logger.debug(f"Target columns: {list(cols_tgt)[:10]}")
        logger.debug(f"Field mappings: {field_mappings}")

        config = {
            "field_mappings": field_mappings,  # Required by architecture
            "blocking": blocking
            or [{"left": "Company", "right": "Name", "k": 3, "preproc": "alnum"}],
            "scoring": scoring
            or {"Company<->Name": {"algo": "token_set_ratio", "weight": 1.0}},
            "threshold": 0.80,  # Single threshold for compatibility
            "thresholds": {"match": 0.80, "review": 0.70},
            "exclusions": {"personal_domains": True},
        }

        # Normalize fallback config too
        return self._normalize_intel_config(config, default_threshold=0.80)

    def _normalize_intel_config(self, cfg, default_threshold: float = 0.75) -> dict:
        """
        Normalize config to canonical dict shape.
        Returns a dict with at least:
          - field_mappings: list[dict]
          - threshold: float
          - blocking: dict (apply_blocking: bool, keys: list[dict])
        Accepts configs shaped as dicts or plain list[mapping].
        """
        # Start clean
        out = {
            "field_mappings": [],
            "threshold": float(default_threshold),
            "blocking": {"apply_blocking": False, "keys": []},
        }

        # 1) List -> treat as just the mappings
        if isinstance(cfg, list):
            out["field_mappings"] = cfg
            return out

        # 2) Dict -> unify alias keys
        if isinstance(cfg, dict):
            # mappings: accept 'field_mappings' or 'mappings'
            fm = cfg.get("field_mappings")
            if fm is None:
                fm = cfg.get("mappings")
            if isinstance(fm, list):
                out["field_mappings"] = fm

            # thresholds: accept 'threshold' or bundle under 'thresholds'
            if "threshold" in cfg and isinstance(cfg["threshold"], (float, int)):
                out["threshold"] = float(cfg["threshold"])
            elif isinstance(cfg.get("thresholds"), dict):
                # prefer 'suggest' if present, else keep default
                t = cfg["thresholds"].get("suggest")
                if isinstance(t, (float, int)):
                    out["threshold"] = float(t)

            # blocking: accept either 'blocking' dict or top-level keys
            blk = cfg.get("blocking") or {}
            if isinstance(blk, dict):
                apply_flag = bool(blk.get("apply_blocking"))
                keys = blk.get("keys")
                # Some configs return a single-column plan instead of keys[]
                if not keys and blk.get("column"):
                    keys = [
                        {
                            "column": blk.get("column"),
                            "variant": blk.get("variant") or "alnum",
                            "key_length": blk.get("key_length") or 3,
                        }
                    ]
                if isinstance(keys, list):
                    out["blocking"] = {"apply_blocking": apply_flag, "keys": keys}

            return out

        # 3) Anything else -> empty but valid
        return out

    def _get_apply_blocking(self, cfg) -> bool:
        """Safe getter for apply_blocking flag"""
        if isinstance(cfg, dict):
            blk = cfg.get("blocking") or {}
            return bool(blk.get("apply_blocking"))
        return False

    def _assert_config_shape(self, cfg: dict):
        """Validate config has expected shape"""
        ok = isinstance(cfg, dict) and isinstance(cfg.get("field_mappings"), list)
        if not ok:
            # Map to published error model
            raise ValueError(
                "INTEL_CONFIG_INVALID: expected dict with 'field_mappings' list"
            )

    def _build_dynamic_baseline(
        self, leads_df: pd.DataFrame, accounts_df: pd.DataFrame
    ) -> tuple:
        """Construct a data-aware baseline config (schema-agnostic)."""
        fm: list[dict] = []
        blocking_keys: list[dict] = []
        thr = 0.75  # start; may adjust at end

        # ---------- Derive domains (don't assume column names) ----------
        def _registrable(v: str | None):
            if not v:
                return None
            try:
                return _registrable_domain(v)
            except Exception:
                return None

        # leads: any email-like field
        lead_email_cols = [
            c
            for c in leads_df.columns
            if "email" in c.lower()
            or leads_df[c].astype(str).str.contains("@").mean() > 0.2
        ]
        leads_df = leads_df.copy()
        if lead_email_cols:
            cand = max(lead_email_cols, key=lambda c: leads_df[c].notna().mean())
            leads_df["email_domain"] = (
                leads_df[cand]
                .astype(str)
                .map(lambda s: _registrable(s.split("@")[-1]) if "@" in s else None)
            )

        # accounts: best website-ish field
        acct_web_cands = [
            c
            for c in accounts_df.columns
            if any(k in c.lower() for k in ("website", "domain", "url"))
        ]
        accounts_df = accounts_df.copy()
        if acct_web_cands:
            aw = max(acct_web_cands, key=lambda c: accounts_df[c].notna().mean())
            accounts_df["website_domain"] = (
                accounts_df[aw].astype(str).map(_registrable)
            )

        # Add domain family if available (let intelligence determine weight and anchor status)
        if "DomainFamily" in leads_df.columns and "DomainFamily" in accounts_df.columns:
            if (
                leads_df["DomainFamily"].notna().mean() > 0.1
                and accounts_df["DomainFamily"].notna().mean() > 0.1
            ):
                # Just add the field, let intelligence decide weight and whether to use as anchor
                fm.append(
                    {
                        "source": "DomainFamily",
                        "reference": "DomainFamily",
                        "algorithm": "Exact",
                    }
                )
                logger.info(
                    "Added DomainFamily field for dynamic baseline L2A intelligence"
                )

        # Add anchor if both sides exist with non-trivial fill
        if (
            "email_domain" in leads_df.columns
            and "website_domain" in accounts_df.columns
        ):
            if (
                leads_df["email_domain"].notna().mean() > 0.2
                and accounts_df["website_domain"].notna().mean() > 0.2
            ):
                fm.append(
                    {
                        "source": "email_domain",
                        "reference": "website_domain",
                        "algorithm": "Domain",
                        "weight": 1.0,
                        "is_anchor": True,
                    }
                )

        # ---------- Discover name-like columns (texty, decent fill) ----------
        def _rank_nameish(df: pd.DataFrame):
            textish = [c for c in df.columns if df[c].dtype == "object"]

            def score(c):
                s = df[c].astype(str)
                fill = s.notna().mean()
                alpha = s.str.contains(r"[A-Za-z]", na=False).mean()
                avglen = s.str.len().mean()
                return fill * 0.55 + alpha * 0.35 + (0.1 if 3 < avglen < 60 else 0)

            return sorted(textish, key=score, reverse=True)

        lead_name = (
            (_rank_nameish(leads_df) or ["Company"])[0]
            if leads_df.shape[1] > 0
            else "Company"
        )
        acct_name = (
            (_rank_nameish(accounts_df) or ["Name"])[0]
            if accounts_df.shape[1] > 0
            else "Name"
        )

        # normalized shadows (idempotent if already present)
        def _norm_name(s: str) -> str:
            if not s:
                return ""
            s = str(s).lower().strip()
            for suf in (
                " inc",
                " llc",
                " ltd",
                " corp",
                " corporation",
                " company",
                " co",
            ):
                if s.endswith(suf):
                    s = s[: -len(suf)]
            # keep letters/numbers/space; be Unicode-safe
            return "".join(ch for ch in s if ch.isalnum() or ch.isspace())

        if lead_name in leads_df.columns:
            leads_df["__name_norm"] = leads_df[lead_name].map(_norm_name)
        else:
            leads_df["__name_norm"] = ""

        if acct_name in accounts_df.columns:
            accounts_df["__name_norm"] = accounts_df[acct_name].map(_norm_name)
        else:
            accounts_df["__name_norm"] = ""

        fm.append(
            {
                "source": "__name_norm",
                "reference": "__name_norm",
                "algorithm": "WRatio",
                "weight": 0.45,
            }
        )

        # Add derived domain mapping if available (from enhancement above)
        if "__derived_domain" in leads_df.columns:
            derived_coverage = leads_df["__derived_domain"].notna().mean()
            if derived_coverage >= 0.10:
                fm.append(
                    {
                        "source": "__derived_domain",
                        "reference": "website_domain",
                        "algorithm": "Domain",
                        "weight": 0.50,  # Even lower for L2A
                    }
                )
                logger.info(
                    f"Added derived domain mapping with {derived_coverage:.1%} coverage"
                )

        # ---------- Blocking strategy (measured) ----------
        # 1) domain block if overlap exists; else 2) name prefix with autotune; else 3) none (let engine sample)
        def _overlap(a: pd.Series, b: pd.Series) -> int:
            A, B = (
                set(a.dropna().astype(str).unique()),
                set(b.dropna().astype(str).unique()),
            )
            return len(A & B)

        applied_block = False
        if (
            "email_domain" in leads_df.columns
            and "website_domain" in accounts_df.columns
        ):
            od = _overlap(leads_df["email_domain"], accounts_df["website_domain"])
            if od > 0:
                leads_df["__block_domain"] = leads_df["email_domain"]
                accounts_df["__block_domain"] = accounts_df["website_domain"]
                blocking_keys.append(
                    {"column": "__block_domain", "variant": "alnum", "key_length": 3}
                )
                applied_block = True
                logger.info(
                    _s(f"Dynamic baseline: using domain blocking (overlap={od})")
                )

        # Check if derived domains can be used for blocking
        if (
            not applied_block
            and "__derived_domain" in leads_df.columns
            and "website_domain" in accounts_df.columns
        ):
            derived_coverage = leads_df["__derived_domain"].notna().mean()
            explicit_coverage = (
                leads_df["email_domain"].notna().mean()
                if "email_domain" in leads_df.columns
                else 0
            )
            if derived_coverage >= 0.25 and explicit_coverage < 0.30:
                od = _overlap(
                    leads_df["__derived_domain"], accounts_df["website_domain"]
                )
                if od > 0:
                    leads_df["__block_domain"] = leads_df["__derived_domain"]
                    accounts_df["__block_domain"] = accounts_df["website_domain"]
                    blocking_keys.append(
                        {
                            "column": "__block_domain",
                            "variant": "alnum",
                            "key_length": 3,
                        }
                    )
                    applied_block = True
                    logger.info(
                        f"DomainEnhanced(L2A): Using derived domains for blocking (overlap={od})"
                    )

        if not applied_block:
            # try k=8 then k=4 prefixes on discovered name columns
            for k in (8, 4):
                lpf = leads_df["__name_norm"].str[:k]
                apf = accounts_df["__name_norm"].str[:k]
                overlap_count = _overlap(lpf, apf)
                if overlap_count > 0:
                    leads_df["__block_name"] = lpf
                    accounts_df["__block_name"] = apf
                    blocking_keys.append(
                        {"column": "__block_name", "variant": "alnum", "key_length": 3}
                    )
                    applied_block = True
                    logger.info(
                        _s(
                            f"Dynamic baseline: using name prefix blocking k={k} (overlap={overlap_count})"
                        )
                    )
                    break

        blocking = {"apply_blocking": bool(blocking_keys), "keys": blocking_keys}

        # ---------- Threshold hint (micro calibration) ----------
        # Use a conservative hint; you already expose L2A_ENGINE_THRESHOLD to truly control it.
        thr = 0.72 if blocking.get("apply_blocking") else 0.66

        return (
            {"field_mappings": fm, "blocking": blocking, "threshold": thr},
            leads_df,
            accounts_df,
        )

    async def ensure_policy(self, account_id: str, integration_id: str) -> L2APolicy:
        """Ensure a policy exists for this account"""
        # Default values for policies
        DEFAULT_THRESHOLDS = {"suggest": 0.75, "auto_link": 0.95}
        DEFAULT_WEIGHTS = {"domain": 0.45, "name": 0.30, "phone": 0.15, "geo": 0.10}

        # Upgrade settings defaults
        DEFAULT_UPGRADE_SETTINGS = {
            "upgrade_min_gain": 0.10,
            "upgrade_min_score": 0.80,
            "upgrade_anchor_required": False,
            "upgrade_enablement": "disabled",
            "link_overwrite_mode": "if_empty",
            "use_multi_algo": False,
        }

        res = await self.db.execute(
            select(L2APolicy).where(
                L2APolicy.account_id == account_id, L2APolicy.status == "ACTIVE"
            )
        )
        pol = res.scalar_one_or_none()
        if pol:
            # Ensure existing policies have all required fields
            updated = False
            if not pol.thresholds:
                pol.thresholds = DEFAULT_THRESHOLDS
                updated = True
            if not pol.weights:
                pol.weights = DEFAULT_WEIGHTS
                updated = True

            # Set upgrade fields if missing (for policies created before upgrade feature)
            for key, default_value in DEFAULT_UPGRADE_SETTINGS.items():
                if not hasattr(pol, key) or getattr(pol, key) is None:
                    setattr(pol, key, default_value)
                    updated = True

            # Save updates if we made any changes
            if updated:
                self.db.add(pol)
                await self.db.commit()
                await self.db.refresh(pol)

            return pol

        # Detect lead lookup field
        lead_lookup = await self._detect_lead_lookup_field(integration_id)

        # Fix 7: Smart defaults for new organizations
        pol = L2APolicy(
            account_id=account_id,
            integration_id=integration_id,
            lead_lookup_field=lead_lookup or None,
            thresholds=DEFAULT_THRESHOLDS,
            weights=DEFAULT_WEIGHTS,
            # Smart defaults for upgrade features
            upgrade_min_gain=0.10,  # 10% minimum improvement
            upgrade_min_score=0.80,  # 80% minimum confidence
            upgrade_anchor_required=True,  # Require strong anchor field by default
            upgrade_enablement="disabled",  # Start disabled, enable consciously
            # Smart writeback defaults
            link_overwrite_mode="if_empty",  # Safe default - only fill blanks
            use_multi_algo=True,  # Enable ensemble scoring by default for accuracy
            auto_link_tier="DISABLED",  # Require manual approval initially
        )
        self.db.add(pol)
        await self.db.commit()
        await self.db.refresh(pol)
        return pol

    async def _detect_lead_lookup_field(self, integration_id: str) -> Optional[str]:
        """Detect the Lead field that references Account"""
        integ = await self._get_sf_integration(integration_id)
        tenant_id = integ.account_id
        gateway = await SalesforceGateway.for_tenant(tenant_id, self.db)

        # Use gateway for describe operation - handles token refresh automatically
        schema = await gateway.describe("Lead")
        for f in schema.get("fields", []):
            if f.get("type") == "reference" and "Account" in (
                f.get("referenceTo") or []
            ):
                return f["name"]
        return None

    async def _get_sf_integration(self, integration_id_or_account: str):
        """Get Salesforce integration"""
        q = select(Integration).where(Integration.id == integration_id_or_account)
        res = await self.db.execute(q)
        integ = res.scalar_one_or_none()
        if integ:
            return integ

        q = select(Integration).where(
            Integration.account_id == integration_id_or_account,
            Integration.provider == IntegrationProvider.SALESFORCE,
        )
        res = await self.db.execute(q)
        return res.scalar_one()

    async def _get_sf_gateway(self, integration):
        """Get SalesforceGateway for tenant"""
        tenant_id = integration.account_id
        gateway = await SalesforceGateway.for_tenant(tenant_id, self.db)
        return gateway, tenant_id

    async def process(
        self,
        source_data: pd.DataFrame,
        target_data: Optional[pd.DataFrame] = None,
        **kwargs,
    ) -> Dict[str, Any]:
        """Process using intelligence - required by BaseIntelligentService"""
        return await self.run(
            account_id=kwargs.get("account_id"),
            threshold_override=kwargs.get("threshold_override"),
            progress_cb=kwargs.get("progress_callback"),
        )

    async def get_salesforce_gateway(self, integ):
        """Get Salesforce gateway for integration"""
        tenant_id = integ.account_id
        return await SalesforceGateway.for_tenant(tenant_id, self.db)

    async def list_writeback_options_for_integration(self, integ, policy) -> dict:
        """
        Inspect Salesforce Lead describe and return Account lookup fields that are updateable.
        """
        gateway = await self.get_salesforce_gateway(integ)
        desc = await gateway.describe("Lead")
        cands: List[WritebackCandidate] = []

        for f in desc.get("fields", []):
            if f.get("type") == "reference" and "Account" in (
                f.get("referenceTo") or []
            ):
                if f.get("updateable") and not f.get("calculated"):
                    reason = None
                    name = f["name"]
                    if name.lower() in {
                        "matched_account__c",
                        "account__c",
                        "target_account__c",
                        "linked_account__c",
                    }:
                        reason = "name_match"
                    cands.append(
                        WritebackCandidate(
                            apiName=name,
                            label=f.get("label") or name,
                            reason=reason,
                            updateable=True,
                            accessible=f.get("accessible", True),
                        )
                    )

        # Pick recommended
        recommended = None
        if policy and policy.link_field_api_name:
            # previously saved choice wins
            recommended = next(
                (c for c in cands if c.apiName == policy.link_field_api_name), None
            )
        if not recommended and cands:
            # try name-match first else first result
            preferred = [c for c in cands if c.reason == "name_match"]
            recommended = preferred[0] if preferred else cands[0]

        return {
            "recommended": recommended.dict() if recommended else None,
            "candidates": [c.dict() for c in cands],
        }

    async def choose_writeback_field(
        self, policy, field_api_name: str, overwrite: OverwriteMode
    ):
        """Save the user's chosen writeback field and overwrite mode to the policy"""
        policy.link_field_api_name = field_api_name
        policy.link_overwrite_mode = (
            overwrite.value if isinstance(overwrite, OverwriteMode) else overwrite
        )
        await self.db.commit()
        await self.db.refresh(policy)
        return {
            "policy_id": str(policy.id),
            "link_field_api_name": policy.link_field_api_name,
            "link_overwrite_mode": policy.link_overwrite_mode,
        }

    async def validate_writeback(
        self, integ, field_api_name: str, sample_lead_id: str
    ) -> dict:
        """
        Performs a safe no-op validation: checks FLS and current value.
        """
        gateway = await self.get_salesforce_gateway(integ)

        # Quick FLS peek
        ld = await gateway.describe("Lead")
        fld = next(
            (f for f in ld.get("fields", []) if f.get("name") == field_api_name), None
        )
        if not fld:
            return {"ok": False, "error": f"Field {field_api_name} not found on Lead"}
        if not fld.get("updateable", False):
            return {
                "ok": False,
                "error": f"Field {field_api_name} is not updateable for current user/profile",
            }

        # Optional: fetch current sample lead value to report overwrite risk
        try:
            # Get single Lead record to check field value
            soql = f"SELECT {field_api_name} FROM Lead WHERE Id = '{sample_lead_id}' LIMIT 1"
            result = await gateway.soql(soql)
            current = result["records"][0] if result.get("records") else {}
            has_value = bool(current.get(field_api_name))
            return {"ok": True, "has_value": has_value}
        except Exception as e:
            # If we can't fetch the lead, just return field is ok
            logger.warning(f"Could not fetch lead {sample_lead_id} for validation: {e}")
            return {"ok": True, "has_value": False}

    def _has_true_anchor(self, field_score_details: list) -> bool:
        """
        FIX 4: Detect true anchor matches (not just high scores).
        Avoids false positives from ID fields like OwnerId, CampaignId.
        """
        if not field_score_details:
            return False

        # Define true anchor field patterns (lowercase)
        ANCHOR_FIELDS = {"email", "domain", "website", "phone"}

        def is_anchor_field_name(field_name: str) -> bool:
            """Check if field name is a true anchor field"""
            if not field_name:
                return False

            field_lower = field_name.lower()

            # Exclude ID fields that might contain anchor words
            if any(
                exclude in field_lower for exclude in ["id", "key", "code", "number"]
            ):
                if not any(anchor == field_lower for anchor in ANCHOR_FIELDS):
                    return False

            # Check for anchor field patterns
            return any(anchor in field_lower for anchor in ANCHOR_FIELDS)

        for detail in field_score_details:
            score = detail.get("score", 0)

            # Need high confidence
            if score < 0.95:
                continue

            # Check field names
            src_field = detail.get("src_field") or detail.get("source_column") or ""
            ref_field = detail.get("ref_field") or detail.get("reference_column") or ""

            if is_anchor_field_name(src_field) or is_anchor_field_name(ref_field):
                logger.debug(
                    f"Found anchor field match: {src_field}->{ref_field} = {score:.2f}"
                )
                return True

            # Check explicit flags from engine
            flag = detail.get("flag") or detail.get("type") or ""
            if flag in {"email_exact", "domain_exact", "website_exact", "phone_exact"}:
                return True

            # Check algorithm type
            algo = (detail.get("algorithm") or "").lower()
            if algo in {"exact", "domain", "emaildomain"} and score >= 0.95:
                # But only if on an anchor field
                if is_anchor_field_name(src_field) or is_anchor_field_name(ref_field):
                    return True

        return False

    async def run(
        self,
        account_id: str,
        threshold_override: Optional[float] = None,
        progress_cb: Optional[Any] = None,
        b2c_mode: str = None,
        b2c_whitelist: List[str] = None,
        mode: str = "incremental",  # 'incremental' or 'rebuild'
        use_multi_algo: Optional[bool] = None,  # Override policy setting
        campaign_id: Optional[str] = None,  # Filter by Campaign membership
        lead_source: Optional[str] = None,  # Filter by LeadSource
        created_date_from: Optional[str] = None,  # Filter by CreatedDate
        audit_mode: str = "unlinked",  # NEW: "unlinked" | "upgrade"
    ) -> Dict[str, Any]:
        """
        Match Leads to Accounts using ONLY intelligence APIs.

        NO HARDCODING ALLOWED:
        - Field mappings come from AutoMapper via intelligence API
        - Thresholds come from data quality analysis
        - Algorithms selected by semantic type detection
        - Blocking strategy optimized by intelligence
        """
        start_time = datetime.now()

        # Fix 5: Respect upgrade_enablement policy
        # FIX 5: Single policy read at the start and extract all settings upfront
        integ = await self._get_sf_integration(account_id)
        policy = await self.ensure_policy(integ.account_id, integ.id)

        # Extract all policy settings upfront (no more re-reading)
        upgrade_enablement = getattr(policy, "upgrade_enablement", "disabled")
        upgrade_min_gain = float(getattr(policy, "upgrade_min_gain", 0.10))
        upgrade_min_score = float(getattr(policy, "upgrade_min_score", 0.80))
        upgrade_anchor_required = bool(
            getattr(policy, "upgrade_anchor_required", False)
        )
        writeback_field = policy.link_field_api_name or policy.lead_lookup_field
        use_multi_algo_policy = bool(getattr(policy, "use_multi_algo", False))

        # FIX 3: Return success (not error) for disabled upgrades
        if audit_mode == "upgrade":
            if upgrade_enablement == "disabled":
                logger.info("Upgrade mode requested but disabled by policy")
                return {
                    "status": "success",  # Not "error"
                    "message": "Upgrade audits are currently disabled for this account",
                    "considered": 0,
                    "suggestions_generated": 0,
                    "upgrades_suggested": 0,
                    "audit_mode": "upgrade",
                    "upgrade_criteria": {
                        "enablement": upgrade_enablement,
                        "min_gain": upgrade_min_gain,
                        "min_score": upgrade_min_score,
                        "anchor_required": upgrade_anchor_required,
                    },
                    "total_duration_seconds": 0,
                }
            elif upgrade_enablement == "pilot":
                logger.info(
                    "Running in pilot upgrade mode - limited features may apply"
                )

        logger.info(
            f"Running L2A with audit_mode={audit_mode}, upgrade_enablement={upgrade_enablement}"
        )

        # JSON breadcrumb for run start (for debugging big datasets)
        import json
        import hashlib

        # Defensive import in case of circular import issues
        try:
            reg  # Check if get_registry is already available
        except NameError:
            from .domain_family import get_registry, DomainRelation  # Lazy fallback

        reg = get_registry()

        # Stable version hash of domain families
        def _reg_version():
            items = sorted(reg.alias_to_family.items())
            return hashlib.sha1(
                json.dumps(items, separators=(",", ":"), ensure_ascii=True).encode(
                    "utf-8"
                )
            ).hexdigest()[:12]

        run_breadcrumb = {
            "type": "FMATCH_RUN",
            "timestamp": start_time.isoformat(),
            "account_id": account_id,
            "audit_mode": audit_mode,
            "mode": mode,
            "use_multi_algo": use_multi_algo,
            "campaign_id": campaign_id,
            "lead_source": lead_source,
            "created_date_from": created_date_from,
            "upgrade_enablement": upgrade_enablement,
            "domain_registry_version": _reg_version(),
        }
        logger.info(f"FMATCH_RUN: {json.dumps(run_breadcrumb)}")

        # Get B2C mode from parameters or environment
        b2c_mode = (b2c_mode or os.getenv("L2A_B2C_DEFAULT_MODE", "block_only")).lower()
        if b2c_mode not in ("strict", "block_only", "off"):
            b2c_mode = "block_only"

        # Build whitelist set for fast lookups
        b2c_whitelist_set = set(b2c_whitelist or [])

        # Create B2C check function that honors whitelist
        def is_b2c_with_whitelist(domain: str) -> bool:
            """Check if domain is B2C, respecting whitelist"""
            if not domain:
                return False
            domain = domain.lower().strip()

            # Check whitelist first - these are always treated as business
            if domain in b2c_whitelist_set:
                return False
            # Also check if email is in whitelist (e.g., "store@gmail.com")
            for wl in b2c_whitelist_set:
                if "@" in wl and domain == wl.split("@")[1]:
                    return False

            # Use the desktop B2C checker
            return is_b2c_domain(domain)

        # Initialize B2C metrics
        b2c_metrics = {
            "leads_b2c_count": 0,
            "accounts_b2c_count": 0,
            "mappings_suppressed": 0,
            "blockkey_coverage_before": 0.0,
            "blockkey_coverage_after": 0.0,
            "b2c_mode": b2c_mode,
            "whitelist_count": len(b2c_whitelist_set),
            "top_bucket_pct": 0.0,
        }

        logger.info(
            "B2C configuration: mode=%s, whitelist=%d domains",
            b2c_mode,
            len(b2c_whitelist_set),
        )

        if progress_cb:
            progress_cb(0.05, "Loading Salesforce integration")

        # Use centralized gateway for all Salesforce operations
        gateway, tenant_id = await self._get_sf_gateway(integ)

        # Pre-flight check - test SF connection before starting
        try:
            await gateway.describe("Account")
        except ValueError as e:
            if "Authentication failed" in str(e) or "401" in str(e):
                raise ValueError(f"Salesforce authentication failed: {str(e)}")
            raise

        # Handle rebuild mode vs incremental mode
        if mode == "rebuild":
            # Delete existing pending suggestions for fresh rebuild
            logger.info(
                _s(
                    "Rebuild mode: deleting existing pending suggestions for account %s"
                ),
                account_id,
            )
            from sqlalchemy import text

            # Delete by account scope to handle policy changes
            delete_result = await self.db.execute(
                text("""
                    DELETE FROM l2a_suggestions
                    WHERE policy_id IN (
                        SELECT id FROM l2a_policies WHERE account_id = :account_id
                    )
                    AND status IN ('pending', 'proposed', 'new')
                """),
                {"account_id": account_id},
            )
            await self.db.commit()
            logger.info(
                _s("Deleted %d existing pending/proposed/new suggestions"),
                delete_result.rowcount,
            )
            existing_lead_ids = set()  # Process all leads in rebuild mode
        else:
            # FIX 2: Scope duplicate prevention to pending upgrades only
            if audit_mode == "upgrade":
                # Check for existing PENDING UPGRADES only (not all suggestions)
                existing_query = select(L2ASuggestion.lead_id).where(
                    L2ASuggestion.policy_id == policy.id,
                    L2ASuggestion.reason_code == "upgrade",  # Only upgrades
                    L2ASuggestion.status == "pending",  # Only pending
                )
                existing_result = await self.db.execute(existing_query)
                existing_lead_ids = {row[0] for row in existing_result.all()}
                logger.info(
                    f"Found {len(existing_lead_ids)} existing pending upgrades to skip"
                )
            else:
                # For regular mode, check all existing suggestions
                existing_query = select(L2ASuggestion.lead_id).where(
                    L2ASuggestion.policy_id == policy.id,
                    L2ASuggestion.status.in_(["pending", "linked"]),
                )
                existing_result = await self.db.execute(existing_query)
                existing_lead_ids = {row[0] for row in existing_result.all()}
                logger.info(
                    f"Incremental mode: Found {len(existing_lead_ids)} existing suggestions to skip"
                )

        if progress_cb:
            progress_cb(0.15, "Fetching Accounts from Salesforce")

        # Fetch ALL available fields for intelligent mapping
        account_schema = await gateway.describe("Account")

        # Get allowed fields from gateway (lowercased)
        allowed_account_fields = SalesforceGateway.ALLOWED_OBJECTS.get("account", set())

        # Intersect with describe results, preserving casing from schema
        account_fields = [
            f["name"]
            for f in account_schema.get("fields", [])
            if f["name"].lower() in allowed_account_fields
        ]

        # Always include Id (defensive)
        if "Id" not in account_fields:
            account_fields = ["Id"] + account_fields

        # Build and execute accounts query with gateway pagination
        business_cap = 50000  # Your business limit
        accounts: list[dict] = []
        acct_query = f"SELECT {', '.join(account_fields)} FROM Account WHERE IsDeleted = false LIMIT {business_cap}"
        async for page in gateway.paginate_soql(acct_query):
            if page:
                accounts.extend(page)
            if len(accounts) >= business_cap:
                accounts = accounts[:business_cap]
                break

        logger.info(
            f"Fetched {len(accounts)} Accounts with {len(account_fields)} fields"
        )

        if progress_cb:
            progress_cb(0.30, "Fetching Leads from Salesforce")

        # Fetch ALL available Lead fields
        lead_schema = await gateway.describe("Lead")

        # Get allowed fields from gateway (lowercased)
        allowed_lead_fields = SalesforceGateway.ALLOWED_OBJECTS.get("lead", set())

        # Intersect with describe results, preserving casing from schema
        lead_fields = [
            f["name"]
            for f in lead_schema.get("fields", [])
            if f["name"].lower() in allowed_lead_fields
        ]

        # Always include Id (defensive)
        if "Id" not in lead_fields:
            lead_fields = ["Id"] + lead_fields

        # Already extracted writeback_field upfront (FIX 5)

        # Add writeback field to lead_fields if not already present
        if writeback_field and writeback_field not in lead_fields:
            lead_fields.append(writeback_field)

        # Build base query for unconverted leads
        leads_base_query = (
            f"SELECT {', '.join(lead_fields)} FROM Lead WHERE IsConverted=false"
        )

        # Build filter conditions
        filter_conditions = ""

        # Only filter nulls in unlinked mode
        if audit_mode == "unlinked" and writeback_field:
            filter_conditions += f" AND {writeback_field} = null"

        # Apply optional filters
        if campaign_id:
            # Use subquery to filter by campaign membership
            filter_conditions += f" AND Id IN (SELECT LeadId FROM CampaignMember WHERE CampaignId = '{campaign_id}')"
            logger.info(f"Filtering leads by Campaign ID: {campaign_id}")

        if lead_source:
            # Escape backslashes and single quotes in lead source
            escaped_source = lead_source.replace("\\", "\\\\").replace("'", "\\'")
            filter_conditions += f" AND LeadSource = '{escaped_source}'"
            logger.info("Filtering leads by LeadSource")  # Don't log PII

        if created_date_from:
            # Parse and format date properly
            from datetime import datetime as dt

            date_obj = dt.strptime(created_date_from, "%Y-%m-%d")
            filter_conditions += (
                f" AND CreatedDate >= {date_obj.strftime('%Y-%m-%dT00:00:00Z')}"
            )
            logger.info(f"Filtering leads created after: {created_date_from}")

        # Fetch leads using gateway pagination (no OFFSET)
        business_cap = 10000  # Safety limit for leads
        all_leads: list[dict] = []
        leads_query = f"{leads_base_query}{filter_conditions} LIMIT {business_cap}"
        async for page in gateway.paginate_soql(leads_query):
            if page:
                all_leads.extend(page)
            if len(all_leads) >= business_cap:
                all_leads = all_leads[:business_cap]
                break

        logger.info(f"Fetched {len(all_leads)} unconverted Leads")

        # Build incumbent map for upgrade mode
        incumbents = {}
        if audit_mode == "upgrade" and writeback_field:
            for lead in all_leads:
                if incumbent_id := lead.get(writeback_field):
                    incumbents[lead["Id"]] = incumbent_id
            logger.info(
                f"Found {len(incumbents)} leads with existing account assignments"
            )

        # Filter out existing suggestions (in incremental mode only)
        if audit_mode == "unlinked":
            leads_to_process = [
                l for l in all_leads if l["Id"] not in existing_lead_ids
            ]
        else:  # upgrade mode - process all leads with incumbents
            leads_to_process = [l for l in all_leads if l["Id"] in incumbents]

        logger.info(f"Processing {len(leads_to_process)} Leads in {audit_mode} mode")

        if not leads_to_process:
            return {
                "considered": 0,
                "suggestions_generated": 0,
                "processing_time_ms": 0,
                "message": "No new leads to process",
            }

        if progress_cb:
            progress_cb(0.45, "Analyzing data with intelligence API")

        # Convert to DataFrames
        leads_df = pd.DataFrame(leads_to_process)
        accounts_df = pd.DataFrame(accounts)

        # Enhanced domain extraction and family enrichment
        from .domain_utils import extract_and_enrich_domains

        # Extract and enrich domains for both datasets
        leads_df, lead_family_coverage = extract_and_enrich_domains(leads_df)
        accounts_df, account_family_coverage = extract_and_enrich_domains(accounts_df)

        if lead_family_coverage > 0 or account_family_coverage > 0:
            logger.info(
                f"Domain family coverage - Leads: {lead_family_coverage:.1f}%, Accounts: {account_family_coverage:.1f}%"
            )

        # Domain enhancement for leads without email domain
        feature_flag = os.getenv("FEATURE_DOMAIN_ENHANCED", "false").lower() == "true"
        if feature_flag and "Company" in leads_df.columns:
            try:
                from .domain_family import get_registry

                reg = get_registry()

                def derive_lead_domain(row):
                    # Only derive if no email domain
                    if pd.notna(row.get("email_domain")):
                        return None
                    if not row.get("Company"):
                        return None
                    try:
                        suggestions = reg.suggest_domains(row["Company"])
                        if suggestions and suggestions[0][1] >= 0.90:
                            return suggestions[0][0]
                    except Exception as e:
                        logger.debug(
                            f"Domain derivation failed for company '{row.get('Company')}': {e}"
                        )
                    return None

                leads_df["__derived_domain"] = leads_df.apply(
                    derive_lead_domain, axis=1
                )
                coverage = leads_df["__derived_domain"].notna().mean()

                # Check for conflicts between derived and explicit domains
                if "email_domain" in leads_df.columns:
                    conflicts = (
                        leads_df["__derived_domain"].notna()
                        & leads_df["email_domain"].notna()
                        & (leads_df["__derived_domain"] != leads_df["email_domain"])
                    )
                    if conflicts.any():
                        conflict_count = conflicts.sum()
                        logger.warning(
                            f"DomainEnhanced: {conflict_count} conflicts between derived and explicit domains"
                        )
                        # Don't use derived for rows with conflicts
                        leads_df.loc[conflicts, "__derived_domain"] = None
                        # Recalculate coverage after removing conflicts
                        coverage = leads_df["__derived_domain"].notna().mean()

                if coverage >= 0.10:
                    # Will be added to mappings later with reduced weight
                    logger.info(
                        f"DomainEnhanced(L2A): {coverage:.1%} coverage for derived domains (after conflict resolution)"
                    )
            except Exception as e:
                logger.warning(f"Domain enhancement failed: {e}")
                # Continue without domain enhancement

        # Update FMATCH_RUN breadcrumb with actual counts
        run_breadcrumb["leads_count"] = len(leads_df)
        run_breadcrumb["accounts_count"] = len(accounts_df)
        logger.info(f"FMATCH_RUN_UPDATE: {json.dumps(run_breadcrumb)}")

        # CRITICAL FIX: Extract domains for proper matching
        # TODO: Upgrade to PSL-aware domain extraction using tldextract library
        # This would handle international domains better (e.g., .co.uk, .com.br)
        # For now using registrable_domain which is PSL-aware but could be more robust

        # Add email_domain column to leads
        if "Email" in leads_df.columns:
            leads_df["email_domain"] = leads_df["Email"].apply(
                lambda x: _registrable_domain(x.split("@")[1])
                if pd.notna(x) and "@" in x
                else None
            )
            logger.info(
                f"Created email_domain for {leads_df['email_domain'].notna().sum()}/{len(leads_df)} leads"
            )

        # Add website_domain column to accounts
        website_cols = [
            "Website",
            "Domain__c",
            "Domain_Append",
            "DHC_Provider_Website__c",
        ]
        for col in website_cols:
            if col in accounts_df.columns:
                accounts_df["website_domain"] = accounts_df[col].apply(
                    lambda x: _registrable_domain(x) if pd.notna(x) else None
                )
                logger.info(
                    f"Created website_domain from {col} for {accounts_df['website_domain'].notna().sum()}/{len(accounts_df)} accounts"
                )
                break

        # EPHEMERAL DOMAIN BLOCKING: Create temporary blocking columns
        domain_blocking_active = False
        if (
            "email_domain" in leads_df.columns
            and "website_domain" in accounts_df.columns
        ):
            # Check coverage
            email_coverage = (
                leads_df["email_domain"].notna().sum() / len(leads_df)
                if len(leads_df) > 0
                else 0
            )
            website_coverage = (
                accounts_df["website_domain"].notna().sum() / len(accounts_df)
                if len(accounts_df) > 0
                else 0
            )

            if (
                email_coverage > 0.2 and website_coverage > 0.2
            ):  # Lower threshold for blocking
                # Create unified blocking column with same name on both sides
                leads_df["__block_domain"] = leads_df["email_domain"]
                accounts_df["__block_domain"] = accounts_df["website_domain"]
                domain_blocking_active = True
                logger.info(
                    f"Created ephemeral domain blocking columns (coverage: {email_coverage:.1%} leads, {website_coverage:.1%} accounts)"
                )

                # Store coverage for breadcrumb
                run_breadcrumb["domain_coverage_leads"] = round(email_coverage, 3)
                run_breadcrumb["domain_coverage_accounts"] = round(website_coverage, 3)

        # Detect B2C domains in leads and accounts (only if B2C mode is enabled)
        if b2c_mode != "off":
            lead_domain_cols = ["Email", "Domain", "Website", "Domain__c"]
            acct_domain_cols = [
                "Website",
                "Domain__c",
                "CleanDomain__c",
                "Domain_Append",
            ]

            # Check for B2C domains in leads
            leads_df["_is_b2c"] = False
            for col in lead_domain_cols:
                if col in leads_df.columns:
                    leads_df["_is_b2c"] |= leads_df[col].apply(
                        lambda x: is_b2c_with_whitelist(_registrable_domain(x))
                        if pd.notna(x)
                        else False
                    )
            b2c_metrics["leads_b2c_count"] = leads_df["_is_b2c"].sum()

            # Check for B2C domains in accounts
            accounts_df["_is_b2c"] = False
            for col in acct_domain_cols:
                if col in accounts_df.columns:
                    accounts_df["_is_b2c"] |= accounts_df[col].apply(
                        lambda x: is_b2c_with_whitelist(_registrable_domain(x))
                        if pd.notna(x)
                        else False
                    )
            b2c_metrics["accounts_b2c_count"] = accounts_df["_is_b2c"].sum()
        else:
            # B2C mode is off - don't flag any rows
            leads_df["_is_b2c"] = False
            accounts_df["_is_b2c"] = False

        logger.info(
            "B2C domain detection: %d/%d leads (%.1f%%), %d/%d accounts (%.1f%%)",
            b2c_metrics["leads_b2c_count"],
            len(leads_df),
            100.0 * b2c_metrics["leads_b2c_count"] / max(1, len(leads_df)),
            b2c_metrics["accounts_b2c_count"],
            len(accounts_df),
            100.0 * b2c_metrics["accounts_b2c_count"] / max(1, len(accounts_df)),
        )

        # GET INTELLIGENT CONFIGURATION - NO HARDCODING!
        intelligent_config = await self.get_intelligent_config(
            source_df=leads_df,
            target_df=accounts_df,
            goal="lead_to_account",
            context={
                "organization_id": account_id,
                "high_precision_required": True,  # L2A needs high precision
                "field_count": len(lead_fields) + len(account_fields),
            },
        )

        # Config is already normalized from get_intelligent_config

        # Diagnostic logging to prevent regression
        logger.info(
            _s(
                f"Intel config type={type(intelligent_config).__name__} keys={sorted(list(intelligent_config.keys()) if isinstance(intelligent_config, dict) else [])}"
            )
        )

        # Validate shape
        self._assert_config_shape(intelligent_config)

        # Safe logging after normalization
        logger.info(
            _s(
                f"Intel config summary: mappings={len(intelligent_config.get('field_mappings', []))}, "
                f"threshold={intelligent_config.get('threshold')}, "
                f"apply_blocking={self._get_apply_blocking(intelligent_config)}"
            )
        )

        # Get normalized field mappings
        field_mappings = intelligent_config.get("field_mappings", [])

        # MANDATORY BASELINE ANCHORS - always merge, never replace
        BASELINE = [
            {
                "source_column": "email_domain",
                "reference_column": "website_domain",
                "algorithm": "Domain",
                "weight": 1.0,
                "is_anchor": True,
            },
            # Strong name corroboration (using WRatio for compatibility)
            {
                "source_column": "Company",
                "reference_column": "Name",
                "algorithm": "WRatio",
                "weight": 0.45,
            },
            # Light phone exact, if present
            {
                "source_column": "Phone",
                "reference_column": "Phone",
                "algorithm": "Exact",
                "weight": 0.10,
            },
        ]

        # Merge baseline with discovered mappings (baseline takes precedence)
        mappings = field_mappings + BASELINE
        intelligent_config["field_mappings"] = mappings

        # Ensure the engine has fields even when discovery is empty
        if not intelligent_config.get("fields"):
            intelligent_config["fields"] = mappings

        logger.info(
            _s(
                "Merged baseline anchors: %d discovery + %d baseline = %d total mappings"
            ),
            len(field_mappings),
            len(BASELINE),
            len(mappings),
        )

        # If discovery returned nothing, warn but continue with baseline
        if not field_mappings:
            logger.warning(
                "Intelligence returned no mappings - using baseline L2A mappings only"
            )

            # Also set a reasonable threshold if not provided
            if not intelligent_config.get("threshold"):
                intelligent_config["threshold"] = 0.75
                logger.info("Set baseline threshold: 0.75")

        # Validate field_mappings
        if not intelligent_config.get("field_mappings"):
            raise ValueError(
                "Intelligence returned no field_mappings and baseline injection failed"
            )
        if not isinstance(intelligent_config["field_mappings"], list):
            raise ValueError("field_mappings must be a list")

        logger.info(
            f"Intelligence API configured: "
            f"{len(intelligent_config['field_mappings'])} mappings, "
            f"threshold={intelligent_config['threshold']:.2f}, "
            f"blocking={intelligent_config.get('blocking', {}).get('apply_blocking', False)}"
        )

        # Helper to coerce field names to strings
        def _coerce_field_name(x: Any) -> Optional[str]:
            if isinstance(x, str):
                return x
            if isinstance(x, dict):
                # try common shapes, else reject
                for k in ("field", "name", "column", "path"):
                    v = x.get(k)
                    if isinstance(v, str):
                        return v
            return None

        # Normalize mappings to handle different formats from intelligence API
        def _normalize_mapping(m: dict) -> dict:
            # Accept several shapes from intelligence, but only strings
            src = (
                _coerce_field_name(m.get("source_column"))
                or _coerce_field_name(m.get("source"))
                or _coerce_field_name(m.get("sourceField"))
                or _coerce_field_name(m.get("field_source"))
            )
            ref = (
                _coerce_field_name(m.get("reference_column"))
                or _coerce_field_name(m.get("ref"))
                or _coerce_field_name(m.get("reference"))
                or _coerce_field_name(m.get("target"))
                or _coerce_field_name(m.get("targetField"))
            )
            if not src or not ref:
                raise ValueError(f"Bad mapping from intelligence: {m}")

            # Prefer a per-mapping algorithm if provided; otherwise try the global map
            algo = m.get("algorithm")
            if not algo:
                algomap = intelligent_config.get("algorithms") or {}
                algo = algomap.get(f"{src}->{ref}") or algomap.get(f"{src}|{ref}")

            return {
                "source_column": src,
                "reference_column": ref,
                "weight": m.get("weight", 0.5),
                "algorithm": algo,  # fine if None - Engine will fall back
            }

        # Define allowed field types for text matching
        # Note: 'id' fields rarely useful for fuzzy matching, may want to exclude or give low weight
        ALLOWED_TYPES = {
            "string",
            "email",
            "url",
            "phone",
            "textarea",
            "picklist",
            "multipicklist",
        }

        def string_like_fieldnames(schema: dict) -> set:
            """Extract field names that are text-like from SF schema"""
            return {
                f["name"]
                for f in schema.get("fields", [])
                if f.get("type") in ALLOWED_TYPES and not f.get("calculated")
            }

        def looks_textual(field: str, rows: list, min_non_empty: int = 3) -> bool:
            """Check if field contains textual data in sample"""
            non_empty = 0
            for r in rows[:50]:  # cheap sniff
                v = r.get(field)
                if v in (None, ""):
                    continue
                if isinstance(
                    v, (str, int, float)
                ):  # ints/floats will be str()'d by engine
                    non_empty += 1
                    if non_empty >= min_non_empty:
                        return True
                else:
                    # dict/list -> not textual for fuzzy algos
                    return False  # bail immediately on non-text
            return non_empty > 0  # require at least one value seen

        # Get text-like fields from schemas
        lead_text_fields = string_like_fieldnames(lead_schema)
        acct_text_fields = string_like_fieldnames(account_schema)

        # Add derived domain columns to field lists (they exist in DataFrames)
        if "email_domain" in leads_df.columns:
            lead_text_fields.add("email_domain")
        if "website_domain" in accounts_df.columns:
            acct_text_fields.add("website_domain")

        # Normalize all mappings first
        normalized_mappings = [
            _normalize_mapping(m) for m in intelligent_config["field_mappings"]
        ]

        # Filter to only usable text fields
        filtered_mappings = []
        for m in normalized_mappings:
            s, r = m["source_column"], m["reference_column"]

            # Skip known problematic fields
            if s.lower() == "attributes" or r.lower() == "attributes":
                logger.debug(f"Skipping attributes field in mapping: {s} -> {r}")
                continue

            # Check if fields are in text-like schema
            if s not in lead_text_fields or r not in acct_text_fields:
                logger.debug(f"Skipping non-text schema field: {s} -> {r}")
                continue

            # Check if fields contain textual data
            if not looks_textual(s, all_leads) or not looks_textual(r, accounts):
                logger.debug(f"Skipping non-textual data field: {s} -> {r}")
                continue

            filtered_mappings.append(m)

        # Service-side mapping adjustments (only if allowed)
        if ALLOW_SERVICE_OVERRIDES:
            # Prune bad L2A mappings and inject real anchors
            BAD_L2A = {
                ("Name", "Name"),
                ("PhotoUrl", "PhotoUrl"),
                ("Fax", "Fax"),
                # Lead->Account: these are weak anchors for matching
                ("Industry", "Industry"),
                ("Rating", "Rating"),
            }
            kept = []
            for m in filtered_mappings:
                pair = (m["source_column"], m["reference_column"])
                if pair in BAD_L2A:
                    logger.info("Dropping weak L2A mapping %s -> %s", *pair)
                    continue
                if pair == ("Phone", "Phone"):
                    # Phone is weak for L2A, reduce weight
                    m = {
                        **m,
                        "weight": min(0.25, m.get("weight", 0.5)),
                        "algorithm": "Exact",
                    }
                # Normalize Website mapping to domain-aware comparison with capped weight
                if pair in {
                    ("Website", "Website"),
                    ("Email", "Website"),
                    ("Email", "DHC_Provider_Website__c"),
                }:
                    m = {
                        **m,
                        "algorithm": "Domain",
                        "weight": min(0.55, m.get("weight", 0.55)),
                    }
                kept.append(m)
            filtered_mappings = kept

            # Add critical L2A mappings if missing
            pairs = {
                (m["source_column"], m["reference_column"]) for m in filtered_mappings
            }
            if (
                ("Company", "Name") not in pairs
                and "Company" in lead_text_fields
                and "Name" in acct_text_fields
            ):
                filtered_mappings.append(
                    {
                        "source_column": "Company",
                        "reference_column": "Name",
                        "weight": 0.65,
                        "algorithm": "Auto",  # Use Auto for global script support
                    }
                )
                logger.info("Added critical mapping: Company -> Name")

            if (
                ("Website", "Website") not in pairs
                and "Website" in lead_text_fields
                and "Website" in acct_text_fields
            ):
                filtered_mappings.append(
                    {
                        "source_column": "Website",
                        "reference_column": "Website",
                        "weight": 0.55,
                        "algorithm": "Domain",  # Use domain-aware comparison
                    }
                )
                logger.info("Added critical mapping: Website -> Website")

            # Use extracted domains for clean matching - ALWAYS add if columns exist
            if ("email_domain", "website_domain") not in pairs:
                # Check if we have the derived columns
                if (
                    "email_domain" in leads_df.columns
                    and "website_domain" in accounts_df.columns
                ):
                    # Always add the mapping - will be suppressed per-row for personal domains
                    filtered_mappings.append(
                        {
                            "source_column": "email_domain",
                            "reference_column": "website_domain",
                            "weight": 1.0,  # Strong anchor for business domains
                            "algorithm": "Domain",
                            "is_anchor": True,  # Mark as anchor for CERTAIN matches
                        }
                    )

                    # Log coverage for monitoring
                    email_coverage = (
                        leads_df["email_domain"].notna().sum() / len(leads_df)
                        if len(leads_df) > 0
                        else 0
                    )
                    website_coverage = (
                        accounts_df["website_domain"].notna().sum() / len(accounts_df)
                        if len(accounts_df) > 0
                        else 0
                    )
                    logger.info(
                        f"Added ANCHOR mapping: email_domain -> website_domain (coverage: {email_coverage:.1%} leads, {website_coverage:.1%} accounts)"
                    )

            # Per-row mapping hygiene: Suppress domain mappings for B2C rows (strict mode only)
            if b2c_mode == "strict":
                domain_mapping_cols = {
                    ("Website", "Website"),
                    ("Email", "Website"),
                    ("Domain", "Domain"),
                    ("Domain__c", "Domain__c"),
                    ("Email", "DHC_Provider_Website__c"),
                }

                # Create sanitized website column for strict mode
                if "Website" in leads_df.columns:
                    leads_df["WebsiteForMatch"] = leads_df["Website"].copy()
                    mask_b2c = leads_df["Website"].apply(
                        lambda v: is_b2c_with_whitelist(_registrable_domain(v))
                        if pd.notna(v)
                        else False
                    )
                    leads_df.loc[mask_b2c, "WebsiteForMatch"] = ""
                    b2c_metrics["website_suppressed_count"] = mask_b2c.sum()

                    # Update mappings to use sanitized column
                    for m in filtered_mappings:
                        if m["source_column"] == "Website":
                            m["source_column"] = "WebsiteForMatch"
                            b2c_metrics["mappings_suppressed"] += 1
                            logger.info(
                                "Strict mode: Using WebsiteForMatch to suppress %d B2C websites",
                                b2c_metrics["website_suppressed_count"],
                            )
        else:
            logger.info(
                "Strict mode: using Intelligence mappings as-is (no service-side edits)."
            )

        if not filtered_mappings:
            raise ValueError("No usable textual mappings after normalization/filtering")

        logger.info(
            "Mappings kept: %d%s",
            len(filtered_mappings),
            " (after service-side adjustments)" if ALLOW_SERVICE_OVERRIDES else "",
        )

        # Debug first values for troubleshooting
        if filtered_mappings and all_leads and accounts:
            first_map = filtered_mappings[0]
            s, r = first_map["source_column"], first_map["reference_column"]
            logger.debug(
                "Sample compare: lead[%s]=%r vs acct[%s]=%r (weight=%.2f, algo=%s)",
                s,
                all_leads[0].get(s),
                r,
                accounts[0].get(r),
                first_map.get("weight", 0.5),
                first_map.get("algorithm", "default"),
            )

            # Log all mappings with weights for debugging
            logger.debug(
                "All filtered mappings: %s",
                [
                    {
                        "src": m["source_column"],
                        "ref": m["reference_column"],
                        "weight": m.get("weight", 0.5),
                    }
                    for m in filtered_mappings
                ],
            )

        # Canonicalize algorithm names and ensure engine compatibility
        # Extended aliases map for common variations
        ALGO_ALIASES = {
            # WRatio family
            "wratio": "WRatio",
            "wr": "WRatio",
            "weighted": "WRatio",
            "weightedration": "WRatio",
            # TokenSet family
            "tokensetratio": "TokenSetRatio",
            "token_set_ratio": "TokenSetRatio",
            "tokenratio": "TokenSetRatio",
            "token": "TokenSetRatio",
            "fuzzy": "TokenSetRatio",
            "fuzz": "TokenSetRatio",
            "tokenset": "TokenSetRatio",
            # TokenSort family
            "tokensort": "TokenSortRatio",
            "tokensortratio": "TokenSortRatio",
            "sort": "TokenSortRatio",
            # Domain matching
            "domain": "Domain",
            "website_domain": "Domain",
            "email_domain": "Domain",
            # Exact matching
            "exact": "Exact",
            "eq": "Exact",
            "=": "Exact",
            "equals": "Exact",
            # Jaro variants
            "jaro": "JaroWinkler",
            "jaro_winkler": "JaroWinkler",
            "jarowinkler": "JaroWinkler",
            # Partial matching
            "partial": "PartialRatio",
            "partialratio": "PartialRatio",
            # Levenshtein
            "levenshtein": "Levenshtein",
            "lev": "Levenshtein",
            "edit": "Levenshtein",
            # QRatio
            "qratio": "QRatio",
            "quick": "QRatio",
            "qr": "QRatio",
        }

        # Algorithms actually supported by the engine
        SUPPORTED = {
            "WRatio",
            "TokenSetRatio",
            "TokenSortRatio",
            "Exact",
            "Domain",
            "JaroWinkler",
            "PartialRatio",
            "Levenshtein",
            "QRatio",
        }

        def _canon_algo(a: str) -> str:
            """Canonicalize algorithm names to match engine expectations"""
            if not a:
                return "WRatio"
            # Normalize: lowercase, remove spaces/dashes/underscores
            key = (
                str(a)
                .strip()
                .lower()
                .replace(" ", "")
                .replace("-", "")
                .replace("_", "")
            )
            out = ALGO_ALIASES.get(key, a)
            # Ensure we only return supported algorithms
            return out if out in SUPPORTED else "WRatio"

        # Fix algorithm names and add alternate keys for engine compatibility
        canonicalized_algos = []
        for m in filtered_mappings:
            if "algorithm" in m and m["algorithm"]:
                orig_algo = m["algorithm"]
                m["algorithm"] = _canon_algo(m["algorithm"])
                if orig_algo != m["algorithm"]:
                    canonicalized_algos.append(f"{orig_algo}->{m['algorithm']}")

            # Include alternate keys many engines look for
            m["source"] = m.get("source") or m.get("source_column")
            m["reference"] = m.get("reference") or m.get("reference_column")

            # Strip whitespace from all column names
            for k in ("source", "reference", "source_column", "reference_column"):
                if m.get(k) and isinstance(m[k], str):
                    m[k] = m[k].strip()

        # Log canonicalization summary once
        if canonicalized_algos:
            try:
                summary = (
                    ", ".join(canonicalized_algos[:3])
                    .encode("ascii", "replace")
                    .decode("ascii")
                )
            except Exception:
                summary = ", ".join(canonicalized_algos[:3])
            logger.info(
                "Canonicalized %d algorithms: %s",
                len(canonicalized_algos),
                summary,
            )

        # Log all mappings at INFO level for visibility
        for m in filtered_mappings:
            logger.info(
                "Mapping: Lead.%s -> Account.%s (w=%.2f, algo=%s)",
                m["source_column"],
                m["reference_column"],
                m.get("weight", 0.5),
                m.get("algorithm", "default"),
            )

        # Blocking configuration - validate intelligence decisions
        blocking_cfg = intelligent_config.get("blocking") or {}
        apply_blocking = bool(blocking_cfg.get("apply_blocking"))
        block_col = blocking_cfg.get("column") or (
            (blocking_cfg.get("keys") or [{}])[0]
        ).get("column")
        blocking_keys = []

        # --- BEGIN: alias-friendly blocking ---
        if apply_blocking and block_col:
            lead_has = any(
                f["name"] == block_col for f in lead_schema.get("fields", [])
            )
            acct_has = any(
                f["name"] == block_col for f in account_schema.get("fields", [])
            )

            # If the ref side lacks the exact name, try known aliases (e.g., Domain -> Domain_Append)
            ref_block_col = block_col
            if not acct_has:
                alias_map = {
                    "Domain": ["Domain_Append", "Website", "DHC_Provider_Website__c"],
                }
                for cand in alias_map.get(block_col, []):
                    if any(f["name"] == cand for f in account_schema.get("fields", [])):
                        ref_block_col = cand
                        acct_has = True
                        break

            if lead_has and acct_has:
                if blocking_cfg.get("keys"):
                    # preserve keys but override ref column if needed
                    blocking_keys = [
                        {
                            "source_column": block_col,
                            "reference_column": ref_block_col,
                            "variant": (
                                blocking_cfg.get("variant")
                                or blocking_cfg.get("keys", [{}])[0].get(
                                    "variant", "alnum"
                                )
                            ),
                            "key_length": blocking_cfg.get("key_length")
                            or blocking_cfg.get("keys", [{}])[0].get("key_length", 3),
                        }
                    ]
                else:
                    blocking_keys = [
                        {
                            "source_column": block_col,
                            "reference_column": ref_block_col,
                            "variant": blocking_cfg.get("variant", "alnum"),
                            "key_length": blocking_cfg.get("key_length", 3),
                        }
                    ]
                logger.info(
                    "Using intelligence blocking with alias: lead=%s, account=%s",
                    block_col,
                    ref_block_col,
                )
            else:
                apply_blocking = False
                blocking_keys = []
                logger.debug(
                    "Disabling blocking: intelligence picked %r but not found on both sides (even with aliases).",
                    block_col,
                )
        # --- END: alias-friendly blocking ---
        else:
            # No usable blocking column provided -> disable blocking to avoid inconsistent config
            apply_blocking = False
            blocking_keys = []
            logger.debug(
                "Disabling blocking: intelligence requested none or gave no column."
            )

        # Check if we have a domain anchor in mappings - if yes, force blocking on
        def _has_domain_anchor(maps: list) -> bool:
            for m in maps:
                alg = (m.get("algorithm") or "").lower()
                if alg in ("domain", "emaildomain"):
                    return True
            return False

        # Override blocking decision if we have a domain anchor
        if _has_domain_anchor(filtered_mappings):
            if not apply_blocking:
                logger.info("Enabling blocking due to domain anchor in mappings")
                apply_blocking = True
                # Engine will adapt or we can add blocking keys later

        # Override with ephemeral domain blocking if available
        if domain_blocking_active:
            apply_blocking = True
            blocking_keys = [
                {
                    "column": "__block_domain"  # Simple column name for both sides
                    # No key_length needed for exact domain matching
                }
            ]
            logger.info("Using ephemeral domain blocking on __block_domain column")

        # Log final blocking configuration
        logger.info(
            "Blocking config: apply_blocking=%s, keys=%s", apply_blocking, blocking_keys
        )

        # Build discovery policy from intelligent config with filtered mappings
        # Threshold handling with compliance mode
        thr = float(intelligent_config["threshold"])

        # Optional: Lower threshold just for engine path (not pre-pass)
        engine_thr = float(os.getenv("L2A_ENGINE_THRESHOLD", str(thr)))
        if engine_thr < thr:
            logger.info(
                _s(f"Using lower engine threshold: {engine_thr:.2f} (main: {thr:.2f})")
            )

        if ALLOW_SERVICE_OVERRIDES and threshold_override is not None:
            # Override mode: allow threshold changes
            original_override = float(threshold_override)
            # Clamp to sane range
            thr = max(0.50, min(0.99, original_override))
            if thr != original_override:
                logger.info(
                    "Threshold override %.2f clamped to %.2f (valid range: 0.50-0.99)",
                    original_override,
                    thr,
                )
            else:
                logger.info("Using request threshold override: %.2f", thr)
        elif threshold_override is not None:
            # Strict mode: ignore overrides
            logger.info(
                "Ignoring threshold_override=%.2f (strict compliance mode)",
                threshold_override,
            )
        elif ALLOW_SERVICE_OVERRIDES:
            # Check env vars only in override mode
            override = os.getenv("INTEL_THRESHOLD_OVERRIDE")
            ceiling = os.getenv("INTEL_THRESHOLD_CEILING")

            if override is not None:
                thr = float(override)  # force exact value for testing
                logger.info("Using env threshold override: %.2f", thr)
            elif ceiling is not None:
                thr = min(thr, float(ceiling))  # cap at ceiling for testing
                logger.info("Applied threshold ceiling: %.2f", thr)

        logger.info(
            "Effective threshold for run: %.2f (intelligence: %.2f)",
            thr,
            intelligent_config["threshold"],
        )

        # Get ensemble scoring preference - override from parameter or use policy setting
        if use_multi_algo is not None:
            # Explicit override from API call
            use_multi_flag = bool(use_multi_algo)
        else:
            # Use policy setting or default to False
            use_multi_flag = (
                bool(getattr(policy, "use_multi_algo", False)) if policy else False
            )

        discovery_policy = {
            "field_mappings": filtered_mappings,
            "threshold": thr,
            "source_id_field": "Id",
            "target_id_field": "Id",
            "algorithms": intelligent_config.get("algorithms", {}),
            # Many engines look for these:
            "apply_blocking": apply_blocking,
            "blocking_keys": blocking_keys,
            # Ensemble scoring configuration
            "use_multi_algo": use_multi_flag,
            "enable_multi_algo": use_multi_flag,  # Some engines use this name
            "include_field_score_details": True,  # Always get detailed scores for UI
            # And some older ones look for this shape:
            "blocking": {
                "apply_blocking": apply_blocking,
                "keys": blocking_keys,
            },
            "optimization_goal": "lead_to_account",
        }

        # Debug blocking configuration
        logger.info(
            "Blocking config: apply_blocking=%s, keys=%s", apply_blocking, blocking_keys
        )

        # --- BEGIN: Ephemeral domain-based blocking (Option 2) ---
        use_ephemeral = (
            os.getenv("L2A_USE_EPHEMERAL_BLOCKKEY", "true").lower() == "true"
        )
        min_cov = float(os.getenv("L2A_BLOCKKEY_MIN_COVERAGE", "0.10"))
        key_len = int(os.getenv("L2A_BLOCKKEY_KEYLEN", "3"))
        max_bucket_pct = float(
            os.getenv("L2A_MAX_BUCKET_PCT", "0.05")
        )  # 5% max bucket size

        blocking_keys_final = (
            blocking_keys[:] if isinstance(blocking_keys, list) else []
        )

        # Build fast sets once for overlap detection
        acct_domains_set = set()
        if "website_domain" in accounts_df.columns:
            acct_domains_set = set(
                x for x in accounts_df["website_domain"].dropna().astype(str).unique()
            )

        lead_domains_series = (
            leads_df["email_domain"].dropna().astype(str)
            if "email_domain" in leads_df.columns
            else None
        )
        overlap_exists = bool(
            lead_domains_series is not None
            and lead_domains_series.isin(acct_domains_set).any()
        )

        # CONDITIONAL domain blocking - only if there's overlap
        if domain_blocking_active and overlap_exists:
            logger.info(
                _s("Domain blocking: enabled (lead<->account domain overlap detected)")
            )
            apply_blocking = True
            # ephemeral blocking column
            leads_df["__block_domain"] = leads_df["email_domain"]
            accounts_df["__block_domain"] = accounts_df["website_domain"]
            blocking_keys_final.append("__block_domain")  # Fixed: string not dict
        else:
            if domain_blocking_active:
                logger.warning(
                    _s("Domain blocking: SKIPPED (no lead<->account domain overlap)")
                )

        # Always add a NAME-based fallback block to preserve fuzzy recall
        def _norm_prefix(s):
            if not s:
                return ""
            s = str(s).lower()
            # light cleanup; avoid heavy regex here
            for suf in (
                " inc",
                " llc",
                " ltd",
                " corp",
                " corporation",
                " company",
                " co",
            ):
                if s.endswith(suf):
                    s = s[: -len(suf)]
            return s[:8].strip()

        if "Company" in leads_df.columns and "Name" in accounts_df.columns:
            leads_df["__block_name"] = leads_df["Company"].map(_norm_prefix)
            accounts_df["__block_name"] = accounts_df["Name"].map(_norm_prefix)
            blocking_keys_final.append("__block_name")  # Fixed: string not dict
            logger.info(_s("Added name-based fallback blocking for fuzzy recall"))

        # Log blocking bucket health for visibility
        if apply_blocking or blocking_keys_final:
            bcols = [
                c for c in ("__block_domain", "__block_name") if c in leads_df.columns
            ]
            if bcols:
                for col in bcols:
                    if col in leads_df.columns:
                        top_buckets = leads_df[col].value_counts().head(5).to_dict()
                        logger.info(
                            _s(
                                f"Blocking column '{col}' top lead buckets: {top_buckets}"
                            )
                        )

            # Prove overlap for name blocking
            if (
                "__block_name" in leads_df.columns
                and "__block_name" in accounts_df.columns
            ):
                L = set(leads_df["__block_name"].dropna().astype(str).unique())
                A = set(accounts_df["__block_name"].dropna().astype(str).unique())
                overlap = len(L & A)
                logger.info(
                    _s(
                        f"Blocking coverage: __block_name overlap_keys={overlap} "
                        f"(lead_keys={len(L)}, acct_keys={len(A)})"
                    )
                )

                if overlap == 0:
                    logger.warning(
                        _s(
                            "WARNING: No name prefix overlap at k=8 - widening to 4-char prefix"
                        )
                    )

                    # Try a shorter prefix for this run
                    def _norm_prefix_short(s):
                        if not s:
                            return ""
                        s = str(s).lower()
                        for suf in (
                            " inc",
                            " llc",
                            " ltd",
                            " corp",
                            " corporation",
                            " company",
                            " co",
                        ):
                            if s.endswith(suf):
                                s = s[: -len(suf)]
                        return s[:4].strip()  # Shorter prefix (4 chars instead of 8)

                    # Re-compute with shorter prefix
                    leads_df["__block_name"] = leads_df["Company"].map(
                        _norm_prefix_short
                    )
                    accounts_df["__block_name"] = accounts_df["Name"].map(
                        _norm_prefix_short
                    )

                    # Check overlap again
                    L = set(leads_df["__block_name"].dropna().astype(str).unique())
                    A = set(accounts_df["__block_name"].dropna().astype(str).unique())
                    overlap = len(L & A)
                    logger.info(
                        _s(
                            f"After widening: __block_name overlap_keys={overlap} "
                            f"(lead_keys={len(L)}, acct_keys={len(A)})"
                        )
                    )

                    if overlap == 0 and os.getenv("L2A_NO_BLOCKING") == "1":
                        logger.warning(
                            _s("L2A_NO_BLOCKING=1: Disabling blocking for this run")
                        )
                        blocking_keys_final = []
                        apply_blocking = False

        # Add normalized name columns for better fuzzy matching
        def _norm_name(s: str) -> str:
            """Normalize company names for fuzzy matching"""
            if not s:
                return ""
            s = str(s).lower().strip()
            for suf in (
                " inc",
                " llc",
                " ltd",
                " corp",
                " corporation",
                " company",
                " co",
            ):
                if s.endswith(suf):
                    s = s[: -len(suf)]
            return "".join(ch for ch in s if ch.isalnum() or ch.isspace())

        # Add normalized columns to both dataframes
        if "Company" in leads_df.columns:
            leads_df["Company_norm"] = leads_df["Company"].map(_norm_name)
        if "Name" in accounts_df.columns:
            accounts_df["Name_norm"] = accounts_df["Name"].map(_norm_name)

        # Use the normalized names in mappings for better fuzzy matching
        if "Company_norm" in leads_df.columns and "Name_norm" in accounts_df.columns:
            intelligent_config.setdefault("field_mappings", []).append(
                {
                    "source": "Company_norm",
                    "reference": "Name_norm",
                    "algorithm": "WRatio",  # WRatio is widely supported by the engine
                    "weight": 0.40,  # Slightly higher to help borderline pairs
                }
            )
            logger.info(
                _s("Added normalized name mapping (WRatio) to improve fuzzy recall")
            )

        if apply_blocking and use_ephemeral:
            # Convert to DataFrames for ephemeral column creation
            source_df = pd.DataFrame(leads_to_process)
            reference_df = pd.DataFrame(accounts)

            # Re-add normalized columns to the new DataFrames
            if "Company" in source_df.columns:
                source_df["Company_norm"] = source_df["Company"].map(_norm_name)
            if "Name" in reference_df.columns:
                reference_df["Name_norm"] = reference_df["Name"].map(_norm_name)

            # Re-add derived domain columns (critical for domain mapping)
            def _safe_reg_domain_from_email(v):
                if pd.isna(v) or "@" not in str(v):
                    return None
                return _registrable_domain(str(v).split("@")[-1])

            def _safe_reg_domain(v):
                return _registrable_domain(v) if pd.notna(v) else None

            if "Email" in source_df.columns:
                source_df["email_domain"] = source_df["Email"].apply(
                    _safe_reg_domain_from_email
                )

            # Try multiple aliases for website domain
            for alias in (
                "Website",
                "Domain__c",
                "Domain_Append",
                "DHC_Provider_Website__c",
            ):
                if alias in reference_df.columns:
                    reference_df["website_domain"] = reference_df[alias].apply(
                        _safe_reg_domain
                    )
                    break  # first present alias wins

            # Determine if we should filter B2C for blocking
            filter_for_blocking = b2c_mode in ("strict", "block_only")

            # Auto-bump key length to 4 if B2C mode is off (to handle larger buckets)
            if b2c_mode == "off" and key_len == 3:
                key_len = 4
                logger.info("B2C mode=off: Auto-bumping key_length to 4 for stability")

            # Create ephemeral col on both sides if missing
            if "BlockKey_Domain3" not in source_df.columns:
                src_candidates = ["Domain", "Website", "Email"]
                # Track coverage before B2C filtering
                source_df["_domain_raw"] = source_df.apply(
                    lambda r: _registrable_domain(_pick(r, src_candidates)), axis=1
                )
                b2c_metrics["blockkey_coverage_before"] = (
                    source_df["_domain_raw"].str.len() > 0
                ).mean()

                # Generate blockkey with optional B2C filtering
                source_df["BlockKey_Domain3"] = source_df.apply(
                    lambda r: _blockkey3(
                        _pick(r, src_candidates),
                        key_len,
                        filter_b2c=filter_for_blocking,
                        b2c_check_fn=is_b2c_with_whitelist
                        if filter_for_blocking
                        else None,
                    ),
                    axis=1,
                )
                logger.debug("Built BlockKey_Domain3 for %d Leads", len(source_df))

            if "BlockKey_Domain3" not in reference_df.columns:
                ref_candidates = [
                    "Domain_Append",
                    "Website",
                    "Domain__c",
                    "CleanDomain__c",
                ]
                # Optional: precompute account domain raw for symmetry and reuse
                reference_df["_acct_domain_raw"] = reference_df.apply(
                    lambda r: _registrable_domain(_pick(r, ref_candidates)), axis=1
                )
                reference_df["BlockKey_Domain3"] = reference_df.apply(
                    lambda r: _blockkey3(
                        _pick(r, ref_candidates),
                        key_len,
                        filter_b2c=filter_for_blocking,
                        b2c_check_fn=is_b2c_with_whitelist
                        if filter_for_blocking
                        else None,
                    ),
                    axis=1,
                )
                logger.debug(
                    "Built BlockKey_Domain3 for %d Accounts", len(reference_df)
                )

            cov = (source_df["BlockKey_Domain3"].astype(str).str.len() > 0).mean()
            b2c_metrics["blockkey_coverage_after"] = cov
            logger.info(
                "Ephemeral block key coverage: before=%.1f%%, after B2C filter=%.1f%%",
                b2c_metrics["blockkey_coverage_before"] * 100.0,
                cov * 100.0,
            )

            # Safety rail: Check for mega-buckets
            if cov >= min_cov:
                # Analyze bucket sizes
                bucket_sizes = source_df["BlockKey_Domain3"].value_counts()
                if len(bucket_sizes) > 0:
                    largest_bucket = bucket_sizes.iloc[0]
                    largest_bucket_pct = largest_bucket / len(source_df)
                    b2c_metrics["top_bucket_pct"] = largest_bucket_pct

                    if largest_bucket_pct > max_bucket_pct:
                        # Auto-bump key length to 4 for this run
                        logger.warning(
                            "Largest bucket contains %.1f%% of records (>%.1f%%), bumping key_length to 4",
                            largest_bucket_pct * 100.0,
                            max_bucket_pct * 100.0,
                        )
                        key_len = 4
                        # Regenerate with longer key
                        source_df["BlockKey_Domain3"] = source_df.apply(
                            lambda r: _blockkey3(
                                _pick(r, src_candidates),
                                key_len,
                                filter_b2c=filter_for_blocking,
                                b2c_check_fn=is_b2c_with_whitelist
                                if filter_for_blocking
                                else None,
                            ),
                            axis=1,
                        )
                        reference_df["BlockKey_Domain3"] = reference_df.apply(
                            lambda r: _blockkey3(
                                _pick(r, ref_candidates),
                                key_len,
                                filter_b2c=filter_for_blocking,
                                b2c_check_fn=is_b2c_with_whitelist
                                if filter_for_blocking
                                else None,
                            ),
                            axis=1,
                        )
                        b2c_metrics["key_length_bumped"] = True
                    else:
                        logger.info(
                            "Largest bucket: %.1f%% of records (safe)",
                            largest_bucket_pct * 100.0,
                        )

                # Same column on both sides -> engine's simple blocking path
                blocking_keys_final = [
                    {
                        "column": "BlockKey_Domain3",
                        "variant": "alnum",
                        "key_length": key_len,
                    }
                ]
                logger.info(
                    "Using ephemeral blocking: column=BlockKey_Domain3, key_length=%d",
                    key_len,
                )
            else:
                logger.info(
                    "Insufficient coverage for ephemeral key (%.1f%% < %.1f%%); using alias-friendly plan if available",
                    cov * 100.0,
                    min_cov * 100.0,
                )
                # keep alias-friendly two-sided plan in blocking_keys_final if present
                if not blocking_keys_final:
                    apply_blocking = False
                    logger.info("Blocking disabled: no viable alias and low coverage.")

            # Convert back to list of dicts for engine
            leads_to_process = source_df.to_dict("records")
            accounts = reference_df.to_dict("records")
        else:
            blocking_keys_final = blocking_keys
            # Ensure all derived columns are in records even without ephemeral blocking
            # The domain columns and normalized names are already in leads_df/accounts_df
            # Just convert to records to preserve them
            leads_to_process = leads_df.to_dict("records")
            accounts = accounts_df.to_dict("records")
        # --- END: Ephemeral domain-based blocking (Option 2) ---

        # Normalize blocking keys to dicts before calling the engine
        def _normalize_blocking_keys(keys):
            """Convert string keys to dict format for engine compatibility"""
            out = []
            for k in keys or []:
                if isinstance(k, str):
                    out.append({"column": k, "variant": "alnum", "key_length": 3})
                elif isinstance(k, dict):
                    out.append(k)
            return out

        # Update discovery policy with normalized blocking configuration
        discovery_policy["blocking_keys"] = _normalize_blocking_keys(
            blocking_keys_final
        )
        discovery_policy["apply_blocking"] = apply_blocking

        # --- Determine effective threshold BEFORE the engine call ---
        # 1) collect candidates
        req_override = threshold_override  # Already passed as parameter
        intel_threshold = None
        try:
            intel_threshold = float(intelligent_config.get("threshold"))
        except Exception:
            intel_threshold = None

        # 2) final choice with sensible fallback
        DEFAULT_THRESHOLD = 0.80

        # Choose the floor that the engine should use
        effective_threshold = min(
            float(os.getenv("L2A_ENGINE_THRESHOLD", str(thr))), float(thr)
        )

        # Override with request if provided
        if req_override is not None:
            effective_threshold = req_override
        elif intel_threshold is not None:
            effective_threshold = min(
                float(os.getenv("L2A_ENGINE_THRESHOLD", str(intel_threshold))),
                float(intel_threshold),
            )

        logger.info(
            _s(
                f"Lowering engine threshold to {effective_threshold:.2f} (main: {thr:.2f})"
            )
        )

        # Track timing for engine call
        t0 = time.perf_counter()

        # ============= EXACT PRE-PASS (before engine) =============
        # Define personal domains to exclude from certain logic
        PERSONAL = {
            "gmail.com",
            "yahoo.com",
            "outlook.com",
            "aol.com",
            "hotmail.com",
            "icloud.com",
            "msn.com",
            "live.com",
            "yandex.com",
            "qq.com",
            "protonmail.com",
        }

        # Create domain lookups
        lead_domains = {}
        acct_domains = {}
        if "email_domain" in leads_df.columns:
            lead_domains = dict(
                zip(leads_df.get("Id", leads_df.index), leads_df["email_domain"])
            )
        if "website_domain" in accounts_df.columns:
            acct_domains = dict(
                zip(
                    accounts_df.get("Id", accounts_df.index),
                    accounts_df["website_domain"],
                )
            )

        exact_suggestions = []
        remaining_lead_ids = set(l["Id"] for l in leads_to_process)

        # Helper to normalize company names
        def normalize_company_name(s):
            """Normalize company name: lower, strip punct & legal suffixes"""
            if not s:
                return ""
            s = str(s).lower().strip()
            # Remove common legal suffixes
            for suffix in [
                " inc",
                " llc",
                " ltd",
                " corp",
                " corporation",
                " company",
                " co",
            ]:
                if s.endswith(suffix):
                    s = s[: -len(suffix)].strip()
            # Remove punctuation
            import string

            s = s.translate(str.maketrans("", "", string.punctuation))
            return s.strip()

        # A) EXACT DOMAIN pre-pass
        if (
            "email_domain" in leads_df.columns
            and "website_domain" in accounts_df.columns
        ):
            # Filter to remaining leads only
            leads_for_domain = leads_df[leads_df["Id"].isin(remaining_lead_ids)].copy()

            # Apply personal domain filter if enabled
            if settings.PERSONAL_DOMAIN_FILTER:
                # Filter out leads with personal email domains
                leads_for_domain = leads_for_domain[
                    ~leads_for_domain["email_domain"].isin(PERSONAL)
                ]
                logger.info(
                    f"Personal domain filter: excluded {len(leads_df) - len(leads_for_domain)} leads with personal domains"
                )

            dom_exact = leads_for_domain.merge(
                accounts_df,
                left_on="email_domain",
                right_on="website_domain",
                how="inner",
                suffixes=("_lead", "_acct"),
            )

            for _, row in dom_exact.iterrows():
                exact_suggestions.append(
                    {
                        "lead_id": row["Id_lead"],
                        "account_id": row["Id_acct"],
                        "score": 1.0,
                        "tier": "CERTAIN",
                        "match_type": "EXACT_DOMAIN",
                        "has_exact_anchor": True,
                        "field_score_details": [
                            {
                                "source_field": "email_domain",
                                "reference_field": "website_domain",
                                "score": 1.0,
                                "algorithm": "exact",
                                "relation": "EXACT",
                            }
                        ],
                    }
                )
                remaining_lead_ids.discard(row["Id_lead"])

            logger.info(f"EXACT DOMAIN pre-pass: {len(dom_exact)} suggestions")

        # B) EXACT NAME pre-pass (on remaining leads)
        if (
            remaining_lead_ids
            and "Company" in leads_df.columns
            and "Name" in accounts_df.columns
        ):
            # Add normalized name columns
            leads_df["name_norm"] = leads_df["Company"].apply(normalize_company_name)
            accounts_df["name_norm"] = accounts_df["Name"].apply(normalize_company_name)

            # Filter to remaining leads only
            leads_for_name = leads_df[leads_df["Id"].isin(remaining_lead_ids)].copy()

            name_exact = leads_for_name[leads_for_name["name_norm"] != ""].merge(
                accounts_df[accounts_df["name_norm"] != ""],
                on="name_norm",
                how="inner",
                suffixes=("_lead", "_acct"),
            )

            for _, row in name_exact.iterrows():
                # Determine tier based on email domain
                lead_domain = (
                    lead_domains.get(row["Id_lead"])
                    if "email_domain" in leads_df.columns
                    else None
                )
                tier = (
                    "LIKELY"
                    if lead_domain and lead_domain not in PERSONAL
                    else "CERTAIN"
                )

                exact_suggestions.append(
                    {
                        "lead_id": row["Id_lead"],
                        "account_id": row["Id_acct"],
                        "score": 0.95,
                        "tier": tier,
                        "match_type": "EXACT_NAME",
                        "has_exact_anchor": False,
                        "field_score_details": [
                            {
                                "source_field": "Company",
                                "reference_field": "Name",
                                "score": 1.0,
                                "algorithm": "exact_normalized",
                            }
                        ],
                    }
                )
                remaining_lead_ids.discard(row["Id_lead"])

            logger.info(f"EXACT NAME pre-pass: {len(name_exact)} suggestions")

        # Update leads_to_process to exclude exact matches
        leads_to_process = [
            l for l in leads_to_process if l["Id"] in remaining_lead_ids
        ]
        logger.info(
            f"After exact pre-pass: {len(exact_suggestions)} exact matches, {len(leads_to_process)} leads remaining"
        )

        # Save exact suggestions to database
        if exact_suggestions:
            exact_l2a_suggestions = []
            for exact_match in exact_suggestions:
                suggestion = L2ASuggestion(
                    policy_id=policy.id,
                    lead_id=exact_match["lead_id"],
                    account_id=exact_match["account_id"],
                    score=exact_match["score"],
                    tier=exact_match["tier"],
                    reasons={
                        "exact_match": 1.0,
                        "match_type": exact_match.get("match_type", "EXACT"),
                    },
                    field_score_details=exact_match.get("field_score_details"),
                    explanation=f"Exact match: {exact_match.get('match_type', 'EXACT')}",
                    status="pending",
                    reason_code="exact",
                    registry_version=_reg_version(),
                    last_verified_at=datetime.now(),
                )
                exact_l2a_suggestions.append(suggestion)

            # Batch save exact suggestions
            self.db.add_all(exact_l2a_suggestions)
            await self.db.commit()
            logger.info(
                f"Saved {len(exact_l2a_suggestions)} exact match suggestions to database"
            )

        # Run diagnostic fuzzy (QA only, does not affect decisions)
        _ = self._run_diagnostic_fuzzy(exact_suggestions, accounts, "lead_to_account")

        if progress_cb:
            progress_cb(0.70, "Running intelligent matching")

        # Adjust anchor weight if coverage is low
        coverage = intelligent_config.get("coverage", {})
        lead_email_corp_pct = coverage.get("lead_email_corporate_pct", 1.0)
        acct_website_pct = coverage.get("acct_website_pct", 1.0)

        if lead_email_corp_pct < 0.6 or acct_website_pct < 0.6:
            logger.info(
                "Low coverage detected (lead_corp=%.1f%%, acct_web=%.1f%%), adjusting weights",
                lead_email_corp_pct * 100,
                acct_website_pct * 100,
            )
            # Adjust domain anchor weight if coverage is poor
            for m in discovery_policy.get("field_mappings", []):
                if m.get("source") == "Email" and m.get("algorithm", "").lower() in (
                    "domain",
                    "emaildomain",
                ):
                    original_weight = m.get("weight", 1.0)
                    m["weight"] = 0.50
                    logger.info(
                        "Reduced Email->Domain weight from %.2f to 0.50 due to low coverage",
                        original_weight,
                    )

        # Build compatibility kwargs based on engine's actual signature
        import inspect

        # Decide what we want if available
        want_field_details = (
            False  # run() returns a summary; details shown later on list/explain
        )
        want_explanations = (
            True  # explanations useful if engine can produce them cheaply
        )

        # Introspect the engine function
        sig = inspect.signature(self.engine.match_salesforce_records)
        accepted = set(sig.parameters.keys())

        # Build a compatibility kwargs set ONLY with supported params
        engine_kwargs = {}

        # Prefer a single include list if supported
        include_tokens = []
        if "include" in accepted:
            if want_explanations:
                include_tokens.append("explanations")
            if want_field_details:
                include_tokens.append("field_details")
            if include_tokens:
                engine_kwargs["include"] = include_tokens
        else:
            # Otherwise set specific knobs if they exist
            if want_explanations and "include_explanations" in accepted:
                engine_kwargs["include_explanations"] = True
            if want_explanations and "return_explanations" in accepted:
                engine_kwargs["return_explanations"] = True
            if want_field_details and "return_field_details" in accepted:
                engine_kwargs["return_field_details"] = True
            # NOTE: newer services sometimes used 'explain_scores'; map it only if supported
            if want_explanations and "explain_scores" in accepted:
                engine_kwargs["explain_scores"] = True

        # Check if threshold is supported
        if "threshold" in accepted:
            engine_kwargs["threshold"] = effective_threshold

        # Drop mappings that point at missing or empty columns
        def _has_values(df, col, min_non_empty=3):
            """Check if column exists and has minimum non-empty values"""
            if isinstance(df, pd.DataFrame):
                return col in df.columns and df[col].notna().sum() >= min_non_empty
            else:
                # Handle list of dicts
                non_empty = sum(
                    1 for r in df if r.get(col) is not None and str(r.get(col)).strip()
                )
                return non_empty >= min_non_empty

        # Convert to DataFrames for validation if needed
        source_df_check = (
            pd.DataFrame(leads_to_process)
            if isinstance(leads_to_process, list)
            else leads_to_process
        )
        reference_df_check = (
            pd.DataFrame(accounts) if isinstance(accounts, list) else accounts
        )

        usable = []
        for m in discovery_policy.get("field_mappings", []):
            s = m.get("source") or m.get("source_column")
            r = m.get("reference") or m.get("reference_column")
            if _has_values(source_df_check, s) and _has_values(reference_df_check, r):
                usable.append(m)
            else:
                logger.warning(
                    _s(f"Dropping mapping with missing/empty columns: {s} -> {r}")
                )

        discovery_policy["field_mappings"] = usable

        if not usable:
            # Last-resort guard: keep a single name mapping so engine can't return 0
            logger.warning(_s("No usable mappings found! Using fallback name mapping"))
            discovery_policy["field_mappings"] = [
                {
                    "source": "Company_norm",
                    "reference": "Name_norm",
                    "algorithm": "WRatio",
                    "weight": 0.8,
                }
            ]

        # Detailed engine payload check
        need = {
            (
                m.get("source") or m.get("source_column"),
                m.get("reference") or m.get("reference_column"),
                m.get("algorithm"),
            )
            for m in discovery_policy["field_mappings"]
        }

        def _col_ok(df, c):
            if isinstance(df, pd.DataFrame):
                return c in df.columns and df[c].notna().sum()
            else:
                return sum(1 for r in df if r.get(c) is not None)

        payload_check = [
            {
                "src": s,
                "src_nonempty": int(_col_ok(source_df_check, s)),
                "ref": r,
                "ref_nonempty": int(_col_ok(reference_df_check, r)),
                "algo": a,
            }
            for (s, r, a) in need
        ]

        logger.info(_s(f"Engine payload check: {payload_check}"))

        # Log engine payload for verification
        logger.info(
            _s(
                f"Engine payload: fields={len(discovery_policy.get('field_mappings', []))} "
                f"block_keys={discovery_policy.get('blocking_keys')} apply={discovery_policy.get('apply_blocking')} "
                f"thr={engine_kwargs.get('threshold')}"
            )
        )

        logger.info(
            "EngineClient.match_salesforce_records() compat kwargs: %s", engine_kwargs
        )

        # Call the engine with safe kwargs (wrapped in try/finally for cleanup)
        try:
            result = await self.engine.match_salesforce_records(
                source_records=leads_to_process,
                target_records=accounts,
                discovery_policy=discovery_policy,
                batch_size=1000,
                **engine_kwargs,
            )

            # Use the coercion helper to handle any result type
            pairs_list = self._coerce_pairs_from_result(result)
            logger.info("Engine found %d matches using intelligence", len(pairs_list))

            # HARD NEGATIVES: Remove domain mismatches
            if lead_domains and acct_domains:
                # Use registry instead of hard-coded aliases
                from .domain_family import get_registry, DomainRelation

                registry = get_registry()

            filtered_pairs = []
            removed_count = 0

            for p in pairs_list:
                lead_id = p.get("source_id") or p.get("lead_id")
                acct_id = p.get("target_id") or p.get("account_id")

                lead_domain = lead_domains.get(lead_id)
                acct_domain = acct_domains.get(acct_id)

                # Check for domain mismatch (excluding personal emails)
                if lead_domain and acct_domain:
                    lead_domain_lower = lead_domain.lower()
                    acct_domain_lower = acct_domain.lower()

                    # Skip hard negative for personal domains
                    if lead_domain_lower in PERSONAL:
                        filtered_pairs.append(p)
                        continue

                    # Hard negative if not EXACT or FAMILY in registry
                    rel, fam = registry.relation(lead_domain_lower, acct_domain_lower)
                    _track_domain_lookup(
                        lead_domain_lower, rel != DomainRelation.NONE, "domain_filter"
                    )
                    _track_domain_lookup(
                        acct_domain_lower, rel != DomainRelation.NONE, "domain_filter"
                    )
                    if rel == DomainRelation.NONE:
                        removed_count += 1
                        logger.debug(
                            f"Hard negative: {lead_domain} != {acct_domain} for {lead_id} -> {acct_id}"
                        )
                        continue  # Skip this pair

                filtered_pairs.append(p)

            if removed_count > 0:
                logger.info(
                    f"Removed {removed_count} domain mismatches via hard negatives"
                )
            pairs_list = filtered_pairs

        finally:
            # CLEANUP: Always remove ephemeral blocking columns, even if engine throws
            for df in [leads_df, accounts_df]:
                if "__block_domain" in df.columns:
                    del df["__block_domain"]
            logger.debug("Cleaned up ephemeral blocking columns")

        # Normalize evidence shape for FE (arrays even if empty) and normalize scores to 0..1
        for p in pairs_list:
            if isinstance(p, dict):
                fsd = (
                    p.get("field_score_details")
                    or p.get("field_details")
                    or p.get("details")
                    or []
                )

                # Normalize field scores to 0..1 before persisting and add domain relations
                norm_fsd = []
                registry = get_registry()

                for d in fsd if isinstance(fsd, list) else []:
                    if isinstance(d, dict):
                        nd = dict(d)
                        try:
                            s = float(nd.get("score") or 0.0)
                            nd["score"] = (
                                s / 100.0 if s > 1.0 else max(0.0, min(1.0, s))
                            )
                        except Exception:
                            nd["score"] = 0.0

                        # Add domain relation if this is a domain field
                        src_col = nd.get("source_column") or nd.get("source_field")
                        ref_col = nd.get("reference_column") or nd.get(
                            "reference_field"
                        )
                        algo = nd.get("algorithm", "").lower()

                        # Check if this is a domain-related field
                        is_domain_field = (
                            algo == "domain"
                            or (src_col and "domain" in src_col.lower())
                            or (ref_col and "domain" in ref_col.lower())
                            or (
                                src_col
                                and src_col.lower() == "email"
                                and ref_col
                                and "website" in ref_col.lower()
                            )
                        )

                        if is_domain_field:
                            # Get the actual values from the pair
                            lead_id = p.get("source_id") or p.get("lead_id")
                            acct_id = p.get("target_id") or p.get("account_id")

                            # Look up domain values
                            lead_domain = None
                            acct_domain = None

                            if lead_id and "lead_domains" in locals():
                                lead_domain = lead_domains.get(lead_id)
                            if acct_id and "acct_domains" in locals():
                                acct_domain = acct_domains.get(acct_id)

                            # Compute relation
                            if lead_domain and acct_domain:
                                relation, family_id = registry.relation(
                                    lead_domain, acct_domain
                                )
                                _track_domain_lookup(
                                    lead_domain,
                                    relation != DomainRelation.NONE,
                                    "score_details",
                                )
                                _track_domain_lookup(
                                    acct_domain,
                                    relation != DomainRelation.NONE,
                                    "score_details",
                                )
                                nd["relation"] = relation.value
                                if family_id:
                                    nd["family_id"] = family_id

                        norm_fsd.append(nd)

                p["field_score_details"] = norm_fsd

                exps = p.get("explanations") or []
                p["explanations"] = exps if isinstance(exps, list) else []

        # Handle case with no matches gracefully
        if not pairs_list:
            # Return normal run summary, not a 500
            return {
                "policy_id": str(policy.id),
                "threshold_used": effective_threshold,
                "total_leads": len(leads_to_process),
                "total_accounts": len(accounts),
                "suggestions_created": 0,
                "duration_ms": int((time.perf_counter() - t0) * 1000),
                "status": "success",
                "message": "No matches found with current threshold",
            }

        # Normalize score column name in pairs_list
        if pairs_list:
            score_candidates = [
                "score",
                "match_score",
                "similarity",
                "confidence",
                "confidence_score",
                "sim",
                "ratio",
            ]

            # Find and normalize score field in the dictionaries
            sample_pair = pairs_list[0] if pairs_list else {}
            logger.info(
                "Sample pair keys: %s", list(sample_pair.keys()) if sample_pair else []
            )

            score_key = next((k for k in score_candidates if k in sample_pair), None)

            if score_key and score_key != "score":
                logger.info("Normalizing score field from '%s' to 'score'", score_key)
                for pair in pairs_list:
                    if score_key in pair:
                        pair["score"] = pair.pop(score_key)
            elif not score_key:
                logger.warning(
                    "No score field found in pairs, adding default score=0.0"
                )
                for pair in pairs_list:
                    pair["score"] = 0.0

            # Check if scores need normalization (0-100 to 0-1)
            max_score = max((p.get("score", 0) for p in pairs_list), default=0)
            if max_score > 1.5:
                logger.info(
                    "Normalizing scores from 0-100 to 0-1 range (max=%.2f)", max_score
                )
                for pair in pairs_list:
                    if "score" in pair:
                        pair["score"] = pair["score"] / 100.0
                max_score = max((p.get("score", 0) for p in pairs_list), default=0)
                logger.info("After normalization: Max score=%.2f", max_score)
        else:
            logger.info("No matches found - check threshold and mappings")

        # Also check stats for additional info
        if hasattr(result, "stats") and result.stats:
            if "max_score" in result.stats:
                logger.info("Stats max_score: %.2f", result.stats["max_score"])

        # Score incumbents if in upgrade mode
        algorithms = intelligent_config.get("algorithms") or {}
        model_id = intelligent_config.get("model_id")
        incumbent_scores = {}
        if audit_mode == "upgrade" and incumbents and pairs_list:
            # Extract scores for existing incumbents from match results
            for match in pairs_list:
                lead_id = (
                    match.get("s_id") or match.get("source_id") or match.get("lead_id")
                )
                account_id = (
                    match.get("r_id")
                    or match.get("target_id")
                    or match.get("account_id")
                )
                if lead_id in incumbents and incumbents[lead_id] == account_id:
                    incumbent_scores[lead_id] = float(match.get("score", 0))

            logger.info(
                f"Found scores for {len(incumbent_scores)}/{len(incumbents)} incumbents in match results"
            )

            # FIX 1: Rescore missing incumbents without DataFrame dependencies
            missing_incumbent_scores = [
                lid for lid in incumbents if lid not in incumbent_scores
            ]
            if missing_incumbent_scores:
                logger.info(
                    f"Rescoring {len(missing_incumbent_scores)} incumbents missed by blocking"
                )

                BATCH_SIZE = 50
                for i in range(0, len(missing_incumbent_scores), BATCH_SIZE):
                    chunk = missing_incumbent_scores[i : i + BATCH_SIZE]

                    # Build pair-specific subsets (no DataFrame dependencies)
                    src_subset = [s for s in leads_to_process if s["Id"] in chunk]
                    tgt_ids = {incumbents[s["Id"]] for s in src_subset}
                    tgt_subset = [a for a in accounts if a["Id"] in tgt_ids]

                    # Sanity check to prevent empty runs
                    if not src_subset or not tgt_subset:
                        logger.warning(
                            f"Empty subset for rescore batch {i//BATCH_SIZE}"
                        )
                        continue

                    # Disable blocking completely for the probe
                    probe_policy = {
                        **intelligent_config,
                        "apply_blocking": False,
                        "blocking": {"apply_blocking": False, "keys": []},
                        "blocking_keys": [],
                    }

                    try:
                        # Convert to DataFrames for engine (but built from records, not indexed)
                        probe_leads_df = pd.DataFrame(src_subset)
                        probe_accounts_df = pd.DataFrame(tgt_subset)
                        probe_leads_df.set_index("Id", inplace=True)
                        probe_accounts_df.set_index("Id", inplace=True)

                        probe = await self.engine.match_async(
                            source=probe_leads_df,
                            reference=probe_accounts_df,
                            mappings=filtered_mappings,
                            blocking_rules=None,  # No blocking
                            algorithms=algorithms,
                            thresholds={"confidence": thr},
                            model_id=model_id,
                            top_k=1,  # Only need the specific pair score
                            progress_callback=None,
                            return_field_details=True,  # Always return field details for reasons
                            explain_scores=True,
                        )

                        # Fill incumbent_scores ONLY for exact incumbent pairs
                        incumbent_evidence = {}  # POLISH: Track evidence too
                        if hasattr(probe, "matches") and not probe.matches.empty:
                            for _, r in probe.matches.iterrows():
                                lead_id = (
                                    r.get("s_id")
                                    if "s_id" in r.index
                                    else r.get("source_id")
                                )
                                account_id = (
                                    r.get("r_id")
                                    if "r_id" in r.index
                                    else r.get("target_id")
                                )

                                # Critical: only store if this is the actual incumbent pair
                                if (
                                    lead_id in incumbents
                                    and incumbents[lead_id] == account_id
                                ):
                                    incumbent_scores[lead_id] = float(
                                        r["score"] if "score" in r.index else 0
                                    )
                                    # POLISH: Also capture evidence
                                    if "field_score_details" in r.index:
                                        incumbent_evidence[lead_id] = r[
                                            "field_score_details"
                                        ]
                                    logger.debug(
                                        f"Recovered incumbent score for {lead_id}: {incumbent_scores[lead_id]:.2f}"
                                    )
                    except Exception as e:
                        logger.error(f"Failed to rescore batch: {e}")
                        continue

                logger.info(
                    f"After rescoring: {len(incumbent_scores)}/{len(incumbents)} incumbents have scores"
                )

        if progress_cb:
            progress_cb(0.90, "Creating suggestions")

        # Already extracted upgrade settings upfront (FIX 5)

        # Convert results to suggestions
        suggestions = []
        suggestions_created = 0
        certain_count = 0
        upgrades_suggested = 0

        # Globalization v1 observability tracking
        suggestions_with_reasons = 0

        # Combine exact pre-pass suggestions with engine results
        combined_pairs = (
            exact_suggestions + pairs_list
            if "exact_suggestions" in locals()
            else pairs_list
        )
        logger.info(
            f"Combined results: {len(exact_suggestions if 'exact_suggestions' in locals() else [])} exact + {len(pairs_list)} engine = {len(combined_pairs)} total"
        )

        # Normalize pair keys so all pairs use consistent lead_id/account_id
        # Now using shared utility function
        normalized = []
        for p in combined_pairs:
            l_id, a_id = extract_id_from_dict(p, mode="l2a")
            if not l_id or not a_id:
                logger.debug(
                    "Dropping pair without ids: %s",
                    {
                        k: p.get(k)
                        for k in (
                            "lead_id",
                            "l_id",
                            "s_id",
                            "source_id",
                            "account_id",
                            "r_id",
                            "target_id",
                        )
                    },
                )
                continue
            q = dict(p)
            q["lead_id"] = l_id
            q["account_id"] = a_id
            normalized.append(q)

        combined_pairs = normalized
        logger.info(
            f"After ID normalization: {len(combined_pairs)} pairs with valid IDs"
        )

        # Build best_by_lead map to ensure one suggestion per lead
        best_by_lead = {}
        if combined_pairs:
            # Filter by threshold
            # Use engine_thr for engine-produced pairs, thr for exact pre-pass
            filtered = []
            for p in combined_pairs:
                score = p.get("score", 0)
                is_exact = p.get("match_type", "").startswith("EXACT_")
                threshold_to_use = thr if is_exact else engine_thr
                if score >= threshold_to_use:
                    filtered.append(p)

            logger.info(
                _s(
                    f"After threshold {thr:.2f} (exact) / {engine_thr:.2f} (fuzzy): {len(filtered)}/{len(combined_pairs)} kept"
                )
            )

            # Group by source (lead) ID and take highest score
            for row in filtered:
                lead_id = row.get("s_id") or row.get("source_id") or row.get("lead_id")
                if not lead_id:
                    continue

                current_best = best_by_lead.get(lead_id)
                if not current_best or row.get("score", 0) > current_best.get(
                    "score", 0
                ):
                    best_by_lead[lead_id] = row

            # Sample log for debugging
            if len(best_by_lead) > 0:
                logger.info("Found best matches for %d leads", len(best_by_lead))

        # Iterate over best matches only (one per lead)
        first_match_logged = False
        for lead_id, match in best_by_lead.items():
            # FIX 2 (continued): Prevent duplicate pending upgrades
            # Skip if we already have a pending suggestion for this lead
            if audit_mode == "upgrade" and lead_id in existing_lead_ids:
                logger.debug(f"Skipping {lead_id}: pending upgrade already exists")
                continue

            # Debug: log the first match to see what columns are available
            if not first_match_logged:
                logger.info(
                    "DEBUG: First match keys: %s",
                    list(match.keys()) if isinstance(match, dict) else [],
                )
                logger.info(
                    "DEBUG: First match values sample: lead_id=%s, s_id=%s, r_id=%s",
                    lead_id,
                    match.get("s_id", "NOT FOUND"),
                    match.get("r_id", "NOT FOUND"),
                )
                first_match_logged = True

            # Access dict values
            score = float(match.get("score", 0))

            # Use the effective threshold (which may be overridden for testing)
            if score < thr:
                continue

            # Get field_score_details if available from engine
            raw_fsd = match.get("field_score_details", [])

            # Initial tier from score
            tier = self.engine._get_tier(score)

            # TIER GATE: No high score without a true anchor
            if tier == "CERTAIN":
                # Get account_id first (needed for domain lookup)
                account_id = (
                    match.get("r_id")
                    or match.get("target_id")
                    or match.get("account_id")
                )

                # Check for true anchors
                has_exact_domain = False
                has_phone_exact = False
                guardrail_reason = None

                # Get domains for this pair
                lead_domain = lead_domains.get(lead_id) if lead_domains else None
                acct_domain = acct_domains.get(account_id) if acct_domains else None

                # Check domain relation via registry
                if lead_domain and acct_domain:
                    rel, fam = registry.relation(lead_domain, acct_domain)
                    _track_domain_lookup(
                        lead_domain, rel != DomainRelation.NONE, "guardrail_check"
                    )
                    _track_domain_lookup(
                        acct_domain, rel != DomainRelation.NONE, "guardrail_check"
                    )
                    if rel == DomainRelation.EXACT:
                        has_exact_domain = True
                    elif rel == DomainRelation.FAMILY:
                        # FAMILY is not a true anchor for CERTAIN tier
                        guardrail_reason = "domain_family_only"

                # Check field scores for rare phone exact (must be truly exact, not fuzzy)
                for detail in raw_fsd:
                    if isinstance(detail, dict):
                        src_field = (
                            detail.get("source_field") or detail.get("field") or ""
                        ).lower()
                        algorithm = (detail.get("algorithm") or "").lower()
                        score = float(detail.get("score", 0))

                        # Phone must be exact match with perfect score
                        if (
                            src_field == "phone"
                            and algorithm == "exact"
                            and score >= 0.99
                        ):
                            has_phone_exact = True
                            break

                # Downgrade if no true anchor found
                if not has_exact_domain and not has_phone_exact:
                    tier = "LIKELY"  # Downgrade from CERTAIN
                    if not guardrail_reason:
                        guardrail_reason = "no_true_anchor"
                    match["guardrail"] = "tier_downgrade"
                    match["guardrail_reason"] = guardrail_reason
                    logger.debug(
                        f"Downgraded {lead_id}->{account_id} from CERTAIN to LIKELY ({guardrail_reason})"
                    )
                else:
                    certain_count += 1
            else:
                # Not CERTAIN, keep original tier
                pass

            # Build reasons dict and persist field_score_details (attach relation if domain evidence present)
            feature_scores = match.get("feature_scores", {})
            reasons = {}

            # If we have field_score_details, also build reasons dict from it
            if raw_fsd:
                # Normalize field_score_details and attach domain relations
                if isinstance(raw_fsd, list):
                    fsd_norm = []
                    for d in raw_fsd:
                        if not isinstance(d, dict):
                            continue
                        # Attach typed domain relation (best-effort)
                        fld = (
                            d.get("field")
                            or d.get("source_field")
                            or d.get("source_column")
                            or ""
                        ).lower()
                        if fld in {"domain", "email_domain", "website_domain"}:
                            # Get domains for this pair
                            lead_id_tmp = (
                                match.get("s_id")
                                or match.get("source_id")
                                or match.get("lead_id")
                            )
                            account_id_tmp = (
                                match.get("r_id")
                                or match.get("target_id")
                                or match.get("account_id")
                            )
                            lead_domain = (
                                lead_domains.get(lead_id_tmp)
                                if "lead_domains" in locals()
                                else None
                            )
                            acct_domain = (
                                acct_domains.get(account_id_tmp)
                                if "acct_domains" in locals()
                                else None
                            )

                            if lead_domain and acct_domain:
                                rel, fam = registry.relation(lead_domain, acct_domain)
                                _track_domain_lookup(
                                    lead_domain,
                                    rel != DomainRelation.NONE,
                                    "final_relation",
                                )
                                _track_domain_lookup(
                                    acct_domain,
                                    rel != DomainRelation.NONE,
                                    "final_relation",
                                )
                                d["relation"] = (
                                    rel.value if hasattr(rel, "value") else str(rel)
                                )
                                if fam:
                                    d["family_id"] = fam
                        fsd_norm.append(d)
                    raw_fsd = fsd_norm

                feature_scores = {
                    f"{(d.get('src_field') or d.get('source_column') or d.get('source') or '?')}->"
                    f"{(d.get('ref_field') or d.get('reference_column') or d.get('reference') or '?')}": float(
                        d.get("score") or 0.0
                    )
                    for d in raw_fsd
                }

            # Enhanced hierarchy policy with region matching
            # Enabled via env L2A_HIERARCHY_GUARDRAIL=on
            if os.getenv("L2A_HIERARCHY_GUARDRAIL", "off").lower() in {
                "on",
                "true",
                "1",
            }:
                try:
                    # Try to get account name from match or accounts_df
                    account_id_tmp = (
                        match.get("r_id")
                        or match.get("target_id")
                        or match.get("account_id")
                    )
                    acct_name = ""
                    acct_parent = None

                    # Try to get from match first
                    acct_name = (
                        match.get("r_name")
                        or match.get("account_name")
                        or match.get("Name")
                        or ""
                    )
                    acct_parent = (
                        match.get("r_parent_id")
                        or match.get("parent_id")
                        or match.get("ParentId")
                    )

                    # If not in match, try to get from accounts_df
                    if not acct_name and account_id_tmp and "accounts_df" in locals():
                        acct_row = accounts_df[
                            accounts_df.get("Id", accounts_df.index) == account_id_tmp
                        ]
                        if not acct_row.empty:
                            acct_name = acct_row.iloc[0].get("Name", "")
                            acct_parent = acct_row.iloc[0].get("ParentId", None)

                    # Expanded geo/region tokens for better detection
                    REGION_TOKENS = {
                        "emea",
                        "apac",
                        "anz",
                        "dach",
                        "uk",
                        "europe",
                        "latin",
                        "latam",
                        "na",
                        "north america",
                        "japan",
                        "india",
                        "france",
                        "germany",
                        "spain",
                        "italy",
                        "benelux",
                        "nordics",
                        "mea",
                        "cee",
                        "sea",
                        "brazil",
                        "canada",
                        "mexico",
                        "china",
                        "korea",
                        "aus",
                        "nz",
                    }
                    has_geo = any(t in acct_name.lower() for t in REGION_TOKENS)

                    # Get lead region hint if available (from country/phone)
                    lead_region = None
                    if "leads_df" in locals() and lead_id_tmp:
                        lead_row = leads_df[
                            leads_df.get("Id", leads_df.index) == lead_id_tmp
                        ]
                        if not lead_row.empty:
                            # Try to extract region from lead country or phone
                            lead_country = lead_row.iloc[0].get("Country", "")
                            if lead_country:
                                lead_region = lead_country.lower()

                    # If account has geo token, no parent, and relation is EXACT/FAMILY, suggest re-parent
                    lead_id_tmp = (
                        match.get("s_id")
                        or match.get("source_id")
                        or match.get("lead_id")
                    )
                    lead_domain = (
                        lead_domains.get(lead_id_tmp)
                        if "lead_domains" in locals()
                        else None
                    )
                    acct_domain = (
                        acct_domains.get(account_id_tmp)
                        if "acct_domains" in locals()
                        else None
                    )

                    if has_geo and not acct_parent and lead_domain and acct_domain:
                        rel, _ = registry.relation(lead_domain, acct_domain)
                        rel_value = rel.value if hasattr(rel, "value") else str(rel)
                        if rel_value in {"EXACT", "FAMILY"}:
                            # Attach a non-blocking guardrail hint
                            match["guardrail"] = "reparent_candidate"
                            match["guardrail_reason"] = (
                                "Out-of-hierarchy regional account detected"
                            )

                            # Add region preference scoring bump if lead region matches account region
                            if lead_region and lead_region in acct_name.lower():
                                match["region_match_bonus"] = (
                                    0.05  # Small scoring bump for region match
                                )
                                logger.debug(
                                    f"Region match bonus for {account_id_tmp}: lead={lead_region}, account={acct_name}"
                                )

                            logger.debug(
                                f"Flagged {account_id_tmp} as reparent_candidate: {acct_name}"
                            )

                    # For FAMILY relations, require corroboration before high tier
                    if rel_value == "FAMILY" and tier == "CERTAIN":
                        # Check for name or phone corroboration
                        has_name_corr = any(
                            d.get("source_field") in {"Company", "Name"}
                            and float(d.get("score", 0)) >= 0.9
                            for d in (raw_fsd or [])
                            if isinstance(d, dict)
                        )
                        has_phone_exact = any(
                            d.get("source_field") in {"Phone", "MobilePhone"}
                            and float(d.get("score", 0)) >= 0.95
                            for d in (raw_fsd or [])
                            if isinstance(d, dict)
                        )

                        if not (has_name_corr or has_phone_exact):
                            tier = "LIKELY"  # Downgrade FAMILY without corroboration
                            logger.debug(
                                "Downgraded FAMILY match to LIKELY (no corroboration)"
                            )

                except Exception as e:
                    logger.debug(f"Could not apply hierarchy guardrail: {e}")

            # Build explanation from filtered mappings
            matched_fields = [
                f"{m['source_column']}->{m['reference_column']}"
                for m in filtered_mappings
                if m["source_column"] in feature_scores
                or any(
                    m["source_column"]
                    in (
                        d.get("src_field", ""),
                        d.get("source_column", ""),
                        d.get("source", ""),
                    )
                    for d in (raw_fsd or [])
                )
            ]

            explanation = (
                (
                    f"Matched on: {', '.join(matched_fields)} "
                    f"using intelligence-selected algorithms"
                )
                if matched_fields
                else match.get("explanation", "")
            )

            # Extract IDs from dict
            lead_id = (
                match.get("s_id") or match.get("source_id") or match.get("lead_id")
            )
            account_id = (
                match.get("r_id") or match.get("target_id") or match.get("account_id")
            )

            # Determine if this is an upgrade or new assignment
            if audit_mode == "upgrade" and lead_id in incumbents:
                incumbent_id = incumbents[lead_id]
                incumbent_score = incumbent_scores.get(lead_id, 0.0)
                gain = score - incumbent_score

                # Skip if not an upgrade (same account or insufficient gain)
                if account_id == incumbent_id:
                    continue  # Same account, not an upgrade
                if score < upgrade_min_score or gain < upgrade_min_gain:
                    continue  # Doesn't meet upgrade criteria

                # FIX 4: Tightened anchor detection
                if upgrade_anchor_required and raw_fsd:
                    has_anchor = self._has_true_anchor(raw_fsd)
                    if not has_anchor:
                        continue  # No strong anchor field

                suggestion = L2ASuggestion(
                    policy_id=policy.id,
                    lead_id=lead_id,
                    account_id=account_id,
                    score=score,
                    tier=tier,
                    reasons=feature_scores,
                    field_score_details=raw_fsd if raw_fsd else None,
                    explanation=f"Upgrade: {explanation}"
                    if explanation
                    else "Upgrade suggestion",
                    status="pending",
                    reason_code="upgrade",
                    incumbent_account_id=incumbent_id,
                    incumbent_score=incumbent_score,
                    gain=gain,
                    last_verified_at=datetime.now(),
                    incumbent_field_score_details=incumbent_evidence.get(lead_id, [])
                    if "incumbent_evidence" in locals()
                    else None,  # POLISH
                    registry_version=_reg_version(),  # Stamp registry version
                )
                upgrades_suggested += 1
            elif audit_mode == "unlinked":
                # Normal assignment suggestion
                suggestion = L2ASuggestion(
                    policy_id=policy.id,
                    lead_id=lead_id,
                    account_id=account_id,
                    score=score,
                    tier=tier,
                    reasons=feature_scores,
                    field_score_details=raw_fsd if raw_fsd else None,
                    explanation=explanation,
                    status="pending",
                    reason_code="assign",
                    registry_version=_reg_version(),  # Stamp registry version
                )
            else:
                continue  # Skip if doesn't match mode criteria

            suggestions.append(suggestion)

            # Count suggestions with non-empty reasons for observability
            if feature_scores and any(v > 0 for v in feature_scores.values()):
                suggestions_with_reasons += 1

            # Batch commit
            if len(suggestions) >= 500:
                self.db.add_all(suggestions)
                await self.db.commit()
                suggestions_created += len(suggestions)
                suggestions = []

        # Commit remaining
        if suggestions:
            self.db.add_all(suggestions)
            await self.db.commit()
            suggestions_created += len(suggestions)

        # Update policy timestamp
        if hasattr(policy, "last_run_at"):
            policy.last_run_at = datetime.now()
        if hasattr(policy, "updated_at"):
            policy.updated_at = datetime.now()

        # Update stats only if present & dict-like (older schemas won't have it)
        stats = getattr(policy, "stats", None)
        if isinstance(stats, dict):
            stats.update(
                {
                    "total_runs": stats.get("total_runs", 0) + 1,
                    "last_run_suggestions": suggestions_created,
                    "last_run_considered": len(leads_to_process),
                    "intelligence_version": "2.0",
                    "fields_mapped": len(filtered_mappings),
                    "threshold_used": thr,
                }
            )
            policy.stats = stats

        self.db.add(policy)
        await self.db.commit()

        # Verify persisted count from DB, not memory
        from sqlalchemy import text

        row = await self.db.execute(
            text("""
                SELECT COUNT(*) AS c
                FROM l2a_suggestions
                WHERE policy_id = :policy_id
                  AND status IN ('pending', 'proposed', 'new')
            """),
            {"policy_id": policy.id},
        )
        actual_count = row.scalar_one()

        # Log persistence count for observability
        logger.info(_s(f"Persisted suggestions count (memory): {suggestions_created}"))
        logger.info(_s(f"Persisted suggestions count (from DB): {actual_count}"))

        if actual_count == 0 and suggestions_created > 0:
            logger.error(
                _s(
                    f"MISMATCH: Tried to persist {suggestions_created} but DB shows {actual_count}"
                )
            )

            # Reconcile status mismatch
            row = await self.db.execute(
                text("""
                    SELECT status, COUNT(*) c
                    FROM l2a_suggestions
                    WHERE policy_id IN (SELECT id FROM l2a_policies WHERE account_id = :aid)
                    GROUP BY status
                """),
                {"aid": account_id},
            )
            status_counts = row.fetchall()
            logger.info(_s(f"Counts by status after insert: {status_counts}"))

        if progress_cb:
            progress_cb(1.0, "Complete")

        duration_seconds = (datetime.now() - start_time).total_seconds()

        # Cleanup ephemeral/debug columns (keep data minimal in responses)
        for df_name in ("source_df", "reference_df", "leads_df", "accounts_df"):
            df = locals().get(df_name)
            if df is not None and hasattr(df, "columns"):
                for col in (
                    "_domain_raw",
                    "_acct_domain_raw",
                    "_is_b2c",
                    "WebsiteForMatch",
                    "BlockKey_Domain3",
                ):
                    if col in df.columns:
                        try:
                            df.drop(columns=[col], inplace=True)
                        except Exception:
                            pass

        # Fix 8: Enhanced return messages
        base_message = f"Processed {len(leads_to_process)} leads"
        if audit_mode == "upgrade":
            if upgrades_suggested > 0:
                message = f"Found {upgrades_suggested} upgrade opportunities with average gain of {upgrade_min_gain*100:.0f}%+"
            else:
                message = (
                    "No upgrade opportunities found that meet the minimum criteria"
                )
        else:
            if suggestions_created > 0:
                message = f"Generated {suggestions_created} new account suggestions ({certain_count} high confidence)"
            else:
                message = "No new matches found above the confidence threshold"

        # Globalization v1 observability: count Auto algorithm usage and reasons persistence
        auto_mapping_count = 0
        reasons_with_content = 0
        total_suggestions = suggestions_created
        base_domain_keys_empty = 0
        total_domain_keys = 0

        # Check existing suggestions for Auto usage (simplified check since we know L2A uses Auto for Company->Name)
        if intelligent_config and intelligent_config.get("mappings"):
            auto_mapping_count = sum(
                1
                for m in intelligent_config["mappings"]
                if m.get("algorithm") == "Auto"
            )

        # Count suggestions with non-empty reasons (actual count from processing loop)
        reasons_with_content = suggestions_with_reasons

        # Check domain key coverage from existing DataFrames if available
        if (
            "source_df" in locals()
            and source_df is not None
            and "BlockKey_Domain3" in source_df.columns
        ):
            total_domain_keys = len(source_df)
            base_domain_keys_empty = (
                source_df["BlockKey_Domain3"].astype(str).str.len() == 0
            ).sum()

            # Observability: warn if empty keys >10%
            if total_domain_keys > 0:
                empty_pct = (base_domain_keys_empty / total_domain_keys) * 100
                if empty_pct > 10.0:
                    logger.warning(
                        "GlobalizationV1: High empty block key ratio %.1f%% (%d/%d) - domain data may be sparse",
                        empty_pct,
                        base_domain_keys_empty,
                        total_domain_keys,
                    )

        logger.info(
            "GlobalizationV1: auto_mappings=%d/%d base_domain_keys_empty=%d/%d reasons_present=%d/%d suggestions=%d",
            auto_mapping_count,
            len(intelligent_config.get("mappings", [])) if intelligent_config else 0,
            base_domain_keys_empty,
            total_domain_keys,
            reasons_with_content,
            total_suggestions,
            suggestions_created,
        )

        # Count domain relations across pairs (after hard-negatives applied)
        relation_counts = {"EXACT": 0, "FAMILY": 0, "NONE": 0}
        top_families = {}

        # Count relations from pairs_list if available
        if "pairs_list" in locals() and pairs_list:
            try:
                for p in pairs_list:
                    lead_id = p.get("source_id") or p.get("lead_id") or p.get("s_id")
                    acct_id = p.get("target_id") or p.get("account_id") or p.get("r_id")
                    ld = (
                        lead_domains.get(lead_id)
                        if "lead_domains" in locals()
                        else None
                    )
                    ad = (
                        acct_domains.get(acct_id)
                        if "acct_domains" in locals()
                        else None
                    )
                    if ld and ad:
                        rel, fam = reg.relation(ld, ad)
                        rel_label = rel.value if hasattr(rel, "value") else str(rel)
                        relation_counts[rel_label] = (
                            relation_counts.get(rel_label, 0) + 1
                        )
                        if rel in (DomainRelation.EXACT, DomainRelation.FAMILY) and fam:
                            top_families[fam] = top_families.get(fam, 0) + 1
            except Exception as e:
                logger.debug(f"Could not count relations from pairs: {e}")

        # Also count from suggestions if available (fallback)
        if "suggestions" in locals() and not relation_counts.get("EXACT"):
            for sugg in suggestions:
                if hasattr(sugg, "field_score_details") and sugg.field_score_details:
                    for detail in sugg.field_score_details:
                        if isinstance(detail, dict) and "relation" in detail:
                            rel = detail.get("relation", "NONE")
                            relation_counts[rel] = relation_counts.get(rel, 0) + 1
                            if rel in ("EXACT", "FAMILY") and "family_id" in detail:
                                fam = detail["family_id"]
                                top_families[fam] = top_families.get(fam, 0) + 1

        # Get top 5 families by count
        top_family_list = sorted(
            top_families.items(), key=lambda x: x[1], reverse=True
        )[:5]

        # JSON breadcrumb for suggestions summary (for debugging big datasets)
        sugg_breadcrumb = {
            "type": "FMATCH_SUGG",
            "timestamp": datetime.now().isoformat(),
            "account_id": account_id,
            "suggestions_created": suggestions_created,
            "certain_count": certain_count,
            "likely_count": suggestions_created - certain_count
            if suggestions_created > certain_count
            else 0,
            "removed_by_hard_negatives": removed_count
            if "removed_count" in locals()
            else 0,
            "domain_coverage": {
                "leads": run_breadcrumb.get("domain_coverage_leads", 0),
                "accounts": run_breadcrumb.get("domain_coverage_accounts", 0),
            },
            "blocking_active": domain_blocking_active,
            "duration_seconds": duration_seconds,
            "relations": relation_counts,
            "top_families": [f"{fam}: {cnt}" for fam, cnt in top_family_list],
        }
        logger.info(f"FMATCH_SUGG: {json.dumps(sugg_breadcrumb)}")

        return {
            "status": "success",
            "message": message,
            "considered": len(leads_to_process),
            "suggestions_generated": suggestions_created,
            "certain_count": certain_count,
            "upgrades_suggested": upgrades_suggested if audit_mode == "upgrade" else 0,
            "audit_mode": audit_mode,
            "upgrade_criteria": {
                "min_gain": upgrade_min_gain,
                "min_score": upgrade_min_score,
                "anchor_required": upgrade_anchor_required,
                "enablement": upgrade_enablement,
            }
            if audit_mode == "upgrade"
            else None,
            "processing_time_ms": result.processing_time_ms,
            "engine_version": result.engine_version,
            "total_duration_seconds": duration_seconds,
            "intelligence_config": {
                "fields_mapped": len(filtered_mappings),
                "threshold": thr,
                "blocking_applied": intelligent_config.get("blocking", {}).get(
                    "apply_blocking", False
                ),
                "optimization_goal": "lead_to_account",
                "multi_algo_enabled": use_multi_flag,
            },
            "b2c_metrics": {
                "mode": b2c_metrics["b2c_mode"],
                "whitelist_count": b2c_metrics["whitelist_count"],
                "leads_b2c_count": b2c_metrics["leads_b2c_count"],
                "leads_b2c_pct": 100.0
                * b2c_metrics["leads_b2c_count"]
                / max(1, len(leads_to_process)),
                "accounts_b2c_count": b2c_metrics["accounts_b2c_count"],
                "accounts_b2c_pct": 100.0
                * b2c_metrics["accounts_b2c_count"]
                / max(1, len(accounts)),
                "website_suppressed_count": b2c_metrics.get(
                    "website_suppressed_count", 0
                ),
                "mappings_suppressed": b2c_metrics["mappings_suppressed"],
                "blockkey_coverage_before": b2c_metrics["blockkey_coverage_before"],
                "blockkey_coverage_after": b2c_metrics["blockkey_coverage_after"],
                "top_bucket_pct": b2c_metrics.get("top_bucket_pct", 0.0),
                "key_length_bumped": b2c_metrics.get("key_length_bumped", False),
                "blockkey_length_used": locals().get("key_len", 3),
                "b2c_filter_version": _b2c_filter_version
                if B2C_AVAILABLE and _b2c_filter_version
                else "fallback",
                "b2c_filter_domains": _b2c_domain_count
                if B2C_AVAILABLE and _b2c_domain_count
                else 12,
                "b2c_bloom_bits": _b2c_bloom_bits
                if B2C_AVAILABLE and _b2c_bloom_bits
                else 0,
                "b2c_bloom_hashes": _b2c_bloom_hashes
                if B2C_AVAILABLE and _b2c_bloom_hashes
                else 0,
            },
            "stats": result.stats,
        }

    async def list_writeback_options_by_policy_id(
        self, policy_id: str
    ) -> List[WritebackCandidate]:
        """List available Salesforce fields for writeback"""
        # Resolve policy and integration, then delegate to the main variant for consistency
        policy_res = await self.db.execute(
            select(L2APolicy).where(L2APolicy.id == policy_id)
        )
        policy = policy_res.scalar_one_or_none()
        if not policy:
            raise ValueError(f"Policy {policy_id} not found")

        integration_res = await self.db.execute(
            select(Integration).where(
                Integration.id == policy.integration_id,
                Integration.provider == IntegrationProvider.SALESFORCE,
            )
        )
        integration = integration_res.scalar_one_or_none()
        if not integration:
            raise ValueError("Salesforce integration not found")

        data = await self.list_writeback_options_for_integration(integration, policy)
        # Return just the candidates as WritebackCandidate list for backward compatibility
        return [
            WritebackCandidate(**c) if isinstance(c, dict) else c
            for c in (data.get("candidates") or [])
        ]

    async def choose_writeback_field(
        self,
        policy_id: str,
        field_api_name: str,
        overwrite_mode: OverwriteMode = OverwriteMode.if_empty,
    ) -> Dict[str, Any]:
        """Set the writeback field and overwrite mode for a policy"""
        policy_res = await self.db.execute(
            select(L2APolicy).where(L2APolicy.id == policy_id)
        )
        policy = policy_res.scalar_one_or_none()
        if not policy:
            raise ValueError(f"Policy {policy_id} not found")

        # Validate field exists
        candidates = await self.list_writeback_options_by_policy_id(policy_id)
        field = next((c for c in candidates if c.apiName == field_api_name), None)
        if not field:
            raise ValueError(f"Field {field_api_name} not available for writeback")
        if not field.updateable:
            raise ValueError(f"Field {field_api_name} is not updateable")

        # Update policy
        policy.link_field_api_name = field_api_name
        policy.link_overwrite_mode = overwrite_mode.value
        self.db.add(policy)
        await self.db.commit()

        return {
            "field": field_api_name,
            "overwrite_mode": overwrite_mode.value,
            "message": f"Writeback configured to {field_api_name} with mode {overwrite_mode.value}",
        }

    async def validate_writeback(self, policy_id: str) -> Dict[str, Any]:
        """Validate current writeback configuration"""
        policy_res = await self.db.execute(
            select(L2APolicy).where(L2APolicy.id == policy_id)
        )
        policy = policy_res.scalar_one_or_none()
        if not policy:
            raise ValueError(f"Policy {policy_id} not found")

        if not policy.link_field_api_name:
            return {
                "valid": False,
                "field": None,
                "overwrite_mode": policy.link_overwrite_mode,
                "message": "No writeback field configured",
            }

        # Check field still exists and is updateable
        try:
            candidates = await self.list_writeback_options_by_policy_id(policy_id)
            field = next(
                (c for c in candidates if c.apiName == policy.link_field_api_name), None
            )

            if not field:
                return {
                    "valid": False,
                    "field": policy.link_field_api_name,
                    "overwrite_mode": policy.link_overwrite_mode,
                    "message": f"Field {policy.link_field_api_name} no longer exists",
                }

            if not field.updateable:
                return {
                    "valid": False,
                    "field": policy.link_field_api_name,
                    "overwrite_mode": policy.link_overwrite_mode,
                    "message": f"Field {policy.link_field_api_name} is not updateable",
                }

            return {
                "valid": True,
                "field": policy.link_field_api_name,
                "overwrite_mode": policy.link_overwrite_mode,
                "message": "Writeback configuration is valid",
            }
        except Exception as e:
            return {
                "valid": False,
                "field": policy.link_field_api_name,
                "overwrite_mode": policy.link_overwrite_mode,
                "message": f"Error validating field: {str(e)}",
            }

    async def migrate_from_old_l2a(self):
        """Migration helper to replace old L2AMatcher"""
        logger.warning("Migrating from old L2AMatcher to IntelligentL2AService")

        # Validate compliance
        compliance = await self.validate_compliance()
        if not compliance["compliant"]:
            raise ValueError(f"Service not compliant: {compliance['violations']}")

        logger.info("OK: IntelligentL2AService is fully compliant!")
        return True
