"""
EngineClient - Single point of access to the matching engine
============================================================
All matching operations MUST go through this client.
No service should reimplement fuzzy matching logic.

CRITICAL RULES:
- No auto-generation of mappings in production
- No direct fuzzy library usage here
- Reference data indexed once and reused
- All scores use 0-1 scale internally
"""

import asyncio
import hashlib
import json
import logging
import os
import time
from typing import Dict, Any, List, Optional, Tuple, Callable, Set
from collections import defaultdict
import pandas as pd
from dataclasses import dataclass
from datetime import datetime
from types import SimpleNamespace
import uuid

try:
    from rapidfuzz import fuzz as _rf_fuzz

    _token_set_ratio = lambda a, b: _rf_fuzz.token_set_ratio(a, b)
except Exception:
    try:
        from fuzzywuzzy import fuzz as _fz_fuzz

        _token_set_ratio = lambda a, b: _fz_fuzz.token_set_ratio(a, b)
    except Exception:
        _token_set_ratio = None

# Import from the actual engine
from ..engine_bridge import (
    match as run_match_async,  # The actual function name is 'match'
    dedupe as run_dedupe_async,  # The actual function name is 'dedupe'
    MatchConfig,
    DedupeConfig,
)

log = logging.getLogger(__name__)

# Engine version for observability
ENGINE_VERSION = "1.2.3"

# Domain enhancement thresholds
DOMAIN_MIN_CONFIDENCE = 0.90  # Minimum confidence to use derived domain
DOMAIN_MIN_COVERAGE_MAPPING = 0.10  # Minimum coverage to add as mapping
DOMAIN_MIN_COVERAGE_BLOCKING = 0.25  # Minimum coverage to use for blocking
DOMAIN_WEIGHT_REDUCTION = 0.7  # Weight reduction factor for derived domains
DOMAIN_DERIVED_WEIGHT = 0.56  # 0.8 * 0.7 = 0.56


@dataclass
class MatchRequest:
    """Request for matching operations"""

    source_data: pd.DataFrame
    reference_data: Optional[pd.DataFrame] = None
    mode: str = "dedupe"  # "dedupe" or "match"
    mappings: List[Dict[str, Any]] = None  # REQUIRED in production
    threshold: float = 0.75  # 0-1 scale
    source_id_column: str = None
    reference_id_column: str = None
    blocking_strategy: Optional[Dict[str, Any]] = None
    job_id: Optional[str] = None
    options: Optional[Dict[str, Any]] = None  # Phase-2 B2B intelligence flags


@dataclass
class MatchResult:
    """Result from matching operations"""

    matches: pd.DataFrame
    stats: Dict[str, Any]
    confidence_scores: Optional[pd.DataFrame] = None
    engine_version: str = ENGINE_VERSION
    config_hash: str = None
    processing_time_ms: int = None
    metrics: Optional[Dict[str, Any]] = None  # Phase-2 metrics tracking


@dataclass
class MatchMetrics:
    """Metrics for Phase-2 B2B intelligence tracking"""

    candidates_total: int = 0
    blocked_pairs: int = 0
    family_hits: int = 0
    website_hits: int = 0
    geo_nudges: int = 0
    b2c_filtered: int = 0
    no_match_rows: int = 0
    engine_ms: int = 0
    write_ms: int = 0
    domain_expansions: int = 0
    # New metrics for surgical improvements
    domain_gated_rows: int = 0  # Count of rows where we applied strict filtering
    domain_gated_hits: int = 0  # Rows that produced candidates under strict gating
    confusable_penalties: int = 0  # Number of pairs penalized for confusability
    account_match_with_domain: int = 0  # % account matches with DOM/FAM
    possible_match_name_only: int = 0  # % possible matches with NAME* only
    fallback_rows: int = 0  # Source rows that fell back to name-only
    row_candidates_empty: int = (
        0  # Rows where strict domain gating yielded no ref candidates
    )
    missing_domain_evidence: int = 0  # Rows missing domain/family precursors


class ReferenceIndex:
    """Holds a pre-built reference index for reuse"""

    def __init__(self, index_id: str, data: pd.DataFrame, config: Dict[str, Any]):
        self.index_id = index_id
        self.data = data
        self.config = config
        self.created_at = datetime.utcnow()
        self.usage_count = 0
        self.blocking_index = None  # Pre-computed blocking keys


class EngineClient:
    """
    Client for all matching operations.
    This is the ONLY way services should perform matching.
    """

    # Algorithm canonicalization map - engine requires exact case
    _ALGO_CANON = {
        "wratio": "WRatio",
        "token_set": "TokenSetRatio",
        "token_set_ratio": "TokenSetRatio",
        "tokenset": "TokenSet",  # Alternative name
        "tokensetratio": "TokenSet",  # Alternative name
        "token_sort": "TokenSortRatio",
        "token_sort_ratio": "TokenSortRatio",
        "partial_ratio": "PartialRatio",
        "exact": "Exact",
        "exactdomain": "ExactDomain",  # For exact domain matching
        "jaro": "Jaro",
        "jarowinkler": "JaroWinkler",
        "jaro_winkler": "JaroWinkler",
        "soundex": "Soundex",
        # Special cases
        "domain": "Domain",
    }

    def __init__(self):
        """Initialize the engine client."""
        self.log = logging.getLogger(__name__)
        # alias: prefer self.logger for new code paths
        self.logger = self.log
        self._reference_indices = {}  # Cache of reference indices

        # Initialize domain family service for Phase-2
        try:
            from .domain_family_service import DomainFamilyService

            self._domain_family_service = DomainFamilyService()
        except ImportError:
            self.log.warning("Domain family service not available")
            self._domain_family_service = None

    def _norm_maps(self, maps):
        """Normalize mapping dict keys to ensure 'source' and 'ref' always exist.

        This avoids silent no-ops in downstream logic (e.g., semantic fallback)
        when callers used alternate keys like source_field/reference_field.
        """
        out = []
        for m in maps or []:
            if not isinstance(m, dict):
                continue
            src = (
                m.get("source")
                or m.get("source_field")
                or m.get("src")
                or m.get("left")
            )
            ref = (
                m.get("ref")
                or m.get("reference")
                or m.get("reference_field")
                or m.get("right")
                or m.get("target")
            )
            algo = (m.get("preferred_algo") or m.get("algorithm") or "").lower()
            nm = {"source": src, "ref": ref, "algorithm": algo}
            nm.update(m)
            out.append(nm)
        return out

    def _canonicalize_algorithm(self, algo_name: str) -> str:
        """
        Convert any algorithm name variation to canonical engine name.

        Args:
            algo_name: Algorithm name in any case/format

        Returns:
            Canonical algorithm name that engine accepts
        """
        if not algo_name:
            return "WRatio"

        raw = algo_name.strip().lower()

        # Map common variations first
        if raw in ("tokensetratio", "token_set"):
            raw = "token_set_ratio"
        elif raw in ("tokensortratio", "token_sort"):
            raw = "token_sort_ratio"
        elif raw in ("jw", "jaro_winkler", "jaro-winkler"):
            raw = "jarowinkler"
        elif raw in ("exactmatch", "exact_match"):
            raw = "exact"
        elif raw in ("exactdomain", "exact_domain", "exact-domain"):
            raw = "exactdomain"
        elif raw in ("domainmatch", "domain_match"):
            raw = "domain"

        # Use canonicalization map, fallback to WRatio if unknown
        return self._ALGO_CANON.get(raw, "WRatio")

    def _normalize_mapping(self, mapping: Dict[str, Any]) -> Dict[str, Any]:
        """
        Normalize a single mapping to canonical format.
        Defensive handling of variations from intelligence vs policy sources.

        Args:
            mapping: Raw mapping dict from any source

        Returns:
            Normalized mapping with preferred_algo for future multi-algo support
        """
        # Handle source column variations
        src = (
            mapping.get("source_field")
            or mapping.get("source_column")
            or mapping.get("source")
            or mapping.get("src")
            or ""
        ).strip()

        # Handle reference column variations
        ref = (
            mapping.get("reference_field")
            or mapping.get("reference_column")
            or mapping.get("reference")
            or mapping.get("ref")
            or mapping.get("target")
            or ""
        ).strip()

        if not src or not ref:
            return None

        # Get algorithm - check both preferred_algo and algorithm
        algo_raw = (
            mapping.get("preferred_algo")
            or mapping.get("algorithm")
            or mapping.get("algo")
            or "WRatio"
        )

        # Canonicalize the algorithm name
        algo = self._canonicalize_algorithm(algo_raw)

        # Normalize and clamp weight to [0, 1]
        try:
            w = float(mapping.get("weight", 0.5))
        except (TypeError, ValueError):
            self.log.warning(
                f"Invalid weight {mapping.get('weight')}, using default 0.5"
            )
            w = 0.5

        if not (0 <= w <= 1):
            self.log.warning(f"Clamping weight {w} to [0,1] range")
            w = min(1.0, max(0.0, w))

        # Guard against NaN
        if w != w:  # NaN check
            self.log.warning("Weight was NaN, using default 0.5")
            w = 0.5

        # Build normalized mapping
        return {
            "source": src,
            "ref": ref,
            "weight": w,
            # Use preferred_algo for future multi-algo support
            "preferred_algo": algo,
            # Keep algorithm for backward compatibility
            "algorithm": algo,
        }

    def _normalize_mappings(self, mappings: List[Any]) -> List[Dict[str, Any]]:
        """
        Normalize a list of mappings from any source.
        Defensive handling of variations from intelligence vs policy sources.

        Args:
            mappings: Raw mappings from any source

        Returns:
            List of normalized mappings ready for engine
        """
        if not mappings:
            return []

        normalized = []
        for m in mappings:
            if isinstance(m, dict):
                norm = self._normalize_mapping(m)
                if norm:
                    normalized.append(norm)
            else:
                self.log.warning(f"Skipping non-dict mapping: {type(m)}")

        return normalized

    async def begin_reference_index(
        self, reference_data: pd.DataFrame, config: Dict[str, Any]
    ) -> str:
        """
        Create a reusable index for reference data (e.g., 50k Accounts).
        This avoids re-indexing for every batch of Leads.

        Args:
            reference_data: Reference DataFrame to index
            config: Configuration including mappings and blocking

        Returns:
            index_id: Unique ID to reference this index
        """
        start_time = time.time()
        index_id = str(uuid.uuid4())

        self.log.info(
            f"Creating reference index {index_id} for {len(reference_data)} records"
        )

        # Create index object
        index = ReferenceIndex(index_id, reference_data, config)

        # Pre-compute blocking keys if provided
        if config.get("blocking"):
            blocking = config["blocking"]
            if isinstance(blocking, dict) and blocking.get("reference_column"):
                ref_col = blocking["reference_column"]
                if ref_col in reference_data.columns:
                    # Build blocking index
                    index.blocking_index = self._build_blocking_index(
                        reference_data, ref_col, blocking.get("key_length", 3)
                    )
                    self.log.info(
                        f"Built blocking index on {ref_col} with {len(index.blocking_index)} blocks"
                    )

        # Store index
        self._reference_indices[index_id] = index

        elapsed_ms = int((time.time() - start_time) * 1000)
        self.log.info("Reference index %s ready in %dms", index_id, elapsed_ms)

        return index_id

    async def match_with_index(
        self, source_data: pd.DataFrame, index_id: str, config: Dict[str, Any]
    ) -> MatchResult:
        """
        Match source data against a pre-built reference index.

        Args:
            source_data: Source DataFrame to match
            index_id: ID of pre-built reference index
            config: Additional config (threshold, etc.)

        Returns:
            MatchResult with matches found
        """
        start_time = time.time()

        # Get cached index
        if index_id not in self._reference_indices:
            raise ValueError(f"Reference index {index_id} not found")

        index = self._reference_indices[index_id]
        index.usage_count += 1

        self.log.info(
            f"Matching {len(source_data)} records against index {index_id} "
            f"(usage #{index.usage_count})"
        )

        # Merge configs
        full_config = {**index.config, **config}
        mappings = full_config.get("mappings")
        if not mappings:
            raise ValueError("Mappings are required for matching")

        # Create match configuration with proper field names
        threshold = full_config.get("threshold", 0.75)

        # Be forgiving on blocking inputs so we never silently drop blocking
        apply_blocking = bool(
            full_config.get("apply_blocking")
            or (full_config.get("blocking") or {}).get("apply_blocking")
        )

        blocking_keys = full_config.get("blocking_keys") or (
            full_config.get("blocking") or {}
        ).get("keys")

        # Fallback: some callers send a single dict as "blocking"
        blocking = full_config.get("blocking")
        if not blocking_keys and isinstance(blocking, dict):
            if {"column", "variant", "key_length"} <= set(blocking.keys()):
                blocking_keys = [blocking]
            elif blocking.get("source_column") or blocking.get("reference_column"):
                # Old format - convert it
                blocking_keys = [
                    {
                        "column": blocking.get("source_column", "Domain"),
                        "variant": blocking.get("variant", "alnum"),
                        "key_length": blocking.get("key_length", 3),
                    }
                ]

        # Use centralized normalization for defensive handling
        normalized_maps = self._normalize_mappings(mappings)

        if not normalized_maps:
            raise ValueError("No valid mappings after normalization")

        # Convert blocking_keys to legacy format expected by MatchConfig
        legacy_blocking = None
        if apply_blocking and blocking_keys:
            first = blocking_keys[0]
            src_col = first.get("source_column") or first.get("column")
            ref_col = first.get("reference_column") or first.get("column")
            keylen = int(first.get("key_length", 3))
            legacy_blocking = {
                "source_column": src_col,
                "reference_column": ref_col,
                "key_length": keylen,
            }

        # Verify both sides exist for the current batch/index
        if (
            legacy_blocking
            and legacy_blocking["source_column"] not in source_data.columns
        ):
            log.warning(
                "Blocking source column '%s' not in source; disabling blocking.",
                legacy_blocking["source_column"],
            )
            legacy_blocking = None
        if (
            legacy_blocking
            and legacy_blocking["reference_column"] not in index.data.columns
        ):
            log.warning(
                "Blocking ref column '%s' not in reference; disabling blocking.",
                legacy_blocking["reference_column"],
            )
            legacy_blocking = None

        match_config = MatchConfig(
            job_id=full_config.get("job_id") or str(uuid.uuid4()),
            src_path=source_data,  # DataFrame
            ref_path=index.data,  # DataFrame from cached index
            maps=normalized_maps,
            threshold=int(threshold * 100),  # Convert 0-1 to 0-100
            src_id_col=full_config.get("source_id_column"),
            ref_id_col=full_config.get("reference_id_column"),
            # Use legacy fields only - MatchConfig doesn't accept blocking_keys
            apply_blocking=bool(legacy_blocking),
            source_block_col=legacy_blocking["source_column"]
            if legacy_blocking
            else None,
            ref_block_col=legacy_blocking["reference_column"]
            if legacy_blocking
            else None,
            block_key_length=legacy_blocking["key_length"] if legacy_blocking else 3,
        )

        log.info(
            "Engine apply_blocking=%s, block_col=%s, key_len=%s",
            bool(legacy_blocking),
            legacy_blocking and legacy_blocking["source_column"],
            legacy_blocking and legacy_blocking["key_length"],
        )

        # Run through engine
        result_df, stats = await run_match_async(
            cfg=match_config, progress_callback=full_config.get("progress_callback")
        )

        # Calculate config hash for reproducibility
        config_str = json.dumps(full_config, sort_keys=True, default=str)
        config_hash = hashlib.md5(config_str.encode()).hexdigest()[:8]

        elapsed_ms = int((time.time() - start_time) * 1000)

        return MatchResult(
            matches=result_df,
            stats=stats.__dict__ if hasattr(stats, "__dict__") else stats,
            engine_version=ENGINE_VERSION,
            config_hash=config_hash,
            processing_time_ms=elapsed_ms,
        )

    async def release_index(self, index_id: str):
        """
        Release a reference index when done.

        Args:
            index_id: ID of index to release
        """
        if index_id in self._reference_indices:
            index = self._reference_indices[index_id]
            self.log.info(f"Releasing index {index_id} after {index.usage_count} uses")
            del self._reference_indices[index_id]

    async def find_duplicates(
        self,
        data: pd.DataFrame,
        mappings: List[Dict[str, Any]],  # REQUIRED - no default
        threshold: float = 0.75,  # 0-1 scale
        id_column: str = None,
        blocking_strategy: Optional[Dict[str, Any]] = None,
        blocking_hints: Optional[Dict[str, Any]] = None,  # From discovery
        progress_callback: Optional[Callable] = None,
        job_id: Optional[str] = None,
    ) -> MatchResult:
        """
        Find duplicates within a single dataset using the core engine.

        Args:
            data: DataFrame to deduplicate
            mappings: REQUIRED column mappings with weights (from discovery)
            threshold: Minimum match score (0-1 scale)
            id_column: ID column name
            blocking_strategy: Optional blocking configuration
            blocking_hints: Optional hints from discovery
            progress_callback: Optional callback for progress updates
            job_id: Optional job ID for tracking

        Returns:
            MatchResult with duplicates found
        """
        start_time = time.time()

        if not mappings:
            raise ValueError(
                "Mappings are required. Use discovery service to generate mappings."
            )

        self.log.info(f"Finding duplicates in dataset with {len(data)} rows")

        # Auto-detect ID column if not provided
        if not id_column:
            id_column = self._detect_id_column(data)

        # Use blocking hints if no strategy provided
        if not blocking_strategy and blocking_hints:
            blocking_strategy = blocking_hints

        # Use centralized normalization for defensive handling
        normalized_maps = self._normalize_mappings(mappings)

        if not normalized_maps:
            raise ValueError("No valid mappings after normalization")

        config = DedupeConfig(
            job_id=job_id or str(uuid.uuid4()),
            input_path=data,  # DedupeConfig expects input_path, can be DataFrame
            maps=normalized_maps,
            threshold=int(threshold * 100),  # Convert 0-1 to 0-100 scale for engine
            rec_id_col=id_column,  # Fixed: DedupeConfig expects rec_id_col, not src_id_col
            algorithm="WRatio",  # Required field - must use canonical case
            ruleset_name="standard",  # Required field
            block_key_length=blocking_strategy.get("key_length", 3)
            if blocking_strategy
            else 3,
            dup_block_limit=50000,  # Required field - limit for blocking bucket size
            apply_blocking=blocking_strategy is not None,
            source_block_col=blocking_strategy.get("source_column")
            if blocking_strategy
            else None,
        )

        # Explicit logging for production visibility
        self.log.info(
            "EngineClient.find_duplicates(): mode=DUPLICATE alg=%s threshold=%s maps=%d blocking=%s",
            config.algorithm,
            config.threshold,
            len(config.maps),
            config.source_block_col or "none",
        )

        # Run through engine with error handling
        try:
            result_df, stats = await run_dedupe_async(
                cfg=config, progress_callback=progress_callback
            )
        except Exception as e:
            error_msg = str(e)
            if "Blocking key" in error_msg and "not found" in error_msg:
                self.log.error(
                    f"Blocking key generation failed - check blocking column configuration: {error_msg}"
                )
                raise ValueError(
                    f"Dedupe failed: blocking column '{config.source_block_col}' could not be processed. Error: {error_msg}"
                )
            else:
                self.log.error(f"Dedupe engine failed: {error_msg}")
                raise

        # Calculate config hash
        config_str = json.dumps(
            {"mappings": mappings, "threshold": threshold}, sort_keys=True, default=str
        )
        config_hash = hashlib.md5(config_str.encode()).hexdigest()[:8]

        elapsed_ms = int((time.time() - start_time) * 1000)

        return MatchResult(
            matches=result_df,
            stats=stats.__dict__ if hasattr(stats, "__dict__") else stats,
            engine_version=ENGINE_VERSION,
            config_hash=config_hash,
            processing_time_ms=elapsed_ms,
        )

    def _extract_etld1(self, domain: str) -> str:
        """Extract eTLD+1 from domain using public suffix list"""
        if not domain or pd.isna(domain):
            return None

        # Cache for performance
        if not hasattr(self, "_etld_cache"):
            self._etld_cache = {}

        domain = str(domain).lower().strip()
        if domain in self._etld_cache:
            return self._etld_cache[domain]

        try:
            # Try tldextract if available
            import tldextract

            ext = tldextract.extract(domain)
            result = ext.registered_domain or domain  # registered_domain is eTLD+1
        except ImportError:
            # Fallback to improved heuristic
            parts = domain.split(".")
            if len(parts) >= 2:
                # Extended list of multi-part TLDs
                if (
                    parts[-2] in ["co", "com", "org", "net", "edu", "gov", "ac"]
                    and len(parts) >= 3
                ):
                    result = ".".join(parts[-3:])
                else:
                    result = ".".join(parts[-2:])
            else:
                result = domain

        self._etld_cache[domain] = result
        return result

    def _is_confusable(self, name1: str, name2: str) -> bool:
        """Check if two names are confusable (edit distance 1 on brand tokens)"""
        if not name1 or not name2:
            return False

        # Normalize
        n1 = str(name1).lower().strip()
        n2 = str(name2).lower().strip()

        # Known confusable pairs
        confusables = [
            ("sanford", "stanford"),
            ("musc", "umc"),
            ("usc", "umc"),
            ("northwestern", "northeastern"),
            ("wellstar", "wellspan"),
            ("advent", "adventist"),
        ]

        for pair in confusables:
            if (pair[0] in n1 and pair[1] in n2) or (pair[1] in n1 and pair[0] in n2):
                return True

        # Check edit distance on main brand token
        import difflib

        tokens1 = n1.split()[:2]  # First 2 tokens
        tokens2 = n2.split()[:2]

        for t1 in tokens1:
            for t2 in tokens2:
                if len(t1) > 4 and len(t2) > 4:  # Only check substantial tokens
                    ratio = difflib.SequenceMatcher(None, t1, t2).ratio()
                    if 0.8 < ratio < 1.0:  # Very similar but not identical
                        return True

        return False

    def _get_sector_bias(
        self, source_domain: str, ref_domain: str, ref_name: str = None
    ) -> float:
        """Apply sector-appropriate bias for .gov/.edu domains"""
        if not source_domain:
            return 0.0

        src = str(source_domain).lower()
        ref = str(ref_domain).lower() if ref_domain else ""
        ref_name_lower = str(ref_name).lower() if ref_name else ""

        # .gov source bias
        if src.endswith(".gov"):
            # Prefer VA/federal matches
            if "va.gov" in ref or "veterans" in ref_name_lower:
                return 5.0  # Small positive bias
            # Slight negative for non-federal unless domains match
            if not ref.endswith(".gov") and src != ref:
                return -3.0

        # .edu source bias
        if src.endswith(".edu"):
            # Prefer university health systems
            if (
                "university" in ref_name_lower
                or "college" in ref_name_lower
                or ref.endswith(".edu")
            ):
                return 2.0
            # Slight negative for state DOH unless domains match
            if ".gov" in ref and "health" in ref_name_lower and src != ref:
                return -2.0

        return 0.0

    def _should_use_row_candidates(
        self,
        source_row_count: int = 0,
        options: Optional[Dict[str, Any]] = None,
    ) -> bool:
        """
        Determine if row-candidate mode should be used for matching.

        Row-candidate mode uses per-row domain gating but incurs Python loop overhead.
        For large jobs, bypassing it and using direct engine matching is faster.

        Args:
            source_row_count: Number of source rows in the job

        Returns:
            True to use row-candidate mode, False to use direct engine matching
        """
        # Per-call override (used by callers that need deterministic batching behavior).
        override = None
        if isinstance(options, dict):
            override = options.get("use_row_candidates")
        if isinstance(override, bool):
            return override
        if override is not None:
            override_text = str(override).strip().lower()
            if override_text in ("0", "false", "no", "off"):
                return False
            if override_text in ("1", "true", "yes", "on"):
                return True

        # Check environment override (allows global forcing on/off)
        env_value = os.getenv("USE_ROW_CANDIDATES", "").strip().lower()
        if env_value in ("0", "false", "no", "off"):
            return False
        if env_value in ("1", "true", "yes", "on"):
            return True

        # Auto-disable for large jobs (bypass per-row overhead)
        threshold = int(os.getenv("LARGE_JOB_THRESHOLD", "100000"))
        if source_row_count >= threshold:
            self.log.info(
                f"[AUTO-BYPASS] Source rows ({source_row_count:,}) >= threshold ({threshold:,}), "
                f"using direct engine mode for better performance"
            )
            return False

        # Default: use row-candidate mode for smaller jobs
        return True

    def _apply_domain_family_expansion(
        self,
        source_data: pd.DataFrame,
        reference_data: pd.DataFrame,
        mappings: List[Dict[str, Any]],
        metrics: MatchMetrics,
        options: Dict[str, Any],
        reference_id_column: str = None,
    ) -> Tuple[pd.DataFrame, List[Dict[str, Any]]]:
        """
        Apply domain family expansion for B2B matching.
        Expands reference candidates based on domain families.
        """
        if not options.get("use_domain_family") or not self._domain_family_service:
            return reference_data, []

        # Find domain mapping
        domain_map = None
        for m in mappings:
            if (
                m.get("preferred_algo") == "Domain"
                or m.get("similarity_measure") == "Domain"
            ):
                domain_map = m
                break

        if not domain_map:
            return reference_data, []

        evidence = []
        expanded_refs = []
        MAX_FAMILY_CANDIDATES = 20

        # Extract unique domains from source
        source_col = domain_map.get("source")
        if source_col and source_col in source_data.columns:
            source_domains = source_data[source_col].dropna().unique()

            for domain in source_domains[:100]:  # Cap processing
                family = self._domain_family_service.get_family(str(domain))
                if family and len(family.all_domains()) > 1:
                    ref_domains = family.all_domains()

                    # Find reference rows with any family domain
                    ref_col = domain_map.get("ref") or domain_map.get("reference")
                    if ref_col in reference_data.columns:
                        family_matches = reference_data[
                            reference_data[ref_col].isin(ref_domains)
                        ]

                        if not family_matches.empty:
                            expanded_refs.append(family_matches)
                            metrics.family_hits += len(family_matches)
                            metrics.domain_expansions += 1
                            # Slim evidence: only store essential info
                            evidence.append(
                                {
                                    "anchor": "domain_family",
                                    "hit": True,
                                    "family": list(ref_domains)[
                                        :5
                                    ],  # Just top 5 domains
                                }
                            )

        # Combine expanded references (limit total)
        if expanded_refs:
            expanded_df = pd.concat(expanded_refs)

            # Determine the best key for de-duplication
            ref_col = domain_map.get("ref") or domain_map.get("reference")
            dedup_key = reference_id_column or ref_col or "Domain_Append"

            # De-duplicate by the determined key if it exists
            if dedup_key in expanded_df.columns:
                expanded_df = expanded_df.drop_duplicates(
                    subset=[dedup_key], keep="first"
                )
            else:
                expanded_df = expanded_df.drop_duplicates()

            if len(expanded_df) > MAX_FAMILY_CANDIDATES * len(source_data):
                expanded_df = expanded_df.head(MAX_FAMILY_CANDIDATES * len(source_data))

            # Merge with original reference data, de-dup by same key
            if dedup_key in reference_data.columns:
                reference_data = pd.concat(
                    [reference_data, expanded_df]
                ).drop_duplicates(subset=[dedup_key], keep="first")
            else:
                reference_data = pd.concat(
                    [reference_data, expanded_df]
                ).drop_duplicates()

        return reference_data, evidence

    def _apply_website_anchor(
        self,
        source_row: pd.Series,
        ref_row: pd.Series,
        score: float,
        metrics: MatchMetrics,
        options: Dict[str, Any],
    ) -> Tuple[float, Dict[str, Any]]:
        """Apply website matching as a weak identity anchor."""
        if not options.get("use_website"):
            return score, {}

        evidence = {}
        website_weight = 0.15  # Weak signal

        # Check for website columns
        src_website = None
        ref_website = None

        for col in source_row.index:
            if "website" in col.lower() or "url" in col.lower():
                src_website = source_row[col]
                break

        for col in ref_row.index:
            if "website" in col.lower() or "url" in col.lower():
                ref_website = ref_row[col]
                break

        if src_website and ref_website:
            # Normalize URLs
            src_host = self._extract_host(str(src_website))
            ref_host = self._extract_host(str(ref_website))

            if src_host and ref_host and src_host == ref_host:
                score = score + (1 - score) * website_weight  # Boost score
                metrics.website_hits += 1
                evidence = {"anchor": "website", "hit": True}

        return score, evidence

    def _apply_geo_nudge(
        self,
        source_row: pd.Series,
        ref_row: pd.Series,
        score: float,
        metrics: MatchMetrics,
        options: Dict[str, Any],
    ) -> Tuple[float, Dict[str, Any]]:
        """Apply geographic nudging as a tie-breaker."""
        if not options.get("use_geo_hint"):
            return score, {}

        evidence = {}
        geo_weight = 0.05  # Very weak signal, tie-breaker only

        # Check for geo columns
        src_city = None
        src_state = None
        ref_city = None
        ref_state = None

        for col in source_row.index:
            col_lower = col.lower()
            if "city" in col_lower:
                src_city = source_row[col]
            elif "state" in col_lower or "province" in col_lower:
                src_state = source_row[col]

        for col in ref_row.index:
            col_lower = col.lower()
            if "city" in col_lower:
                ref_city = ref_row[col]
            elif "state" in col_lower or "province" in col_lower:
                ref_state = ref_row[col]

        # Apply nudge if geo matches
        if src_city and ref_city and str(src_city).lower() == str(ref_city).lower():
            score = score + (1 - score) * geo_weight
            metrics.geo_nudges += 1
            evidence["hit"] = True

        if src_state and ref_state and str(src_state).lower() == str(ref_state).lower():
            score = score + (1 - score) * geo_weight
            if "hit" not in evidence:
                metrics.geo_nudges += 1
            evidence["hit"] = True

        if evidence:
            evidence["anchor"] = "geo"

        return score, evidence

    async def _score_rows_with_candidates(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        normalized_maps: List[Dict[str, Any]],
        threshold: float,
        source_id_column: Optional[str],
        reference_id_column: Optional[str],
        candidate_payload: Dict[str, Any],
        job_id: Optional[str] = None,
        progress_callback: Optional[Callable] = None,
        blocking_strategy: Optional[Dict[str, Any]] = None,
    ) -> Tuple[pd.DataFrame, Dict[str, Any]]:
        """Score each source row against its candidate reference set."""

        start = time.perf_counter()
        row_durations: List[float] = []
        candidates = candidate_payload.get("candidates", [])
        if not candidates:
            return pd.DataFrame(), {
                "mode": "row_candidates",
                "rows_processed": 0,
                "matches": 0,
                "candidates_considered": 0,
                "fallback_rows": 0,
                "elapsed_ms": 0,
                "row_runtime_avg_ms": 0.0,
                "row_runtime_p95_ms": 0.0,
                "row_runtime_max_ms": 0.0,
                "max_concurrency": 0,
            }

        positions = candidate_payload.get("source_positions", [])
        fallback_positions = set(candidate_payload.get("fallback_positions", set()))

        max_concurrency_env = os.getenv("ROW_CANDIDATE_CONCURRENCY", "4")
        try:
            max_concurrency = max(1, int(max_concurrency_env))
        except ValueError:
            max_concurrency = 4

        semaphore = asyncio.Semaphore(max_concurrency)
        combined_results: List[pd.DataFrame] = []
        stats = {
            "mode": "row_candidates",
            "rows_processed": len(candidates),
            "matches": 0,
            "candidates_considered": 0,
            "fallback_rows": len(fallback_positions),
            "row_candidates_empty": sum(1 for cand in candidates if not cand),
            "elapsed_ms": 0,
            "row_runtime_avg_ms": 0.0,
            "row_runtime_p95_ms": 0.0,
            "row_runtime_max_ms": 0.0,
            "max_concurrency": max_concurrency,
        }

        async def score_row(index: int) -> pd.DataFrame:
            row_start = time.perf_counter()
            row_candidates = candidates[index] or []
            fallback = index in fallback_positions or not row_candidates
            stats["candidates_considered"] += len(row_candidates)

            try:
                src_label = positions[index]
            except IndexError:
                src_label = index

            try:
                src_slice = source_df.loc[[src_label]]
            except KeyError:
                src_slice = source_df.iloc[[index]]

            if not row_candidates or fallback:
                ref_slice = reference_df
            else:
                try:
                    ref_slice = reference_df.loc[row_candidates]
                except KeyError:
                    ref_slice = reference_df.iloc[row_candidates]

            if ref_slice.empty:
                fallback = True
                ref_slice = reference_df
            cfg = MatchConfig(
                job_id=f"{job_id or uuid.uuid4()}-row-{index}",
                src_path=src_slice,
                ref_path=ref_slice,
                maps=normalized_maps,
                threshold=int(threshold * 100),
                src_id_col=source_id_column,
                ref_id_col=reference_id_column,
                apply_blocking=False,
                source_block_col=None,
                ref_block_col=None,
                block_key_length=3,
            )

            try:
                async with semaphore:
                    row_df, _ = await run_match_async(cfg=cfg, progress_callback=None)

                if row_df.empty:
                    return row_df

                if "score" not in row_df.columns and "match_score" in row_df.columns:
                    row_df["score"] = row_df["match_score"]

                row_df["__row_position"] = index
                row_df["__fallback"] = fallback
                return row_df
            finally:
                row_durations.append(time.perf_counter() - row_start)

        tasks = [score_row(idx) for idx in range(len(candidates))]
        row_frames = await asyncio.gather(*tasks)

        for frame in row_frames:
            if frame is not None and not frame.empty:
                combined_results.append(frame)

        if combined_results:
            result_df = pd.concat(combined_results, ignore_index=True)
        else:
            result_df = pd.DataFrame()

        stats["matches"] = len(result_df)

        elapsed = time.perf_counter() - start
        stats["elapsed_ms"] = int(elapsed * 1000)
        if row_durations:
            total_row_runtime = sum(row_durations)
            avg_runtime = total_row_runtime / len(row_durations)
            sorted_durations = sorted(row_durations)
            p95_index = max(
                0,
                min(len(sorted_durations) - 1, int(0.95 * (len(sorted_durations) - 1))),
            )
            stats["row_runtime_avg_ms"] = avg_runtime * 1000
            stats["row_runtime_p95_ms"] = sorted_durations[p95_index] * 1000
            stats["row_runtime_max_ms"] = sorted_durations[-1] * 1000

        if "__fallback" in result_df.columns:
            fallback_flags = result_df["__fallback"].astype(bool)
            if "fallback_marker" not in result_df.columns:
                result_df["fallback_marker"] = fallback_flags
            else:
                result_df["fallback_marker"] = (
                    result_df["fallback_marker"] | fallback_flags
                )
            result_df = result_df.drop(columns=["__fallback"])

        # Ensure explanation column exists before downstream adjustments
        if "explanation" not in result_df.columns:
            result_df["explanation"] = ""

        return result_df, SimpleNamespace(**stats)

    def _decorate_with_evidence(
        self,
        result_df: pd.DataFrame,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        source_id_column: Optional[str],
        reference_id_column: Optional[str],
        candidate_payload: Optional[Dict[str, Any]],
        mappings: List[Dict[str, Any]],
        source_domain_lookup: Optional[Dict[str, str]],
        source_etld_lookup: Optional[Dict[str, str]],
        source_domain_priority: List[str],
        reference_domain_priority: List[str],
        threshold: float,
        metrics: MatchMetrics,
    ) -> pd.DataFrame:
        """Attach domain/name/web/geo evidence chips and compact explanation."""

        if result_df.empty:
            for column_name in (
                "reasons",
                "explanation",
                "matched_domain",
                "matched_family",
                "source_etld",
                "came_via_family",
                "source_key",
            ):
                if column_name not in result_df.columns:
                    result_df[column_name] = []
            return result_df

        env_name = os.getenv("ENV", "development").lower()
        prod_mode = env_name in ("prod", "production")

        def _resolve_lookup(
            df: pd.DataFrame, id_col: Optional[str]
        ) -> Dict[str, pd.Series]:
            lookup: Dict[str, pd.Series] = {}
            if df is None or df.empty:
                return lookup
            if id_col and id_col in df.columns:
                for _, row in df.iterrows():
                    key = row.get(id_col)
                    if key is None or (isinstance(key, float) and pd.isna(key)):
                        continue
                    norm_key = str(key).strip()
                    if not norm_key or norm_key.lower() == "nan":
                        continue
                    if norm_key not in lookup:
                        lookup[norm_key] = row
                for idx, row in df.iterrows():
                    lookup.setdefault(str(idx), row)
            else:
                for idx, row in df.iterrows():
                    lookup[str(idx)] = row
            return lookup

        source_lookup = _resolve_lookup(source_df, source_id_column)
        reference_lookup = _resolve_lookup(reference_df, reference_id_column)

        def _append_flag(flags_value: str, new_flag: str) -> str:
            if not new_flag:
                return flags_value or ""
            parts = [token for token in str(flags_value or "").split("|") if token]
            if new_flag not in parts:
                parts.append(new_flag)
            return "|".join(parts)

        def _add_chip(chip_list: List[str], chip: Optional[str]) -> None:
            if chip and chip not in chip_list:
                chip_list.append(chip)

        fallback_marker_series = result_df.get("fallback_marker")
        fallback_flags_list: List[bool] = []
        if fallback_marker_series is not None:
            try:
                fallback_flags_list = list(fallback_marker_series.astype(bool))
            except Exception:
                fallback_flags_list = [bool(val) for val in fallback_marker_series]

        source_domain_lookup = source_domain_lookup or {}
        source_etld_lookup = source_etld_lookup or {}
        source_domain_priority = source_domain_priority or []
        reference_domain_priority = reference_domain_priority or []

        def _normalize_key(val: Any) -> Optional[str]:
            if val is None:
                return None
            if isinstance(val, float) and pd.isna(val):
                return None
            text_val = str(val).strip()
            if not text_val or text_val.lower() == "nan":
                return None
            return text_val

        def _resolve_name_columns() -> Tuple[Optional[str], Optional[str]]:
            src_col = None
            ref_col = None
            for mapping in mappings:
                candidate_src = (
                    mapping.get("source")
                    or mapping.get("src")
                    or mapping.get("source_column")
                )
                candidate_ref = (
                    mapping.get("ref")
                    or mapping.get("reference")
                    or mapping.get("reference_column")
                )
                algo = (
                    mapping.get("preferred_algo")
                    or mapping.get("algorithm")
                    or mapping.get("similarity_measure")
                    or ""
                ).lower()
                if candidate_src and candidate_ref:
                    if "token" in algo or "name" in str(candidate_src).lower():
                        src_col = candidate_src
                        ref_col = candidate_ref
                        break
            if not src_col:
                for col in source_df.columns:
                    if "name" in str(col).lower():
                        src_col = col
                        break
            if not ref_col:
                for col in reference_df.columns:
                    if "name" in str(col).lower():
                        ref_col = col
                        break
            return src_col, ref_col

        name_source_col, name_ref_col = _resolve_name_columns()

        def _pick_column(df: pd.DataFrame, keywords: Tuple[str, ...]) -> Optional[str]:
            for col in df.columns:
                lower = str(col).lower()
                if any(keyword in lower for keyword in keywords):
                    return col
            return None

        website_source_col = _pick_column(source_df, ("website", "url"))
        website_ref_col = _pick_column(reference_df, ("website", "url"))

        def _extract_geo(row: pd.Series) -> Tuple[Optional[str], Optional[str]]:
            city = None
            state = None
            for col in row.index:
                val = row.get(col)
                if val is None or (isinstance(val, float) and pd.isna(val)):
                    continue
                lower = str(col).lower()
                if city is None and "city" in lower:
                    city = str(val).strip()
                elif state is None and any(
                    token in lower for token in ("state", "province", "region")
                ):
                    state = str(val).strip()
            return city, state

        threshold_pct = float(threshold * 100)

        reasons_list: List[str] = []
        explanations: List[str] = []
        matched_domains: List[str] = []
        matched_families: List[str] = []
        source_etlds: List[str] = []
        came_via_family_flags: List[bool] = []
        source_keys_output: List[str] = []

        family_payload = (
            candidate_payload.get("family_etlds", {}) if candidate_payload else {}
        )
        source_etld_payload = (
            candidate_payload.get("source_etld", {}) if candidate_payload else {}
        )

        for idx, row in result_df.iterrows():
            key_variants: List[str] = []
            if source_id_column and source_id_column in row.index:
                normalized = _normalize_key(row.get(source_id_column))
                if normalized:
                    key_variants.append(normalized)
            for alt_col in ("source_id", "src_id", "srcid", "source_key"):
                if alt_col in row.index:
                    normalized = _normalize_key(row.get(alt_col))
                    if normalized:
                        key_variants.append(normalized)
            key_variants.append(str(idx))
            key_variants = list(dict.fromkeys([kv for kv in key_variants if kv]))
            source_key = key_variants[0] if key_variants else str(idx)

            src_series = None
            for candidate_key in key_variants:
                if candidate_key in source_lookup:
                    src_series = source_lookup.get(candidate_key)
                    source_key = candidate_key
                    break
            if src_series is None:
                src_series = source_lookup.get(str(idx))

            ref_key_variants: List[str] = []
            if reference_id_column and reference_id_column in row.index:
                normalized = _normalize_key(row.get(reference_id_column))
                if normalized:
                    ref_key_variants.append(normalized)
            for alt_col in ("reference_id", "target_id", "ref_id"):
                if alt_col in row.index:
                    normalized = _normalize_key(row.get(alt_col))
                    if normalized:
                        ref_key_variants.append(normalized)
            ref_key_variants.append(str(idx))
            ref_key_variants = list(
                dict.fromkeys([kv for kv in ref_key_variants if kv])
            )
            ref_series = None
            ref_key = ref_key_variants[0] if ref_key_variants else str(idx)
            for candidate_key in ref_key_variants:
                if candidate_key in reference_lookup:
                    ref_series = reference_lookup.get(candidate_key)
                    ref_key = candidate_key
                    break

            source_keys_output.append(source_key or "")

            source_domain_value = None
            source_etld = None
            for candidate_key in key_variants:
                if (
                    source_domain_value is None
                    and candidate_key in source_domain_lookup
                ):
                    source_domain_value = source_domain_lookup.get(candidate_key)
                if source_etld is None and candidate_key in source_etld_lookup:
                    source_etld = source_etld_lookup.get(candidate_key)
            if source_domain_value is None and src_series is not None:
                source_domain_value = self._derive_domain_from_columns(
                    src_series, source_domain_priority
                )
                if source_domain_value:
                    for candidate_key in key_variants:
                        source_domain_lookup.setdefault(
                            candidate_key, source_domain_value
                        )
            if source_etld is None and source_domain_value:
                source_etld = self._extract_etld1(source_domain_value)
                if source_etld:
                    for candidate_key in key_variants:
                        source_etld_lookup.setdefault(candidate_key, source_etld)
            if source_etld is None and candidate_payload:
                for candidate_key in key_variants:
                    payload_etld = source_etld_payload.get(candidate_key)
                    if payload_etld:
                        source_etld = str(payload_etld).lower()
                        break
            if source_etld:
                source_etld = str(source_etld).lower()

            matched_domain = None
            if "matched_domain" in row.index:
                candidate_matched = row.get("matched_domain")
                if candidate_matched is not None and not pd.isna(candidate_matched):
                    candidate_str = str(candidate_matched)
                    matched_domain = (
                        self._extract_etld1(candidate_str) or candidate_str.lower()
                    )

            ref_domain_value = None
            ref_etld_value = None
            if ref_series is not None:
                if "_ref_etld" in ref_series.index:
                    ref_etld_candidate = ref_series.get("_ref_etld")
                    if ref_etld_candidate is not None and not pd.isna(
                        ref_etld_candidate
                    ):
                        ref_etld_value = str(ref_etld_candidate).lower()
                if "_ref_domain_raw" in ref_series.index:
                    ref_domain_candidate = ref_series.get("_ref_domain_raw")
                    if ref_domain_candidate is not None and not (
                        isinstance(ref_domain_candidate, float)
                        and pd.isna(ref_domain_candidate)
                    ):
                        ref_domain_value = str(ref_domain_candidate).lower()
            if matched_domain is None and ref_etld_value:
                matched_domain = ref_etld_value
            if ref_domain_value is None and ref_series is not None:
                ref_domain_value = self._derive_domain_from_columns(
                    ref_series, reference_domain_priority
                )
                if ref_domain_value:
                    ref_domain_value = str(ref_domain_value).lower()
            if matched_domain is None and ref_domain_value:
                matched_domain = (
                    self._extract_etld1(ref_domain_value) or ref_domain_value
                )

            if matched_domain:
                matched_domain = str(matched_domain).lower()

            source_etlds.append(source_etld or "")
            matched_domains.append(matched_domain or "")

            family_value = str(row.get("matched_family") or "")
            same_family = False
            family_domains = set()
            if matched_domain and self._domain_family_service:
                try:
                    matched_family_obj = self._domain_family_service.get_family(
                        matched_domain
                    )
                except Exception:
                    matched_family_obj = None
                else:
                    if matched_family_obj:
                        if getattr(matched_family_obj, "canonical", None):
                            family_value = matched_family_obj.canonical or family_value
                        try:
                            family_domains = {
                                str(val).lower()
                                for val in matched_family_obj.all_domains()
                                if val
                            }
                        except Exception:
                            family_domains = set()
                        if source_etld and source_etld.lower() in family_domains:
                            same_family = True

            family_candidates = (
                family_payload.get(source_key, set()) if family_payload else set()
            )
            if isinstance(family_candidates, list):
                family_candidates = set(family_candidates)
            if family_candidates:
                family_candidates = {
                    str(val).lower() for val in family_candidates if val
                }
            came_via_family = bool(
                matched_domain
                and family_candidates
                and matched_domain in family_candidates
            )
            came_via_family_flags.append(came_via_family)

            chips: List[str] = []
            missing_precursor = not source_etld or not matched_domain
            if missing_precursor:
                metrics.missing_domain_evidence += 1
                if not prod_mode:
                    self.log.warning(
                        "Hunter evidence gap: missing domain data for source %s -> %s",
                        source_key,
                        ref_key,
                    )
                matched_domain = matched_domain or ""
                source_etld = source_etld or ""

            if source_etld and matched_domain:
                if source_etld == matched_domain:
                    _add_chip(chips, "DOM")
                elif came_via_family or same_family:
                    _add_chip(chips, "FAM")
                    metrics.family_hits += 1

            name_chip = None
            ratio = None
            if (
                name_source_col
                and name_ref_col
                and src_series is not None
                and ref_series is not None
            ):
                src_name = str(src_series.get(name_source_col, "")).strip()
                ref_name = str(ref_series.get(name_ref_col, "")).strip()
                if src_name and ref_name and _token_set_ratio:
                    try:
                        ratio = float(_token_set_ratio(src_name, ref_name))
                    except Exception:
                        ratio = None
            if ratio is not None:
                if ratio >= 90:
                    name_chip = "NAME90"
                elif ratio >= 80:
                    name_chip = "NAME80"
                elif ratio >= 70:
                    name_chip = "NAME70"
                elif ratio > 0:
                    name_chip = "NAME*"
            if not name_chip:
                raw_score = row.get("score", row.get("match_score", 0))
                try:
                    score_float = float(raw_score)
                except Exception:
                    score_float = 0.0
                score_pct_value = (
                    score_float * 100 if score_float <= 1.0 else score_float
                )
                if score_pct_value >= max(threshold_pct - 5, 50):
                    name_chip = "NAME*"
            if not name_chip:
                name_chip = "NAME*"

            _add_chip(chips, name_chip)

            if (
                website_source_col
                and website_ref_col
                and src_series is not None
                and ref_series is not None
            ):
                src_host = self._extract_host(src_series.get(website_source_col))
                ref_host = self._extract_host(ref_series.get(website_ref_col))
                if src_host and ref_host and src_host == ref_host:
                    _add_chip(chips, "WEB")

            if name_chip and src_series is not None and ref_series is not None:
                src_city, src_state = _extract_geo(src_series)
                ref_city, ref_state = _extract_geo(ref_series)
                geo_match = False
                if src_city and ref_city and src_city.lower() == ref_city.lower():
                    geo_match = True
                elif src_state and ref_state and src_state.lower() == ref_state.lower():
                    geo_match = True
                if geo_match:
                    _add_chip(chips, "GEO")
                    metrics.geo_nudges += 1

            matched_families.append(family_value)

            reason_str = "|".join(dict.fromkeys(chips))
            reasons_list.append(reason_str)

            raw_score = row.get("score", row.get("match_score", 0))
            try:
                score_float = float(raw_score)
            except Exception:
                score_float = 0.0
            score_pct = (
                round(score_float * 100, 1)
                if score_float <= 1.0
                else round(score_float, 1)
            )

            is_fallback = False
            if fallback_flags_list:
                try:
                    is_fallback = fallback_flags_list[idx]
                except Exception:
                    is_fallback = False

            flags_value = row.get("flags", "") or ""
            if is_fallback:
                flags_value = _append_flag(flags_value, "[NO_DOM_CANDIDATES]")
            if missing_precursor and prod_mode:
                flags_value = _append_flag(flags_value, "[MISSING_DOMAIN_DATA]")

            explanation_payload = {
                "chips": reason_str,
                "score": score_pct,
                "dom": matched_domain or "",
                "fam": family_value or "",
                "flags": flags_value,
                "source_etld": source_etld or "",
                "came_via_family": came_via_family,
                "source_key": source_key,
            }
            explanation_json = json.dumps(
                explanation_payload, separators=(",", ":"), ensure_ascii=False
            )
            if len(explanation_json) > 9500:
                explanation_json = explanation_json[:9500]
            explanations.append(explanation_json)

        result_df["reasons"] = reasons_list
        result_df["explanation"] = explanations
        result_df["matched_domain"] = matched_domains
        result_df["matched_family"] = matched_families
        result_df["source_etld"] = source_etlds
        result_df["came_via_family"] = came_via_family_flags
        result_df["source_key"] = [str(val or "") for val in source_keys_output]

        return result_df

    def _compute_domain_series(self, df: pd.DataFrame, columns: List[str]) -> pd.Series:
        """Vectorised helper to pull first registrable domain per row."""
        if df is None:
            return pd.Series(dtype="object")

        if df.empty:
            return pd.Series([None] * len(df), index=df.index, dtype="object")

        result = pd.Series([None] * len(df), index=df.index, dtype="object")

        if not columns:
            return result

        for col in columns:
            if not col or col not in df.columns:
                continue
            series = df[col]
            if series is None:
                continue
            extracted = series.map(self._extract_registrable_domain)
            if extracted is None:
                continue
            current_mask = result.isna()
            if not current_mask.any():
                break
            normalized = extracted.where(~extracted.isna(), None)
            assign_mask = current_mask & normalized.notna()
            if assign_mask.any():
                result.loc[assign_mask] = normalized.loc[assign_mask]

        return result

    def _derive_domain_from_columns(
        self,
        row: pd.Series,
        columns: List[str],
    ) -> Optional[str]:
        """Return the first registrable domain discovered using provided column priority."""
        if row is None:
            return None
        for col in columns:
            if not col:
                continue
            try:
                candidate = (
                    row.get(col) if hasattr(row, "get") else getattr(row, col, None)
                )
            except Exception:
                candidate = None
            if candidate is None:
                continue
            if isinstance(candidate, float) and pd.isna(candidate):
                continue
            domain = self._extract_registrable_domain(candidate)
            if domain:
                return domain.lower()
        return None

    def _extract_host(self, url: str) -> Optional[str]:
        """Extract and normalize host from URL."""
        if not url:
            return None
        import re

        url = url.lower().strip()

        # Remove protocol
        url = re.sub(r"^https?://", "", url)

        # Remove www prefix
        url = re.sub(r"^www\.", "", url)

        # Extract domain (before path)
        url = url.split("/")[0]

        # Remove port if present
        url = re.sub(r":\d+$", "", url)

        # Remove trailing dot (FQDN)
        url = url.rstrip(".")

        # Basic validation - must have at least one dot
        if url and "." in url:
            return url
        return None

    async def match_records(
        self,
        source_data: pd.DataFrame,
        reference_data: pd.DataFrame,
        mappings: List[Dict[str, Any]],  # REQUIRED - no default
        threshold: float = 0.75,  # 0-1 scale
        source_id_column: str = None,
        reference_id_column: str = None,
        blocking_strategy: Optional[Dict[str, Any]] = None,
        blocking_hints: Optional[Dict[str, Any]] = None,  # From discovery
        progress_callback: Optional[Callable] = None,
        job_id: Optional[str] = None,
        options: Optional[Dict[str, Any]] = None,  # Phase-2 B2B intelligence options
    ) -> MatchResult:
        """
        Match records between two datasets using the core engine.

        Args:
            source_data: Source DataFrame
            reference_data: Reference DataFrame to match against
            mappings: REQUIRED column mappings with weights (from discovery)
            threshold: Minimum match score (0-1 scale)
            source_id_column: Source ID column
            reference_id_column: Reference ID column
            blocking_strategy: Optional blocking configuration
            blocking_hints: Optional hints from discovery
            progress_callback: Optional callback for progress updates
            job_id: Optional job ID for tracking

        Returns:
            MatchResult with matches found
        """
        start_time = time.time()

        if not mappings:
            raise ValueError(
                "Mappings are required. Use discovery service to generate mappings."
            )

        self.log.info(
            f"Matching {len(source_data)} source records against "
            f"{len(reference_data)} reference records"
        )

        # Auto-detect ID columns if not provided
        if not source_id_column:
            source_id_column = self._detect_id_column(source_data)
        if not reference_id_column:
            reference_id_column = self._detect_id_column(reference_data)

        # Use blocking hints if no strategy provided
        if not blocking_strategy and blocking_hints:
            blocking_strategy = blocking_hints

        job_label = job_id or "inline"

        # Initialize metrics tracking for Phase-2
        metrics = MatchMetrics()
        options = options or {}

        # Set default options if not provided
        if "use_domain_family" not in options:
            options["use_domain_family"] = True
        if "use_website" not in options:
            options["use_website"] = True
        if "use_geo_hint" not in options:
            options["use_geo_hint"] = True

        # A) STRICT CANDIDATE PRE-FILTER with domain gating (per-row)
        gating_timer = time.perf_counter()
        original_ref_count = len(reference_data)
        domain_gated_index: Dict[str, List[int]] = defaultdict(list)
        use_row_candidates = self._should_use_row_candidates(
            source_row_count=len(source_data),
            options=options,
        )
        row_candidate_payload = {
            "candidates": [],
            "fallback_ids": set(),
            "source_positions": [],
            "source_ids": [],
            "candidate_etlds": {},
            "family_etlds": {},
            "source_etld": {},
            "source_domain": {},
            "fallback_positions": set(),
            "position_keys": [],
        }
        source_domain_lookup: Dict[str, str] = {}
        source_etld_lookup: Dict[str, str] = {}

        def _dedupe_preserve(items: List[Optional[str]]) -> List[str]:
            seen: Set[str] = set()
            ordered: List[str] = []
            for item in items:
                if not item:
                    continue
                key = str(item)
                if not key or key in seen:
                    continue
                seen.add(key)
                ordered.append(item)
            return ordered

        def _normalize_key(val: Any) -> Optional[str]:
            if val is None:
                return None
            if isinstance(val, float) and pd.isna(val):
                return None
            text_val = str(val).strip()
            if not text_val or text_val.lower() == "nan":
                return None
            return text_val

        source_lower_cols = {col: str(col).lower() for col in source_data.columns}
        reference_lower_cols = {col: str(col).lower() for col in reference_data.columns}

        email_domain_col = next(
            (
                col
                for col, lower in source_lower_cols.items()
                if "email" in lower and "domain" in lower
            ),
            None,
        )

        source_domain_cols = [
            col for col, lower in source_lower_cols.items() if "domain" in lower
        ]
        source_email_cols = [
            col
            for col, lower in source_lower_cols.items()
            if "email" in lower and "domain" not in lower
        ]
        source_website_cols = [
            col
            for col, lower in source_lower_cols.items()
            if any(
                token in lower
                for token in ("website", "url", "homepage", "site", "web")
            )
        ]

        source_domain_priority = _dedupe_preserve(
            [email_domain_col]
            + source_domain_cols
            + source_email_cols
            + source_website_cols
        )

        ref_domain_cols: List[str] = []
        for col, lower in reference_lower_cols.items():
            if "domain_append" in lower or "safe_domain_append" in lower:
                ref_domain_cols.append(col)
        for col, lower in reference_lower_cols.items():
            if "domain" in lower and col not in ref_domain_cols:
                ref_domain_cols.append(col)
        ref_website_cols = [
            col
            for col, lower in reference_lower_cols.items()
            if any(
                token in lower
                for token in ("website", "url", "homepage", "site", "web")
            )
        ]
        reference_domain_priority = _dedupe_preserve(ref_domain_cols + ref_website_cols)

        for mapping in mappings:
            src_field = (
                mapping.get("source")
                or mapping.get("src")
                or mapping.get("source_column")
            )
            if isinstance(src_field, str):
                lower = src_field.lower()
                if (
                    any(token in lower for token in ("domain", "website", "url"))
                    and src_field not in source_domain_priority
                ):
                    source_domain_priority.append(src_field)
            ref_field = (
                mapping.get("ref")
                or mapping.get("reference")
                or mapping.get("reference_column")
            )
            if isinstance(ref_field, str):
                lower = ref_field.lower()
                if (
                    any(token in lower for token in ("domain", "website", "url"))
                    and ref_field not in reference_domain_priority
                ):
                    reference_domain_priority.append(ref_field)

        def _lower_or_none(value: Any) -> Optional[str]:
            if value is None or (isinstance(value, float) and pd.isna(value)):
                return None
            text = str(value).strip()
            return text.lower() if text else None

        ref_domain_series = self._compute_domain_series(
            reference_data, reference_domain_priority
        )
        ref_domain_series = ref_domain_series.map(_lower_or_none)
        ref_etld_series = ref_domain_series.map(self._extract_etld1).map(_lower_or_none)

        for idx, etld in ref_etld_series.dropna().items():
            domain_gated_index[etld].append(idx)

        reference_data["_ref_domain_raw"] = ref_domain_series
        reference_data["_ref_etld"] = ref_etld_series
        ref_has_domains = bool(domain_gated_index)

        source_domain_series = self._compute_domain_series(
            source_data, source_domain_priority
        )
        source_domain_series = source_domain_series.map(_lower_or_none)
        src_etld_series = source_domain_series.map(self._extract_etld1).map(
            _lower_or_none
        )

        if source_id_column and source_id_column in source_data.columns:
            raw_id_series = source_data[source_id_column].map(_normalize_key)
        else:
            raw_id_series = pd.Series(
                [None] * len(source_data), index=source_data.index
            )

        index_key_series = pd.Series(
            [str(idx) for idx in source_data.index], index=source_data.index
        )
        primary_key_series = raw_id_series.combine_first(index_key_series)

        row_candidate_payload["source_positions"] = list(source_data.index)
        row_candidate_payload["source_ids"] = primary_key_series.tolist()
        row_candidate_payload["position_keys"] = primary_key_series.tolist()

        candidate_index_map: Dict[str, List[int]] = {}
        candidate_etlds_map: Dict[str, Set[str]] = {}
        family_only_map: Dict[str, Set[str]] = {}

        unique_source_etlds = {
            etld for etld in src_etld_series.dropna().unique() if etld
        }

        if unique_source_etlds and ref_has_domains:
            for etld in unique_source_etlds:
                candidate_etlds = {etld}
                family_only: Set[str] = set()
                if self._domain_family_service:
                    try:
                        family = self._domain_family_service.get_family(etld)
                    except Exception:
                        family = None
                    if family:
                        for alias in list(getattr(family, "aliases", []))[:10]:
                            alias_etld = self._extract_etld1(alias)
                            alias_norm = _lower_or_none(alias_etld)
                            if alias_norm:
                                candidate_etlds.add(alias_norm)
                                if alias_norm != etld:
                                    family_only.add(alias_norm)
                        for parent in list(getattr(family, "parents", []))[:3]:
                            parent_etld = self._extract_etld1(parent)
                            parent_norm = _lower_or_none(parent_etld)
                            if parent_norm:
                                candidate_etlds.add(parent_norm)
                                if parent_norm != etld:
                                    family_only.add(parent_norm)
                        for sub in list(getattr(family, "subsidiaries", []))[:10]:
                            sub_etld = self._extract_etld1(sub)
                            sub_norm = _lower_or_none(sub_etld)
                            if sub_norm:
                                candidate_etlds.add(sub_norm)
                                if sub_norm != etld:
                                    family_only.add(sub_norm)
                candidate_etlds_map[etld] = candidate_etlds
                family_only_map[etld] = family_only

                candidate_indices: Set[int] = set()
                for candidate_etld in candidate_etlds:
                    candidate_indices.update(domain_gated_index.get(candidate_etld, []))
                candidate_index_map[etld] = sorted(candidate_indices)

        candidate_lists: List[List[int]] = []
        fallback_ids: Set[str] = set()
        fallback_positions: Set[int] = set()
        domain_hits = 0
        row_candidates_empty = 0
        fallback_rows = 0

        for pos_idx, idx in enumerate(source_data.index):
            primary_key = primary_key_series.at[idx]
            norm_id = raw_id_series.at[idx]
            key_variants = [kv for kv in (norm_id, str(idx)) if kv]
            if primary_key not in key_variants:
                key_variants.insert(0, primary_key)
            key_variants = list(dict.fromkeys(key_variants))

            domain_value = source_domain_series.at[idx]
            etld_value = src_etld_series.at[idx]

            if domain_value:
                for key in key_variants:
                    source_domain_lookup.setdefault(key, domain_value)
                row_candidate_payload["source_domain"][primary_key] = domain_value

            if etld_value:
                for key in key_variants:
                    source_etld_lookup.setdefault(key, etld_value)
                row_candidate_payload["source_etld"][primary_key] = etld_value

            if etld_value and ref_has_domains:
                candidate_etlds = candidate_etlds_map.get(etld_value, {etld_value})
                family_only = family_only_map.get(etld_value, set())
            else:
                candidate_etlds = set()
                family_only = set()
            row_candidate_payload["candidate_etlds"][primary_key] = candidate_etlds
            row_candidate_payload["family_etlds"][primary_key] = family_only

            candidate_indices = candidate_index_map.get(etld_value, [])
            if candidate_indices:
                domain_hits += 1
                candidate_lists.append(candidate_indices)
            else:
                candidate_lists.append([])
                if etld_value and ref_has_domains:
                    row_candidates_empty += 1
                if ref_has_domains and use_row_candidates:
                    fallback_ids.add(primary_key)
                    fallback_positions.add(pos_idx)
                    fallback_rows += 1
                    if etld_value:
                        self.log.debug(
                            "[A] No domain matches for source %s (%s), marking for fallback",
                            idx,
                            etld_value,
                        )

        row_candidate_payload["candidates"] = candidate_lists
        row_candidate_payload["fallback_ids"] = fallback_ids
        row_candidate_payload["fallback_positions"] = fallback_positions

        metrics.domain_gated_rows = (
            int(src_etld_series.notna().sum()) if ref_has_domains else 0
        )
        metrics.domain_gated_hits = domain_hits
        metrics.row_candidates_empty = row_candidates_empty
        metrics.fallback_rows = fallback_rows
        gating_elapsed = time.perf_counter() - gating_timer
        if use_row_candidates:
            self.log.info(
                "[TIMING] job=%s phase=row_candidate_prepare rows=%d ref_rows=%d unique_src_domains=%d domain_hits=%d fallback_rows=%d elapsed=%.2fs",
                job_label,
                len(source_data),
                len(reference_data),
                len(unique_source_etlds),
                domain_hits,
                fallback_rows,
                gating_elapsed,
            )

        if ref_has_domains and not use_row_candidates and metrics.domain_gated_hits > 0:
            all_candidate_indices = set()
            for candidates in row_candidate_payload["candidates"]:
                if candidates:
                    all_candidate_indices.update(candidates)
            if all_candidate_indices:
                reference_data = reference_data.loc[list(all_candidate_indices)]
                self.log.info(
                    "[A] Domain pre-filter: %s/%s candidates (%s/%s sources found matches)",
                    len(reference_data),
                    original_ref_count,
                    metrics.domain_gated_hits,
                    metrics.domain_gated_rows,
                )

        if not ref_has_domains:
            row_candidate_payload = None
        # Apply domain family expansion (Phase-2)
        domain_evidence = []
        if options.get("use_domain_family"):
            reference_data, domain_evidence = self._apply_domain_family_expansion(
                source_data,
                reference_data,
                mappings,
                metrics,
                options,
                reference_id_column,
            )
            if domain_evidence:
                self.log.info(
                    f"[Phase-2] Domain family expansion: {metrics.family_hits} hits, {metrics.domain_expansions} expansions"
                )

        # Create match configuration with proper field names
        # Normalize maps for engine consumption
        normalized_maps = self._normalize_mappings(mappings)
        # Also build a tolerant view for downstream semantics/lookups
        tolerant_maps = self._norm_maps(mappings)

        if not normalized_maps:
            raise ValueError("No valid mappings after normalization")

        engine_start = time.time()
        if (
            use_row_candidates
            and row_candidate_payload
            and (
                any(row_candidate_payload["candidates"])
                or row_candidate_payload["fallback_ids"]
            )
        ):
            result_df, stats = await self._score_rows_with_candidates(
                source_df=source_data,
                reference_df=reference_data,
                normalized_maps=normalized_maps,
                threshold=threshold,
                source_id_column=source_id_column,
                reference_id_column=reference_id_column,
                candidate_payload=row_candidate_payload,
                job_id=job_id,
                progress_callback=progress_callback,
                blocking_strategy=blocking_strategy,
            )
            row_scoring_elapsed = getattr(stats, "elapsed_ms", 0) / 1000.0
            self.log.info(
                "[TIMING] job=%s phase=row_candidate_scoring rows=%d matches=%d candidates=%d fallbacks=%d concurrency=%d total_s=%.2f avg_ms=%.2f p95_ms=%.2f max_ms=%.2f",
                job_label,
                getattr(stats, "rows_processed", len(source_data)),
                getattr(stats, "matches", len(result_df)),
                getattr(stats, "candidates_considered", 0),
                getattr(stats, "fallback_rows", 0),
                getattr(stats, "max_concurrency", 0),
                row_scoring_elapsed,
                getattr(stats, "row_runtime_avg_ms", 0.0),
                getattr(stats, "row_runtime_p95_ms", 0.0),
                getattr(stats, "row_runtime_max_ms", 0.0),
            )
        else:
            config = MatchConfig(
                job_id=job_id or str(uuid.uuid4()),
                src_path=source_data,
                ref_path=reference_data,
                maps=normalized_maps,
                threshold=int(threshold * 100),
                src_id_col=source_id_column,
                ref_id_col=reference_id_column,
                apply_blocking=blocking_strategy is not None,
                source_block_col=blocking_strategy.get("source_column")
                if blocking_strategy
                else None,
                ref_block_col=blocking_strategy.get("reference_column")
                if blocking_strategy
                else None,
                block_key_length=blocking_strategy.get("key_length", 3)
                if blocking_strategy
                else 3,
            )
            result_df, stats = await run_match_async(
                cfg=config, progress_callback=progress_callback
            )
        metrics.engine_ms = int((time.time() - engine_start) * 1000)

        # Calculate config hash
        config_str = json.dumps(
            {"mappings": mappings, "threshold": threshold}, sort_keys=True, default=str
        )
        config_hash = hashlib.md5(config_str.encode()).hexdigest()[:8]

        elapsed_ms = int((time.time() - start_time) * 1000)

        # Count basic metrics
        if hasattr(stats, "candidates_considered"):
            metrics.candidates_total = stats.candidates_considered
        if hasattr(stats, "row_candidates_empty"):
            metrics.row_candidates_empty = stats.row_candidates_empty
        if hasattr(stats, "pairs_blocked"):
            metrics.blocked_pairs = stats.pairs_blocked

        # Count no-match rows (sources without matches)
        # Result_df typically only contains matched rows, so we need total source count
        total_sources = len(source_data)
        if source_id_column and source_id_column in result_df.columns:
            matched_sources = result_df[source_id_column].nunique()
            metrics.no_match_rows = total_sources - matched_sources
        else:
            # Fallback: assume all rows in result are matches
            metrics.no_match_rows = max(0, total_sources - len(result_df))

        # Standardize output format
        if "match_score" in result_df.columns:
            # Ensure scores are 0-1 scale
            result_df["score"] = result_df["match_score"]

        # Apply row-level fallback markers for domain-gated rows without candidates
        if row_candidate_payload and not result_df.empty:
            fallback_ids = {
                str(v) for v in row_candidate_payload.get("fallback_ids", set())
            }
            fallback_positions = set(
                row_candidate_payload.get("fallback_positions", set())
            )

            marker_series = None
            if (
                source_id_column
                and source_id_column in result_df.columns
                and fallback_ids
            ):
                marker_series = (
                    result_df[source_id_column].astype(str).isin(fallback_ids)
                )
            elif "__row_position" in result_df.columns and fallback_positions:
                marker_series = result_df["__row_position"].apply(
                    lambda idx: idx in fallback_positions
                )

            if marker_series is not None:
                result_df["fallback_marker"] = marker_series.astype(bool)
            elif "fallback_marker" not in result_df.columns:
                result_df["fallback_marker"] = False

        else:
            if "fallback_marker" not in result_df.columns:
                result_df["fallback_marker"] = False

        if "__row_position" in result_df.columns:
            result_df = result_df.drop(columns=["__row_position"])

        # B) CONFUSABLE BRAND PENALTY and C) SECTOR SANITY NUDGES
        if not result_df.empty and source_id_column and reference_id_column:
            # Find name columns
            src_name_col = None
            ref_name_col = None
            src_domain_col = None
            ref_domain_col = None

            for m in mappings:
                if m.get("weight", 0) >= 0.2:
                    algo = m.get("algorithm", "").lower()
                    if "token" in algo or "name" in m.get("source", "").lower():
                        src_name_col = m.get("source")
                        ref_name_col = m.get("ref") or m.get("reference")
                    elif "domain" in algo or "domain" in m.get("source", "").lower():
                        src_domain_col = m.get("source")
                        ref_domain_col = m.get("ref") or m.get("reference")

            # Apply adjustments row by row
            adjustments = []
            for idx, row in result_df.iterrows():
                score = row.get("score", row.get("match_score", 0))
                adjustment = 0
                chips = []

                # Get source and reference data
                src_id = row.get(source_id_column)
                ref_id = row.get(reference_id_column)

                # Safely get source and reference data
                try:
                    src_matches = source_data[source_data[source_id_column] == src_id]
                    src_data = (
                        src_matches.iloc[0]
                        if not src_matches.empty and src_id is not None
                        else None
                    )
                except (IndexError, KeyError):
                    src_data = None

                try:
                    ref_matches = reference_data[
                        reference_data[reference_id_column] == ref_id
                    ]
                    ref_data = (
                        ref_matches.iloc[0]
                        if not ref_matches.empty and ref_id is not None
                        else None
                    )
                except (IndexError, KeyError):
                    ref_data = None

                if src_data is not None and ref_data is not None:
                    key_candidates: List[str] = []
                    if src_id is not None and not (
                        isinstance(src_id, float) and pd.isna(src_id)
                    ):
                        norm_src = str(src_id).strip()
                        if norm_src and norm_src.lower() != "nan":
                            key_candidates.append(norm_src)
                    key_candidates.append(str(idx))
                    key_candidates = list(
                        dict.fromkeys([kv for kv in key_candidates if kv])
                    )

                    src_domain_value = None
                    src_etld_value = None
                    for candidate_key in key_candidates:
                        if (
                            src_domain_value is None
                            and candidate_key in source_domain_lookup
                        ):
                            src_domain_value = source_domain_lookup.get(candidate_key)
                        if (
                            src_etld_value is None
                            and candidate_key in source_etld_lookup
                        ):
                            src_etld_value = source_etld_lookup.get(candidate_key)
                    if src_domain_value is None:
                        src_domain_value = self._derive_domain_from_columns(
                            src_data, source_domain_priority
                        )
                    if src_domain_value:
                        src_domain_value = str(src_domain_value).lower()
                        for candidate_key in key_candidates:
                            source_domain_lookup.setdefault(
                                candidate_key, src_domain_value
                            )
                    if src_etld_value is None and src_domain_value:
                        src_etld_value = self._extract_etld1(src_domain_value)
                        if src_etld_value:
                            src_etld_value = str(src_etld_value).lower()
                    if src_etld_value:
                        for candidate_key in key_candidates:
                            source_etld_lookup.setdefault(candidate_key, src_etld_value)

                    ref_domain_value = None
                    ref_etld_value = None
                    if "_ref_domain_raw" in ref_data.index:
                        ref_domain_candidate = ref_data.get("_ref_domain_raw")
                        if ref_domain_candidate is not None and not (
                            isinstance(ref_domain_candidate, float)
                            and pd.isna(ref_domain_candidate)
                        ):
                            ref_domain_value = str(ref_domain_candidate).lower()
                    if "_ref_etld" in ref_data.index:
                        ref_etld_candidate = ref_data.get("_ref_etld")
                        if ref_etld_candidate is not None and not pd.isna(
                            ref_etld_candidate
                        ):
                            ref_etld_value = str(ref_etld_candidate).lower()
                    if ref_domain_value is None:
                        ref_domain_value = self._derive_domain_from_columns(
                            ref_data, reference_domain_priority
                        )
                        if ref_domain_value:
                            ref_domain_value = str(ref_domain_value).lower()
                    if ref_etld_value is None and ref_domain_value:
                        ref_etld_value = self._extract_etld1(ref_domain_value)
                        if ref_etld_value:
                            ref_etld_value = str(ref_etld_value).lower()

                    has_domain_support = bool(
                        src_etld_value
                        and ref_etld_value
                        and src_etld_value == ref_etld_value
                    )

                    # B) Confusable penalty
                    if src_name_col and ref_name_col:
                        src_name = src_data.get(src_name_col, "")
                        ref_name = ref_data.get(ref_name_col, "")
                        if (
                            self._is_confusable(src_name, ref_name)
                            and not has_domain_support
                        ):
                            adjustment -= 0.15  # -15 points on 0-1 scale
                            chips.append("CONF?")
                            metrics.confusable_penalties = (
                                metrics.confusable_penalties + 1
                            )

                    # C) Sector sanity nudge
                    if (src_domain_value or src_etld_value) and (
                        ref_domain_value or ref_etld_value
                    ):
                        ref_name = (
                            ref_data.get(ref_name_col, "") if ref_name_col else ""
                        )
                        sector_bias = self._get_sector_bias(
                            src_domain_value or src_etld_value or "",
                            ref_domain_value or ref_etld_value or "",
                            ref_name,
                        )
                        if sector_bias != 0:
                            adjustment += sector_bias / 100  # Convert to 0-1 scale
                            chips.append("SECT+" if sector_bias > 0 else "SECT-")

                # Apply adjustment
                new_score = max(0, min(1, score + adjustment))
                adjustments.append({"score": new_score, "chips": chips})

            # Update scores in result_df
            for i, adj in enumerate(adjustments):
                result_df.at[result_df.index[i], "score"] = adj["score"]
                if adj["chips"]:
                    # Add adjustment chips to explanation
                    existing = result_df.at[result_df.index[i], "explanation"]
                    result_df.at[result_df.index[i], "explanation"] = (
                        f"{existing} [{','.join(adj['chips'])}]".strip()
                    )

        result_df = self._decorate_with_evidence(
            result_df=result_df,
            source_df=source_data,
            reference_df=reference_data,
            source_id_column=source_id_column,
            reference_id_column=reference_id_column,
            candidate_payload=row_candidate_payload if use_row_candidates else None,
            mappings=mappings,
            source_domain_lookup=source_domain_lookup,
            source_etld_lookup=source_etld_lookup,
            source_domain_priority=source_domain_priority,
            reference_domain_priority=reference_domain_priority,
            threshold=threshold,
            metrics=metrics,
        )

        # Add feature scores column as fallback if raw field_scores absent
        if (
            "field_scores" not in result_df.columns
            and "feature_scores" not in result_df.columns
        ):
            result_df["feature_scores"] = [{}] * len(result_df)

        # Add explanations if not present
        if "explanation" not in result_df.columns:
            result_df["explanation"] = result_df.get("match_explanation", "")

        # Progressive pipeline annotation (WOW feature flag)
        if os.getenv("PROGRESSIVE_PIPELINE_ENABLED", "true").lower() in (
            "1",
            "true",
            "yes",
        ):
            try:
                result_df = self._annotate_progressive_details(result_df, threshold)
            except Exception as e:
                self.logger.debug(f"Progressive annotation skipped: {e}")

        # Ensure field_score_details exists (seed precedence: field_scores, feature_scores)
        if "field_score_details" not in result_df.columns:
            seed = (
                result_df["field_scores"]
                if "field_scores" in result_df.columns
                else result_df["feature_scores"]
                if "feature_scores" in result_df.columns
                else [{}] * len(result_df)
            )
            result_df["field_score_details"] = seed

        # Semantic fallback (borderline-only, conservative blending)
        try:
            # Lazy import metrics to avoid tight coupling
            try:
                from .metrics import metrics as _metrics
            except Exception:
                _metrics = None
            sem_enabled = os.getenv("SEMANTIC_FALLBACK_ENABLED", "false").lower() in (
                "1",
                "true",
                "yes",
            )
            if sem_enabled and not result_df.empty:
                try:
                    sem_band = float(os.getenv("SEMANTIC_BAND", "0.05"))
                except Exception:
                    sem_band = 0.05
                try:
                    sem_timeout_ms = int(os.getenv("SEMANTIC_TIMEOUT_MS", "150"))
                except Exception:
                    sem_timeout_ms = 150

                # Choose a representative mapping for semantics (prefer non-domain textual)
                chosen_map = None
                for m in tolerant_maps:
                    algo = (m.get("preferred_algo") or m.get("algorithm") or "").lower()
                    if algo != "domain":
                        chosen_map = m
                        break
                if chosen_map is None and tolerant_maps:
                    chosen_map = tolerant_maps[0]

                src_col = (chosen_map or {}).get("source")
                ref_col = (chosen_map or {}).get("ref")

                # Standardize id columns in result for lookup convenience
                if (
                    "source_id" not in result_df.columns
                    and "src_id" in result_df.columns
                ):
                    result_df["source_id"] = result_df["src_id"]
                if (
                    "target_id" not in result_df.columns
                    and "ref_id" in result_df.columns
                ):
                    result_df["target_id"] = result_df["ref_id"]

                # Only proceed if we can retrieve text pairs reliably
                can_lookup = (
                    isinstance(src_col, str)
                    and isinstance(ref_col, str)
                    and src_col in source_data.columns
                    and ref_col in reference_data.columns
                )
                if can_lookup:
                    # Build lookup dicts from ids to text values
                    try:
                        src_id_col = source_id_column
                        ref_id_col = reference_id_column
                        src_map = dict(
                            zip(
                                source_data[src_id_col].astype(str),
                                source_data[src_col].astype(str),
                            )
                        )
                        ref_map = dict(
                            zip(
                                reference_data[ref_id_col].astype(str),
                                reference_data[ref_col].astype(str),
                            )
                        )
                    except Exception as e:
                        src_map, ref_map = {}, {}
                        self.logger.debug(f"Semantic lookup maps failed to build: {e}")

                    # Borderline window around suggest threshold
                    low = max(0.0, threshold - sem_band)
                    high = min(1.0, threshold + sem_band)

                    # Iterate only borderline rows
                    from fmatch.saas.core import semantic_adapter as sem

                    for i, row in result_df.iterrows():
                        try:
                            base = float(
                                row.get("score", row.get("match_score", 0.0)) or 0.0
                            )
                        except Exception:
                            base = 0.0
                        if base < low or base > high:
                            continue
                        sid = str(row.get("source_id") or row.get("src_id") or "")
                        tid = str(row.get("target_id") or row.get("ref_id") or "")
                        a = src_map.get(sid, "")
                        b = ref_map.get(tid, "")
                        if not a or not b:
                            continue
                        if _metrics:
                            try:
                                # Count all semantic invocations
                                await _metrics.increment("semantic_invoked")
                            except Exception:
                                pass
                        sem_score = sem.score_pair(a, b, timeout_ms=sem_timeout_ms)
                        if not isinstance(sem_score, (float, int)):
                            continue
                        # Clamp for safety
                        sem_score = max(0.0, min(1.0, float(sem_score)))
                        # Conservatively blend
                        new_score = max(base, sem_score)
                        if new_score != base:
                            if _metrics:
                                try:
                                    await _metrics.increment("semantic_boosted")
                                except Exception:
                                    pass
                            result_df.at[i, "score"] = new_score
                            # Append semantic chip
                            cur = result_df.at[i, "field_score_details"]
                            try:
                                if isinstance(cur, list):
                                    cur.append(
                                        {
                                            "algo": "semantic",
                                            "contribution": sem_score,
                                            "reason": "borderline boost",
                                        }
                                    )
                                elif isinstance(cur, dict):
                                    cur = dict(cur)
                                    cur["semantic"] = sem_score
                                    result_df.at[i, "field_score_details"] = cur
                                else:
                                    result_df.at[i, "field_score_details"] = {
                                        "semantic": sem_score
                                    }
                            except Exception:
                                # Never fail matching due to explainability enrichments
                                pass
        except Exception as e:
            # Defensive: semantic fallback is best-effort only
            self.logger.debug(f"Semantic fallback skipped: {e}")

        # Note: Domain evidence is tracked in metrics, not per-row
        # This avoids index alignment issues when result_df is filtered/reordered
        if domain_evidence:
            self.log.info(
                f"[Phase-2] Domain family evidence collected for {len(domain_evidence)} expansions"
            )

        # Convert metrics dataclass to dict
        # Calculate percentages for Account Match with domain and Possible Match name-only
        if "reasons" in result_df.columns and "match_status" in result_df.columns:
            account_matches = result_df[result_df["match_status"] == "Account Match"]
            if len(account_matches) > 0:
                with_domain = (
                    account_matches["reasons"].str.contains("DOM|FAM", na=False).sum()
                )
                metrics.account_match_with_domain = int(
                    (with_domain / len(account_matches)) * 100
                )

            possible_matches = result_df[result_df["match_status"] == "Possible Match"]
            if len(possible_matches) > 0:
                name_only = (
                    possible_matches["reasons"].str.contains("NAME", na=False).sum()
                )
                no_domain = ~possible_matches["reasons"].str.contains(
                    "DOM|FAM", na=False
                )
                name_only_count = (
                    possible_matches["reasons"].str.contains("NAME", na=False)
                    & no_domain
                ).sum()
                metrics.possible_match_name_only = int(
                    (name_only_count / len(possible_matches)) * 100
                )

        metrics_dict = {
            "candidates_total": metrics.candidates_total,
            "blocked_pairs": metrics.blocked_pairs,
            "family_hits": metrics.family_hits,
            "website_hits": metrics.website_hits,
            "geo_nudges": metrics.geo_nudges,
            "b2c_filtered": metrics.b2c_filtered,
            "no_match_rows": metrics.no_match_rows,
            "engine_ms": metrics.engine_ms,
            "write_ms": metrics.write_ms,
            "domain_expansions": metrics.domain_expansions,
            # New surgical improvement metrics
            "domain_gated_rows": metrics.domain_gated_rows,
            "domain_gated_hits": metrics.domain_gated_hits,
            "confusable_penalties": metrics.confusable_penalties,
            "account_match_with_domain_pct": metrics.account_match_with_domain,
            "possible_match_name_only_pct": metrics.possible_match_name_only,
        }

        return MatchResult(
            matches=result_df,
            stats=stats.__dict__ if hasattr(stats, "__dict__") else stats,
            engine_version=ENGINE_VERSION,
            config_hash=config_hash,
            processing_time_ms=elapsed_ms,
            metrics=metrics_dict,
        )

    def _annotate_progressive_details(
        self, df: pd.DataFrame, threshold: float
    ) -> pd.DataFrame:
        """Lightweight progressive pipeline tagging and details.

        Stages:
        1) exact
        2) name_exact
        3) fuzzy_a
        4) fuzzy_b
        5) heuristics
        """
        stage_col = []
        details_col = []
        for _, row in df.iterrows():
            fsd = row.get("field_score_details") or row.get("feature_scores") or []
            stage = "heuristics"
            try:
                # Normalize to list[dict]
                if isinstance(fsd, str):
                    import json

                    fsd = json.loads(fsd)
                if isinstance(fsd, dict):
                    fsd = [fsd]
            except Exception:
                fsd = []

            # Heuristics to infer stage
            joined = json.dumps(fsd).lower() if fsd else ""
            if "exact" in joined or "domain" in joined or "email" in joined:
                stage = "exact_keys"
            elif "normalized" in joined or "name" in joined:
                stage = "name_normalized_exact"
            elif "wratio" in joined or "token_set" in joined:
                stage = "fuzzy_a"
            elif "token_sort" in joined or "partial_ratio" in joined:
                stage = "fuzzy_b"
            # Attach stage and keep details
            stage_col.append(stage)
            if fsd:
                details_col.append(fsd)
            else:
                details_col.append(
                    [
                        {
                            "stage": stage,
                            "signals": {"score": float(row.get("score") or 0.0)},
                            "score_contrib": float(row.get("score") or 0.0),
                        }
                    ]
                )

        df = df.copy()
        df["stage_hit"] = stage_col
        # Fill details if column missing OR individual cell is falsy
        if "field_score_details" not in df.columns:
            df["field_score_details"] = details_col
        else:
            current = df["field_score_details"].tolist()
            df["field_score_details"] = [
                c if c not in (None, {}, [], "") else b
                for c, b in zip(current, details_col)
            ]
        return df

    async def match_salesforce_records(
        self,
        source_records: List[Dict],  # From SF query (e.g., Leads)
        target_records: List[Dict],  # From SF query (e.g., Accounts)
        discovery_policy: Dict,  # REQUIRED - from discovery agent
        batch_size: int = 1000,
        progress_callback: Optional[Callable] = None,
    ) -> MatchResult:
        """
        Match Salesforce records using discovery policy.
        Handles batching and index reuse automatically for optimal performance.

        Args:
            source_records: List of source records from Salesforce
            target_records: List of target records from Salesforce
            discovery_policy: REQUIRED policy from discovery service with mappings
            batch_size: Size of batches for processing
            progress_callback: Optional progress callback

        Returns:
            MatchResult with all matches
        """
        start_time = time.time()

        if not discovery_policy:
            raise ValueError("Discovery policy is required for Salesforce matching")

        if "field_mappings" not in discovery_policy:
            raise ValueError("Discovery policy must include field_mappings")

        self.log.info(
            f"Matching {len(source_records)} source records against "
            f"{len(target_records)} target records using discovery policy"
        )

        # Convert to DataFrames
        source_df = pd.DataFrame(source_records)
        target_df = pd.DataFrame(target_records)

        # Get configuration from discovery policy
        mappings = discovery_policy["field_mappings"]
        threshold = discovery_policy.get("threshold", 0.75)

        # Extract blocking configuration from various possible locations
        apply_blocking = discovery_policy.get("apply_blocking", False)
        blocking_keys = discovery_policy.get("blocking_keys", [])

        # Also check nested blocking structure
        if not apply_blocking and discovery_policy.get("blocking"):
            blocking_dict = discovery_policy["blocking"]
            apply_blocking = blocking_dict.get("apply_blocking", False)
            blocking_keys = blocking_dict.get("keys", blocking_keys)

        self.log.info(
            "Salesforce match: %d source, %d target, %d mappings, blocking=%s",
            len(source_records),
            len(target_records),
            len(mappings),
            apply_blocking,
        )

        # Ensure threshold is 0-1 scale
        if threshold > 1:
            threshold = threshold / 100.0

        # Convert blocking_keys to legacy format for begin_reference_index
        legacy_blocking = None
        if apply_blocking and blocking_keys:
            first = blocking_keys[0]
            src_col = first.get("source_column") or first.get("column")
            ref_col = first.get("reference_column") or first.get("column")
            keylen = int(first.get("key_length", 3))
            legacy_blocking = {
                "source_column": src_col,
                "reference_column": ref_col,
                "key_length": keylen,
            }

        # Belt-and-suspenders: verify blocking column exists in reference data
        if (
            legacy_blocking
            and legacy_blocking["reference_column"] not in target_df.columns
        ):
            self.log.warning(
                "Blocking ref column '%s' not in reference; disabling blocking.",
                legacy_blocking["reference_column"],
            )
            apply_blocking = False
            blocking_keys = []
            legacy_blocking = None

        # Create reference index once for all batches
        index_id = await self.begin_reference_index(
            target_df,
            {
                "mappings": mappings,
                "blocking": legacy_blocking,  # Use legacy format
                "reference_id_column": discovery_policy.get("target_id_field", "Id"),
                "blocking_column": legacy_blocking and legacy_blocking["source_column"],
            },
        )

        try:
            all_matches = []
            total_batches = (len(source_df) + batch_size - 1) // batch_size

            for i in range(0, len(source_df), batch_size):
                batch_num = (i // batch_size) + 1
                batch = source_df.iloc[i : i + batch_size]

                self.log.info(f"Processing batch {batch_num}/{total_batches}")

                if progress_callback:
                    await progress_callback(
                        {
                            "phase": "matching",
                            "message": f"Processing batch {batch_num} of {total_batches}",
                            "progress": i / len(source_df),
                        }
                    )

                result = await self.match_with_index(
                    batch,
                    index_id,
                    {
                        "threshold": threshold,
                        "source_id_column": discovery_policy.get(
                            "source_id_field", "Id"
                        ),
                        "progress_callback": progress_callback,
                        "apply_blocking": apply_blocking,
                        "blocking_keys": blocking_keys,
                        "blocking": legacy_blocking,  # Also pass legacy format
                    },
                )

                # Standardize the output format
                matches_df = result.matches
                if not matches_df.empty:
                    # Ensure standard column names
                    if (
                        "source_id" not in matches_df.columns
                        and "src_id" in matches_df.columns
                    ):
                        matches_df["source_id"] = matches_df["src_id"]
                    if (
                        "target_id" not in matches_df.columns
                        and "ref_id" in matches_df.columns
                    ):
                        matches_df["target_id"] = matches_df["ref_id"]

                    all_matches.append(matches_df)

            # Combine all batch results
            if all_matches:
                final_matches = pd.concat(all_matches, ignore_index=True)
            else:
                final_matches = pd.DataFrame()

            elapsed_ms = int((time.time() - start_time) * 1000)

            # Calculate config hash for reproducibility
            config_str = json.dumps(
                {
                    "mappings": mappings,
                    "threshold": threshold,
                    "batch_size": batch_size,
                },
                sort_keys=True,
                default=str,
            )
            config_hash = hashlib.md5(config_str.encode()).hexdigest()[:8]

            return MatchResult(
                matches=final_matches,
                stats={
                    "total_source_records": len(source_records),
                    "total_target_records": len(target_records),
                    "batches_processed": total_batches,
                    "matches_found": len(final_matches),
                    "index_reuse_count": total_batches,
                },
                engine_version=ENGINE_VERSION,
                config_hash=config_hash,
                processing_time_ms=elapsed_ms,
            )

        finally:
            # Always release the index
            await self.release_index(index_id)

    async def dedupe(
        self,
        records: list[dict],
        discovery_policy: dict,  # expects field_mappings[], threshold, blocking hints, etc.
        id_field: str = "Id",
        batch_size: int = 1000,
        progress_callback=None,
        job_id: Optional[str] = None,
    ):
        """
        Back-compat wrapper for older services that expect engine.dedupe(...).
        Internally calls match_records against the same dataset and removes self-matches.
        Returns a MatchResult with matches between distinct records.
        """
        import pandas as pd

        df = pd.DataFrame(records)
        mappings = (
            discovery_policy.get("field_mappings")
            or discovery_policy.get("mappings")
            or []
        )
        threshold = float(discovery_policy.get("threshold", 0.75))

        # Ensure threshold is 0-1 scale
        if threshold > 1:
            threshold = threshold / 100.0

        # Preflight assertions - catch misconfig early
        if id_field not in df.columns:
            available = list(df.columns)[:10]  # Show first 10 columns
            raise ValueError(
                f"id_field '{id_field}' not found in DataFrame. Available columns: {available}..."
            )

        if df.empty:
            raise ValueError("Cannot dedupe empty DataFrame")

        # Domain enhancement for accounts with missing Website
        feature_flag = os.getenv("FEATURE_DOMAIN_ENHANCED", "false").lower() == "true"
        if feature_flag and "Name" in df.columns:
            try:
                from .domain_family import get_registry

                reg = get_registry()

                # Create derived domain column
                def derive_domain(name):
                    if not name:
                        return None
                    try:
                        suggestions = reg.suggest_domains(name)
                        if suggestions and suggestions[0][1] >= DOMAIN_MIN_CONFIDENCE:
                            return suggestions[0][0]
                    except Exception as e:
                        self.log.debug(f"Domain derivation failed for '{name}': {e}")
                    return None

                df["__derived_domain"] = df["Name"].apply(derive_domain)
                coverage = df["__derived_domain"].notna().mean()

                if coverage >= DOMAIN_MIN_COVERAGE_MAPPING:
                    # Add to mappings with reduced weight
                    derived_mapping = {
                        "source": "__derived_domain",
                        "reference": "__derived_domain",
                        "algorithm": "Domain",
                        "weight": DOMAIN_DERIVED_WEIGHT,  # Lower than explicit domains
                    }
                    mappings = list(mappings) if mappings else []
                    mappings.append(derived_mapping)

                # Only use for blocking if coverage is sufficient
                if (
                    coverage >= DOMAIN_MIN_COVERAGE_BLOCKING
                    and not discovery_policy.get("blocking", {}).get("apply_blocking")
                ):
                    # Set blocking strategy
                    discovery_policy = dict(discovery_policy)  # Make a copy
                    discovery_policy["blocking"] = {
                        "apply_blocking": True,
                        "keys": [{"column": "__derived_domain", "key_length": 3}],
                    }
                    self.log.info(
                        f"DomainEnhanced(Dedupe): {coverage:.1%} coverage, using for blocking"
                    )
                else:
                    self.log.info(
                        f"DomainEnhanced(Dedupe): {coverage:.1%} coverage, mapping only"
                    )
            except Exception as e:
                self.log.warning(f"Domain enhancement failed: {e}")
                # Continue without domain enhancement

        # Extract blocking configuration from discovery_policy
        # Handle both new format (from intelligent services) and legacy format
        blocking_config = discovery_policy.get("blocking", {})

        # Convert blocking config to engine format
        blocking_strategy = None
        blocking_hints = {
            "apply_blocking": False,
            "blocking_keys": [],
        }

        if isinstance(blocking_config, dict):
            if blocking_config.get("apply_blocking"):
                blocking_hints["apply_blocking"] = True
                keys = blocking_config.get("keys", [])
                if keys and isinstance(keys, list) and len(keys) > 0:
                    # Use first key for blocking strategy (engine currently supports single column blocking)
                    first_key = keys[0]
                    # Handle both 'source_column' (normalized) and 'column' (legacy) keys
                    column_name = first_key.get("source_column") or first_key.get(
                        "column"
                    )

                    # Validate column name before using it
                    if column_name and isinstance(column_name, str):
                        blocking_strategy = {
                            "source_column": column_name,
                            "key_length": first_key.get("key_length", 3),
                        }
                        blocking_hints["blocking_keys"] = keys
                        self.log.info(
                            f"Using blocking on column: {column_name} with key_length={first_key.get('key_length', 3)}"
                        )
                    else:
                        self.log.warning(
                            f"Invalid blocking column: {column_name} - disabling blocking"
                        )
                        blocking_hints["apply_blocking"] = False
                        blocking_strategy = None

        # Safety: inject default blocking if missing and dataset is large
        if len(records) > 500 and not blocking_hints.get("apply_blocking"):
            self.log.warning("Large dedupe without blocking - injecting safe defaults")

            # Try domain-based blocking first (safest for Accounts)
            domain_cols = ["Website", "website", "Domain", "domain", "Email", "email"]
            for col in domain_cols:
                if col in df.columns and df[col].notna().sum() > len(df) * 0.2:
                    # Extract domain and create blocking column
                    if "email" in col.lower():
                        df["__block_domain"] = (
                            df[col].str.lower().str.extract(r"@([^@]+)$", expand=False)
                        )
                    else:
                        df["__block_domain"] = df[col].apply(
                            self._extract_registrable_domain
                        )

                    if df["__block_domain"].notna().sum() > len(df) * 0.2:
                        blocking_strategy = {
                            "source_column": "__block_domain",
                            "key_length": 3,  # Use prefix blocking for domains
                        }
                        blocking_hints["apply_blocking"] = True
                        blocking_hints["blocking_keys"] = [
                            {
                                "column": "__block_domain",
                                "variant": "alnum",
                                "key_length": 3,
                            }
                        ]
                        self.log.info(
                            f"Using domain blocking from {col} (coverage: {df['__block_domain'].notna().sum() / len(df):.1%})"
                        )
                        break
            else:
                # Fall back to Name/Company blocking
                name_cols = [
                    "Name",
                    "name",
                    "CompanyName",
                    "company_name",
                    "Company",
                    "company",
                    "FirstName",
                    "LastName",
                    "first_name",
                    "last_name",
                ]
                for col in name_cols:
                    if col in df.columns and df[col].notna().sum() > len(df) * 0.2:
                        blocking_strategy = {
                            "source_column": col,
                            "key_length": 3,  # Conservative for names
                        }
                        blocking_hints["apply_blocking"] = True
                        blocking_hints["blocking_keys"] = [
                            {"column": col, "variant": "alnum", "key_length": 3}
                        ]
                        self.log.info(
                            f"Using name-based blocking on {col} (coverage: {df[col].notna().sum() / len(df):.1%})"
                        )
                        break

        # Validate blocking strategy if provided
        if blocking_strategy:
            if "source_column" in blocking_strategy:
                source_col = blocking_strategy.get("source_column")
                if not isinstance(source_col, str):
                    # Log more details about what went wrong
                    self.log.error(
                        f"Invalid blocking strategy: source_column={source_col}, type={type(source_col)}"
                    )
                    self.log.error(f"Full blocking_strategy: {blocking_strategy}")
                    self.log.error(f"Full blocking_config: {blocking_config}")
                    # Set blocking_strategy to None to avoid crash
                    blocking_strategy = None
                    blocking_hints["apply_blocking"] = False
                    self.log.warning("Disabling blocking due to invalid source_column")
                else:
                    # For dedupe, reference should use the same blocking column as source
                    blocking_strategy["reference_column"] = source_col
                    self.log.info(
                        f"Dedupe: Using same blocking column for both source and reference: {source_col}"
                    )

        # Normalize mappings defensively before passing to engine
        normalized_mappings = self._normalize_mappings(mappings)
        if not normalized_mappings:
            self.log.warning(
                "No valid mappings after normalization, using default name comparison"
            )
            # Fallback to basic name matching if no mappings provided
            normalized_mappings = [
                {
                    "source": "Name",
                    "ref": "Name",
                    "weight": 1.0,
                    "preferred_algo": "WRatio",
                    "algorithm": "WRatio",
                }
            ]

        # Create DedupeConfig for the engine
        dedupe_cfg = DedupeConfig(
            job_id=job_id or str(uuid.uuid4()),
            input_path=df,  # Can pass DataFrame directly
            rec_id_col=id_field,
            # The core engine only accepts canonical algorithm keys (e.g., "WRatio")
            # "l2a" is not a real algorithm in ALGORITHM_MAP.
            algorithm="WRatio",
            threshold=int(threshold * 100),  # Convert 0-1 to 0-100 scale
            ruleset_name="default",
            block_key_length=int(blocking_strategy.get("key_length", 3))
            if blocking_strategy
            else 3,
            dup_block_limit=1000,
            maps=normalized_mappings,  # Use normalized mappings with canonical algorithms
            apply_blocking=bool(blocking_strategy),
            # Tell the engine which column to DERIVE keys from; the engine writes the
            # actual block into its fixed column name (_ACTIVE_BLOCK_KEY_).
            source_block_col=blocking_strategy.get("source_column")
            if blocking_strategy
            else None,
        )

        # Explicit logging before async call for production visibility
        self.log.info(
            "EngineClient.dedupe(): mode=DUPLICATE alg=%s threshold=%s maps=%d blocking=%s",
            dedupe_cfg.algorithm,
            dedupe_cfg.threshold,
            len(dedupe_cfg.maps),
            dedupe_cfg.source_block_col or "none",
        )

        # Use the proper dedupe function from engine_bridge (DUPLICATE mode)
        # This was the fix: using dedupe instead of match_records
        result_df, run_stats = await run_dedupe_async(
            cfg=dedupe_cfg, progress_callback=progress_callback
        )

        # Convert to MatchResult format
        result = MatchResult(
            matches=result_df,
            stats={
                "total_comparisons": getattr(run_stats, "total_comparisons", 0),
                "matches_found": len(result_df),
                "duration": getattr(run_stats, "duration", 0),
                "engine_version": ENGINE_VERSION,
            },
        )

        return result

    # Note: The old dedupe method was using match_records which caused all matches
    # to be self-matches. Now fixed to use dedupe_records which properly uses DUPLICATE mode.

    async def score_pair(
        self,
        record1: Dict[str, Any],
        record2: Dict[str, Any],
        mappings: List[Dict[str, Any]],  # REQUIRED
    ) -> Dict[str, Any]:
        """
        Score a single pair of records using the engine.

        Args:
            record1: First record as dictionary
            record2: Second record as dictionary
            mappings: REQUIRED column mappings with weights

        Returns:
            Score details including confidence and field contributions
        """
        if not mappings:
            raise ValueError("Mappings are required for scoring")

        # Convert to DataFrames
        df1 = pd.DataFrame([record1])
        df2 = pd.DataFrame([record2])

        # Use match with threshold 0 to get all scores
        result = await self.match_records(
            source_data=df1,
            reference_data=df2,
            mappings=mappings,
            threshold=0.0,  # Get all scores
        )

        if not result.matches.empty:
            row = result.matches.iloc[0]
            score = row.get("score", row.get("match_score", 0))

            # Ensure score is 0-1 scale
            if score > 1:
                score = score / 100.0

            return {
                "source_id": row.get("source_id", record1.get("Id")),
                "target_id": row.get("target_id", record2.get("Id")),
                "score": score,
                "confidence_percent": int(score * 100),
                "tier": self._get_tier(score),
                "feature_scores": row.get(
                    "feature_scores", row.get("field_scores", {})
                ),
                "explanation": row.get("explanation", row.get("match_explanation", "")),
                "engine_version": ENGINE_VERSION,
                "processing_time_ms": result.processing_time_ms,
            }

        return {
            "source_id": record1.get("Id"),
            "target_id": record2.get("Id"),
            "score": 0.0,
            "confidence_percent": 0,
            "tier": "NO_MATCH",
            "feature_scores": {},
            "explanation": "No match found",
            "engine_version": ENGINE_VERSION,
        }

    def _build_blocking_index(
        self, df: pd.DataFrame, column: str, key_length: int = 3
    ) -> Dict[str, List[int]]:
        """Build a blocking index for fast lookups."""
        index = {}

        if column not in df.columns:
            return index

        for idx, value in enumerate(df[column]):
            if pd.notna(value):
                # Create blocking key
                key = str(value)[:key_length].lower()
                if key not in index:
                    index[key] = []
                index[key].append(idx)

        return index

    def _extract_registrable_domain(self, val: str) -> str:
        """
        Extract registrable domain from email or URL.
        Minimal implementation: strip scheme/paths, extract domain from email.
        """
        if pd.isna(val) or not val:
            return None

        val = str(val).lower().strip()

        # Handle email addresses
        if "@" in val:
            domain = val.split("@")[-1]
        else:
            # Handle URLs - remove scheme
            domain = (
                val.replace("http://", "").replace("https://", "").replace("www.", "")
            )
            # Remove path
            domain = domain.split("/")[0].split("?")[0].split("#")[0]

        # Basic validation - must have at least one dot
        if "." not in domain:
            return None

        return domain

    def _detect_id_column(self, df: pd.DataFrame) -> Optional[str]:
        """Auto-detect the ID column in a DataFrame."""
        import re

        candidates: list[tuple[str, str]] = []
        for col in df.columns:
            normalized = re.sub(r"[^a-z0-9]", "", str(col).lower())
            if not normalized:
                continue
            if normalized in {"id", "recordid", "_id"}:
                return col
            if normalized.endswith("id") or "id" in normalized:
                candidates.append((col, normalized))

        if candidates:
            for col, normalized in candidates:
                if normalized.endswith("id"):
                    return col
            return candidates[0][0]

        return df.columns[0] if len(df.columns) > 0 else None

    def _get_tier(self, score: float) -> str:
        """Get confidence tier from score (0-1 scale)."""
        if score >= 0.90:
            return "CERTAIN"
        elif score >= 0.75:
            return "LIKELY"
        elif score >= 0.50:
            return "POSSIBLE"
        elif score >= 0.30:
            return "REVIEW"
        else:
            return "UNLIKELY"


# Singleton instance
_engine_client = None


def get_engine_client() -> EngineClient:
    """Get the singleton engine client instance."""
    global _engine_client
    if _engine_client is None:
        _engine_client = EngineClient()
    return _engine_client
