import asyncio
import json
import logging
import math
import os
import string
import time
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Tuple
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor

import pandas as pd

try:
    from fmatch.saas.services.domain_utils import (
        extract_domain_from_email,
        extract_domain_from_url,
    )
    from fmatch.saas.services.domain_family import DomainRelation, DomainFamilyRegistry
    from fmatch.saas.services.domain_family_duckdb import DuckDBDomainRegistry
except Exception:  # pragma: no cover - keep service importable during tests
    from ..services.domain_utils import (
        extract_domain_from_email,
        extract_domain_from_url,
    )
    from ..services.domain_family import DomainRelation, DomainFamilyRegistry
    from ..services.domain_family_duckdb import DuckDBDomainRegistry


log = logging.getLogger(__name__)


@dataclass
class MappingOptions:
    casefold: bool = False
    strip_punctuation: bool = False


@dataclass
class MappingConfig:
    source: str
    reference: str
    algo: str
    weight: float
    opts: MappingOptions
    meta: dict = (
        None  # For preserving metadata like original_source, enriched_by_identity
    )

    def __post_init__(self):
        if self.meta is None:
            self.meta = {}


class ManualMatchServiceError(Exception):
    """Custom error for manual match failures."""


_SCORE_BANDS = [
    (0.90, "M:NAME90"),
    (0.80, "M:NAME80"),
    (0.70, "M:NAME70"),
]

_TOKEN_SET_FALLBACK = None
_JARO_FALLBACK = None


def _exact_match_score(a: str, b: str) -> float:
    return 1.0 if a and b and a == b else 0.0


def _contains_score(a: str, b: str) -> float:
    return 1.0 if a and b and (a in b or b in a) else 0.0


_JUNK_PLACEHOLDERS = {
    "n/a",
    "na",
    "n.a.",
    "none",
    "null",
    "unknown",
    "not provided",
    "[not provided]",
    "tbd",
    "tba",
    "test",
    "testing",
    "?",
}

_JUNK_GENERIC_WORDS = {
    "home",
    "job",
    "private",
    "self",
    "personal",
    "individual",
}


def _normalize_company_name(value: Any) -> str:
    if value is None:
        return ""
    return str(value).strip()


def _is_junk_company_name(value: str) -> bool:
    """Detect junk/placeholder company names that should not be matched."""
    if not value:
        return True

    lowered = value.lower()

    if lowered in _JUNK_PLACEHOLDERS or lowered in _JUNK_GENERIC_WORDS:
        return True

    alnum = "".join(ch for ch in lowered if ch.isalnum())
    if not alnum:
        return True

    if alnum.isdigit():
        return True

    if len(alnum) <= 1:
        return True

    if len(alnum) <= 2 and not alnum.isalpha():
        return True

    return False


def _has_name_token_overlap(source: str, reference: str) -> bool:
    """Return True when the company names share a meaningful token (len>=3)."""
    translator = str.maketrans("", "", string.punctuation)
    source_tokens = {
        token
        for token in source.translate(translator).lower().split()
        if len(token) >= 3
    }
    reference_tokens = {
        token
        for token in reference.translate(translator).lower().split()
        if len(token) >= 3
    }
    if not source_tokens or not reference_tokens:
        return False
    return bool(source_tokens.intersection(reference_tokens))


def _tokenize_company_name(value: str) -> List[str]:
    translator = str.maketrans("", "", string.punctuation)
    tokens = [
        token
        for token in value.translate(translator).lower().split()
        if token and not token.isdigit()
    ]
    return tokens


_PROCESS_POOL_CONTEXT: Dict[str, Any] = {}


def _extract_registry_config(domain_registry: Any) -> Optional[Dict[str, Any]]:
    """Extract picklable configuration from domain registry instance."""
    if domain_registry is None:
        return None

    # Check if it's a DuckDBDomainRegistry
    if hasattr(domain_registry, "db_path"):
        return {
            "type": "DuckDBDomainRegistry",
            "db_path": domain_registry.db_path,
            "ttl_s": getattr(domain_registry, "ttl_s", 300),
        }

    # Check if it's a YAML/seed-backed DomainFamilyRegistry
    if isinstance(domain_registry, DomainFamilyRegistry) or hasattr(
        domain_registry, "seed_path"
    ):
        return {
            "type": "DomainFamilyRegistry",
            "seed_path": getattr(domain_registry, "seed_path", None),
            "ttl_s": getattr(domain_registry, "ttl_s", 300),
            "learned_path": getattr(domain_registry, "learned_path", None),
        }

    # Unknown registry type
    return None


def _init_process_pool_worker(context: Dict[str, Any]) -> None:
    global _PROCESS_POOL_CONTEXT
    service_logger = logging.getLogger(context.get("logger_name", __name__))
    if os.getenv("FM_MATCH_TRACE", "").lower() in ("1", "true", "yes", "debug"):
        if not logging.getLogger().handlers:
            logging.basicConfig(level=logging.INFO)
        if service_logger.getEffectiveLevel() > logging.INFO:
            service_logger.setLevel(logging.INFO)

    # Recreate domain registry in worker process from config
    # Each worker creates its own instance to avoid threading.RLock pickle issues
    domain_registry = None
    registry_config = context.get("domain_registry_config")
    if registry_config:
        try:
            registry_type = registry_config.get("type")
            if registry_type == "DuckDBDomainRegistry":
                from fmatch.saas.services.domain_family_duckdb import (
                    DuckDBDomainRegistry,
                )

                domain_registry = DuckDBDomainRegistry(
                    db_path=registry_config.get("db_path"),
                    ttl_s=registry_config.get("ttl_s", 300),
                )
                service_logger.info(
                    f"[WORKER] DuckDB registry initialized: {domain_registry.to_dict()['total_families']} families"
                )
            elif registry_type == "DomainFamilyRegistry":
                from fmatch.saas.services.domain_family import DomainFamilyRegistry

                domain_registry = DomainFamilyRegistry(
                    seed_path=registry_config.get("seed_path"),
                    ttl_s=registry_config.get("ttl_s", 300),
                    learned_path=registry_config.get("learned_path"),
                )
                service_logger.info(
                    f"[WORKER] DomainFamilyRegistry initialized: {domain_registry.to_dict()['total_families']} families"
                )
            else:
                service_logger.warning(
                    f"[WORKER] Unknown registry type: {registry_type}"
                )
        except Exception as e:
            service_logger.warning(
                f"[WORKER] Could not initialize domain registry: {e}"
            )

    service = ManualMatchService(logger=service_logger, domain_registry=domain_registry)
    service.strip_b2c = context.get("strip_b2c", True)
    context_copy = dict(context)
    context_copy["service"] = service
    _PROCESS_POOL_CONTEXT = context_copy


def _process_rows_chunk_in_worker(chunk_indices: List[Any]) -> Dict[str, Any]:
    if not _PROCESS_POOL_CONTEXT:
        raise RuntimeError("Process pool worker context is not initialized")
    service = _PROCESS_POOL_CONTEXT["service"]
    return service._process_rows_chunk(
        chunk_indices,
        _PROCESS_POOL_CONTEXT["source_df"],
        _PROCESS_POOL_CONTEXT["reference_df"],
        _PROCESS_POOL_CONTEXT["runtime_mappings"],
        _PROCESS_POOL_CONTEXT["domain_mapping"],
        _PROCESS_POOL_CONTEXT["all_domain_mappings"],
        _PROCESS_POOL_CONTEXT["domain_index"],
        _PROCESS_POOL_CONTEXT["state_mapping"],
        _PROCESS_POOL_CONTEXT["state_index"],
        _PROCESS_POOL_CONTEXT["anchor_mapping"],
        _PROCESS_POOL_CONTEXT["anchor_kind"],
        _PROCESS_POOL_CONTEXT["engine_block_info"],
        _PROCESS_POOL_CONTEXT["best_only"],
        _PROCESS_POOL_CONTEXT["threshold"],
        _PROCESS_POOL_CONTEXT["source_name_col"],
        _PROCESS_POOL_CONTEXT["reference_name_col"],
        _PROCESS_POOL_CONTEXT.get("source_id_col"),
        _PROCESS_POOL_CONTEXT["reference_id_col"],
        _PROCESS_POOL_CONTEXT.get("source_id_lookup"),
    )


_PUNCT_TABLE = str.maketrans("", "", string.punctuation)


def _normalize_threshold(value: float) -> float:
    if value is None:
        return 0.8
    try:
        val = float(value)
    except (TypeError, ValueError):
        return 0.8
    if val > 1:
        val = val / 100.0
    return min(max(val, 0.0), 1.0)


def _normalize_weight(value: float) -> float:
    try:
        val = float(value)
    except (TypeError, ValueError):
        return 0.0
    if val > 1:
        val = val / 100.0
    return min(max(val, 0.0), 1.0)


_LOCAL_ALLOWED_ALGOS = {
    "exact",
    "contains",
    "domain",
    "website",
    "email",
    "email_domain",
}


def _requires_gateway(mappings: List[MappingConfig]) -> bool:
    for mapping in mappings:
        algo = (mapping.algo or "").strip().lower()
        if not algo or algo not in _LOCAL_ALLOWED_ALGOS:
            return True
    return False


def _map_algo_to_engine(algo: str) -> str:
    normalized = (algo or "").strip().lower()
    if normalized in ("domain", "website", "email_domain", "email"):
        return "ExactDomain"
    if normalized in ("exact",):
        return "Exact"
    if normalized in ("contains",):
        return "PartialRatio"
    if normalized in ("token_set", "token_set_ratio", "name", "fuzzy"):
        return "TokenSetRatio"
    if normalized in ("jaro",):
        return "Jaro"
    if normalized in ("jaro_winkler", "jarowinkler"):
        return "JaroWinkler"
    return "WRatio"


def _normalize_company_name(value: Any) -> str:
    """
    Normalize company names using desktop app logic:
    - Strip corporate suffixes (LLC, Inc, Corp, Ltd, GmbH, SA, etc.)
    - Remove FKA/AKA clauses
    - NFKC Unicode normalization
    - Remove extra punctuation and whitespace
    """
    if value is None or (isinstance(value, float) and math.isnan(value)):
        return ""

    text = str(value).strip()
    if not text:
        return ""

    # Import desktop app's company name normalizer
    try:
        from fmatch.utils_normalize import normalize_company_name

        return normalize_company_name(text)
    except ImportError:
        # Fallback to basic normalization
        import unicodedata

        # NFKC normalization
        text = unicodedata.normalize("NFKC", text)
        # Basic casefold
        text = text.casefold()
        # Strip extra whitespace
        text = " ".join(text.split())
        return text.strip()


def _normalize_text(value: Any, opts: MappingOptions) -> str:
    if value is None or (isinstance(value, float) and math.isnan(value)):
        text = ""
    else:
        text = str(value)
    if opts.casefold:
        text = text.casefold()
    if opts.strip_punctuation:
        text = text.translate(_PUNCT_TABLE)
    return text.strip()


def _normalize_domain_idna(domain: str) -> str:
    """
    Normalize domain using IDNA encoding/decoding for consistent handling of
    Unicode and punycode (xn--) domains.
    """
    try:
        # Encode to ASCII (handles Unicode → punycode)
        ascii_domain = domain.encode("idna").decode("ascii")
        # Decode back to Unicode for consistency
        return ascii_domain.lower()
    except (UnicodeError, UnicodeDecodeError):
        # If IDNA fails, just lowercase
        return domain.lower()


def _extract_domain(value: Any, strip_b2c: bool = True) -> tuple[Optional[str], bool]:
    """
    Extract registrable domain (eTLD+1) from URL/website field.
    Optionally strips B2C/personal domains (gmail.com, yahoo.com, etc.)

    Returns:
        tuple: (domain or None, was_b2c_stripped)

    Examples:
        https://www.wellstar.org/about -> ('wellstar.org', False)
        wellstar.org/contact -> ('wellstar.org', False)
        sub.wellstar.org -> ('wellstar.org', False)
        https://gmail.com (with strip_b2c=True) -> (None, True)
    """
    if value is None:
        return None, False
    text = str(value).strip()
    if not text:
        return None, False

    # First extract basic domain using existing function
    domain = extract_domain_from_url(text)
    if not domain:
        return None, False

    # IDNA normalization for Unicode/punycode safety
    domain = _normalize_domain_idna(domain)

    # Now extract eTLD+1 (registrable root) - LRU cached
    from fmatch.saas.core.psl_root import registrable_root

    etld1 = registrable_root(domain)
    if etld1:
        domain = etld1.lower()
    else:
        domain = domain.lower()

    # Check if B2C domain (strip personal email domains)
    if strip_b2c:
        from fmatch.core.b2c_bloom_filter import is_b2c_domain

        if is_b2c_domain(domain):
            return None, True  # Domain was B2C and stripped

    return domain, False


def _extract_email_domain(
    value: Any, strip_b2c: bool = True
) -> tuple[Optional[str], bool]:
    """
    Extract registrable domain (eTLD+1) from email address.
    Optionally strips B2C/personal email domains (gmail.com, yahoo.com, etc.)

    Returns:
        tuple: (domain or None, was_b2c_stripped)

    Examples:
        john@wellstar.org -> ('wellstar.org', False)
        john@mail.wellstar.org -> ('wellstar.org', False)
        john@gmail.com (with strip_b2c=True) -> (None, True)
    """
    if value is None:
        return None, False
    text = str(value).strip()
    if not text:
        return None, False

    # Try email extraction first
    domain = extract_domain_from_email(text)
    if not domain:
        # Fallback to URL extraction
        domain = extract_domain_from_url(text)
    if not domain and "@" in text:
        # Last resort: split on @
        domain = text.split("@", 1)[-1].lower()

    if not domain:
        return None, False

    # IDNA normalization for Unicode/punycode safety
    domain = _normalize_domain_idna(domain)

    # Extract eTLD+1 (registrable root) - LRU cached
    from fmatch.saas.core.psl_root import registrable_root

    etld1 = registrable_root(domain)
    if etld1:
        domain = etld1.lower()
    else:
        domain = domain.lower()

    # Check if B2C domain (strip personal email domains)
    if strip_b2c:
        from fmatch.core.b2c_bloom_filter import is_b2c_domain

        if is_b2c_domain(domain):
            return None, True  # Domain was B2C and stripped

    return domain, False


def _compute_reason_band(score: float) -> Optional[str]:
    for threshold, chip in _SCORE_BANDS:
        if score >= threshold:
            return chip
    if score >= 0.30:
        return "M:NAME*"
    return None


def _select_display_column(df: pd.DataFrame, preference: List[str]) -> Optional[str]:
    lowered = {col.lower(): col for col in df.columns}

    # First try exact preference matches
    for pref in preference:
        if pref.lower() in lowered:
            return lowered[pref.lower()]

    # Second, prioritize company/account/organization columns (but exclude person names and IDs)
    person_name_keywords = [
        "last name",
        "first name",
        "full name",
        "person name",
        "contact name",
        "employee",
        "owner",
    ]

    for col in df.columns:
        col_lower = col.lower()
        # Skip person name columns
        if any(keyword in col_lower for keyword in person_name_keywords):
            continue
        # Skip ID columns using robust detection
        # Match: "Account ID", "Lead_ID", "ContactID", "ID (18)", etc.
        # But allow: "Video", "identifier", "valid", etc. (where 'id' is part of word)
        has_id_marker = (
            " id" in col_lower
            or "_id" in col_lower
            or "id(" in col_lower
            or "id)" in col_lower
            or "id-" in col_lower
            or col_lower == "id"
        )
        # Special case: ends with 'id' but only if preceded by capital letter (camelCase like "ContactID")
        if not has_id_marker and col_lower.endswith("id") and len(col) > 2:
            # Check if there's a capital letter before 'id' or 'ID' in original
            if col[-3].isupper() or col.endswith("ID"):
                has_id_marker = True
        if has_id_marker:
            continue
        # Prioritize company/account/org columns
        if any(
            keyword in col_lower
            for keyword in [
                "company",
                "account",
                "organization",
                "org name",
                "business",
            ]
        ):
            if "name" in col_lower or "account" in col_lower:
                return col

    # Third, fall back to any column with "name" (excluding person names and IDs)
    for col in df.columns:
        col_lower = col.lower()
        if any(keyword in col_lower for keyword in person_name_keywords):
            continue
        # Skip ID columns (same logic as above)
        has_id_marker = (
            " id" in col_lower
            or "_id" in col_lower
            or "id(" in col_lower
            or "id)" in col_lower
            or "id-" in col_lower
            or col_lower == "id"
        )
        if not has_id_marker and col_lower.endswith("id") and len(col) > 2:
            if col[-3].isupper() or col.endswith("ID"):
                has_id_marker = True
        if has_id_marker:
            continue
        if "name" in col_lower:
            return col

    # Last resort: first column
    if len(df.columns):
        return df.columns[0]
    return None


def _select_id_column(df: pd.DataFrame) -> Optional[str]:
    candidates = ["id", "uuid", "account_id", "record_id", "source_id", "reference_id"]
    lowered = {col.lower(): col for col in df.columns}

    # First try exact matches
    for cand in candidates:
        if cand in lowered:
            return lowered[cand]

    # Second, look for columns containing "id" (handles "Account ID (18)", "Lead ID", etc.)
    for col in df.columns:
        col_lower = col.lower()
        # Match patterns like "account id", "lead id", "contact id", etc.
        if any(
            pattern in col_lower
            for pattern in ["account id", "lead id", "contact id", "record id"]
        ):
            return col

    # Third, look for columns ending with "id"
    for col in df.columns:
        if col.lower().endswith("id"):
            return col

    return None


@dataclass
class _RuntimeMapping:
    config: MappingConfig
    source_series: pd.Series
    reference_series: pd.Series
    score_func: Callable[[str, str], float]
    raw_source: pd.Series
    raw_reference: pd.Series


@dataclass
class _CandidateResult:
    reference_index: int
    final_score: float
    domain_score: float
    name_score: float
    status: str
    reasons: List[str]
    explanation: Dict[str, Any]
    binary_alert: bool = False
    binary_alert_sample: Optional[Tuple[str, str]] = None
    matched_family_id: Optional[str] = None


class ManualMatchService:
    """Core manual match execution service."""

    def __init__(
        self,
        logger: Optional[logging.Logger] = None,
        domain_registry: Optional[DuckDBDomainRegistry] = None,
    ) -> None:
        self.logger = logger or log
        self.domain_registry = domain_registry
        trace_flag = os.getenv("FM_MATCH_TRACE", "")
        self._trace_match_perf = trace_flag.lower() in ("1", "true", "yes", "debug")
        if self._trace_match_perf and self.logger.getEffectiveLevel() > logging.INFO:
            self.logger.setLevel(logging.INFO)

    def _resolve_id_columns(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        source_id_column: Optional[str],
        reference_id_column: Optional[str],
    ) -> Tuple[Optional[str], Optional[str], Optional[Dict[Any, str]]]:
        def _normalize_id_column_name(value: Optional[str]) -> Optional[str]:
            if value is None:
                return None
            text = str(value).strip()
            if not text or text.lower() == "auto":
                return None
            return text

        source_id_col = _normalize_id_column_name(source_id_column)
        if source_id_col and source_id_col not in source_df.columns:
            self.logger.warning(
                "[MANUAL MODE] Provided source ID column '%s' not found; falling back to auto-detect",
                source_id_col,
            )
            source_id_col = None
        if not source_id_col:
            source_id_col = _select_id_column(source_df)
        if source_id_col:
            self.logger.info("[MANUAL MODE] Using source ID column: %s", source_id_col)

        source_id_lookup: Optional[Dict[Any, str]] = None
        if source_id_col:
            id_series = source_df[source_id_col]
            source_id_lookup = {}
            for idx, raw_value in id_series.items():
                if pd.notna(raw_value):
                    text = str(raw_value).strip()
                    if text:
                        source_id_lookup[idx] = text
            if not source_id_lookup:
                source_id_lookup = None

        reference_id_col = None
        if reference_df is not None:
            reference_id_col = _normalize_id_column_name(reference_id_column)
            if reference_id_col and reference_id_col not in reference_df.columns:
                self.logger.warning(
                    "[MANUAL MODE] Provided reference ID column '%s' not found; falling back to auto-detect",
                    reference_id_col,
                )
                reference_id_col = None
            if not reference_id_col:
                reference_id_col = _select_id_column(reference_df)
            if reference_id_col:
                self.logger.info(
                    "[MANUAL MODE] Using reference ID column: %s", reference_id_col
                )

        return source_id_col, reference_id_col, source_id_lookup

    def _run_async(self, coro):
        try:
            loop = asyncio.get_running_loop()
        except RuntimeError:
            return asyncio.run(coro)
        if loop.is_running():
            with ThreadPoolExecutor(max_workers=1) as executor:
                future = executor.submit(asyncio.run, coro)
                return future.result()
        return loop.run_until_complete(coro)

    def _build_gateway_mappings(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        mappings: List[MappingConfig],
    ) -> List[Dict[str, Any]]:
        gateway_mappings: List[Dict[str, Any]] = []
        for mapping in mappings:
            source_col = str(mapping.source or "").strip()
            reference_col = str(mapping.reference or "").strip()
            if not source_col or not reference_col:
                continue
            if source_col not in source_df.columns or reference_col not in reference_df.columns:
                self.logger.warning(
                    "[MANUAL MODE] Skipping mapping %s -> %s (missing columns)",
                    source_col,
                    reference_col,
                )
                continue
            weight = _normalize_weight(mapping.weight)
            if weight <= 0:
                continue
            gateway_mappings.append(
                {
                    "source": source_col,
                    "reference": reference_col,
                    "weight": weight,
                    "preferred_algo": _map_algo_to_engine(mapping.algo),
                }
            )
        if not gateway_mappings:
            raise ManualMatchServiceError(
                "No valid mappings available for MatchGateway execution."
            )
        return gateway_mappings

    def _build_gateway_rows(
        self,
        matches_df: pd.DataFrame,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        threshold: float,
        best_only: bool,
        source_name_col: Optional[str],
        reference_name_col: Optional[str],
        source_id_col: Optional[str],
        reference_id_col: Optional[str],
        source_id_lookup: Optional[Dict[Any, str]],
        *,
        default_status: str = "No Match",
        default_reason: Optional[str] = None,
    ) -> List[Dict[str, Any]]:
        rows: List[Dict[str, Any]] = []

        def _resolve_source_identifier(source_idx: Any, source_row: pd.Series) -> str:
            if source_id_lookup is not None and source_idx in source_id_lookup:
                return source_id_lookup[source_idx]
            if source_id_col:
                raw_value = source_row.get(source_id_col)
                if raw_value is not None and not pd.isna(raw_value):
                    text = str(raw_value).strip()
                    if text:
                        return text
            try:
                return str(int(source_idx) + 1)
            except (TypeError, ValueError):
                return str(source_idx)

        def _resolve_reference_identifier(reference_idx: Any, reference_row: Optional[pd.Series]) -> str:
            if reference_row is not None and reference_id_col:
                raw_value = reference_row.get(reference_id_col)
                if raw_value is not None and not pd.isna(raw_value):
                    text = str(raw_value).strip()
                    if text:
                        return text
            return ""

        def _coerce_index(value: Any) -> Any:
            try:
                return int(value)
            except (TypeError, ValueError):
                return value

        def _json_explanation(payload: Any, fallback: Optional[Dict[str, Any]] = None) -> str:
            if isinstance(payload, str):
                return payload[:10000]
            data = payload if payload is not None else (fallback or {})
            try:
                return json.dumps(data)[:10000]
            except Exception:
                return "{}"

        if matches_df is None or matches_df.empty:
            for source_idx, source_row in source_df.iterrows():
                source_identifier = _resolve_source_identifier(source_idx, source_row)
                status = default_status
                reasons = default_reason or ""
                explanation = _json_explanation({"reason": default_reason} if default_reason else {})
                result = {
                    "source_id": source_identifier,
                    "source_name": str(source_row.get(source_name_col, "")) if source_name_col else "",
                    "candidate_rank": 1,
                    "reference_id": "",
                    "reference_name": "",
                    "match_score": 0,
                    "match_status": status,
                    "reasons": reasons,
                    "matched_domain": "",
                    "matched_family": "",
                    "b2c_explanation": "",
                    "source_row_index": _coerce_index(source_idx),
                    "reference_row_index": None,
                    "explanation": explanation,
                }
                if source_identifier:
                    result["source_key"] = source_identifier
                    result["source_record_id"] = source_identifier
                    result["source_primary_id"] = source_identifier
                rows.append(result)
            return rows

        df = matches_df.copy()
        if "score" not in df.columns:
            if "match_score" in df.columns:
                df["score"] = df["match_score"]
            elif "confidence_score" in df.columns:
                df["score"] = df["confidence_score"]
            else:
                df["score"] = 0.0
        if df["score"].max() > 1.0:
            df["score"] = df["score"] / 100.0
        if "source_id" not in df.columns:
            if "__source_id" in df.columns:
                df["source_id"] = df["__source_id"]
        if "reference_id" not in df.columns:
            if "__reference_id" in df.columns:
                df["reference_id"] = df["__reference_id"]

        matched_sources = set()
        grouped = df.groupby("source_id") if "source_id" in df.columns else []
        for source_id, group in grouped:
            source_idx = source_id
            if source_idx not in source_df.index:
                try:
                    coerced = int(source_id)
                except (TypeError, ValueError):
                    coerced = None
                if coerced is None or coerced not in source_df.index:
                    continue
                source_idx = coerced
            matched_sources.add(source_idx)
            group_sorted = group.sort_values("score", ascending=False)
            if best_only:
                group_sorted = group_sorted.head(1)
            for rank, (_, row) in enumerate(group_sorted.iterrows()):
                reference_idx = row.get("reference_id")
                if reference_idx not in reference_df.index:
                    try:
                        coerced_ref = int(reference_idx)
                    except (TypeError, ValueError):
                        coerced_ref = None
                    if coerced_ref is not None and coerced_ref in reference_df.index:
                        reference_idx = coerced_ref
                reference_row = (
                    reference_df.loc[reference_idx]
                    if reference_idx in reference_df.index
                    else None
                )
                source_row = source_df.loc[source_idx]
                source_identifier = _resolve_source_identifier(source_idx, source_row)
                reference_identifier = _resolve_reference_identifier(
                    reference_idx, reference_row
                )
                raw_score = row.get("score")
                try:
                    score = float(raw_score) if raw_score is not None and not pd.isna(raw_score) else 0.0
                except (TypeError, ValueError):
                    score = 0.0
                match_score = int(round(score * 100))
                match_status = row.get("match_status")
                if not match_status:
                    if score >= threshold:
                        match_status = "Account Match"
                    elif score > 0:
                        match_status = "Possible Match"
                    else:
                        match_status = "No Match"
                reasons = row.get("reasons") or ""
                if isinstance(reasons, list):
                    reasons = "|".join(str(r) for r in reasons if r)
                explanation = _json_explanation(
                    row.get("field_score_details") or row.get("explanation"),
                    {"threshold": threshold, "total_score": score},
                )
                result = {
                    "source_id": source_identifier,
                    "source_name": str(source_row.get(source_name_col, "")) if source_name_col else "",
                    "candidate_rank": rank + 1,
                    "reference_id": reference_identifier,
                    "reference_name": str(reference_row.get(reference_name_col, "")) if reference_row is not None and reference_name_col else "",
                    "match_score": match_score,
                    "match_status": match_status,
                    "reasons": reasons,
                    "matched_domain": row.get("matched_domain") or "",
                    "matched_family": row.get("matched_family") or "",
                    "b2c_explanation": "",
                    "source_row_index": _coerce_index(source_idx),
                    "reference_row_index": _coerce_index(reference_idx),
                    "explanation": explanation,
                }
                if source_identifier:
                    result["source_key"] = source_identifier
                    result["source_record_id"] = source_identifier
                    result["source_primary_id"] = source_identifier
                rows.append(result)

        for source_idx, source_row in source_df.iterrows():
            if source_idx in matched_sources:
                continue
            source_identifier = _resolve_source_identifier(source_idx, source_row)
            explanation = _json_explanation({"threshold": threshold, "total_score": 0.0})
            result = {
                "source_id": source_identifier,
                "source_name": str(source_row.get(source_name_col, "")) if source_name_col else "",
                "candidate_rank": 1,
                "reference_id": "",
                "reference_name": "",
                "match_score": 0,
                "match_status": default_status,
                "reasons": default_reason or "",
                "matched_domain": "",
                "matched_family": "",
                "b2c_explanation": "",
                "source_row_index": _coerce_index(source_idx),
                "reference_row_index": None,
                "explanation": explanation,
            }
            if source_identifier:
                result["source_key"] = source_identifier
                result["source_record_id"] = source_identifier
                result["source_primary_id"] = source_identifier
            rows.append(result)

        return rows

    def _execute_with_gateway(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        mappings: List[MappingConfig],
        threshold: float,
        best_only: bool,
        source_name_col: Optional[str],
        reference_name_col: Optional[str],
        source_id_col: Optional[str],
        reference_id_col: Optional[str],
        source_id_lookup: Optional[Dict[Any, str]],
    ) -> Tuple[pd.DataFrame, Dict[str, Any]]:
        try:
            from fmatch.saas.services.match_gateway import MatchGateway
        except Exception as exc:
            self.logger.warning("MatchGateway unavailable: %s", exc)
            rows = self._build_gateway_rows(
                matches_df=pd.DataFrame(),
                source_df=source_df,
                reference_df=reference_df,
                threshold=threshold,
                best_only=best_only,
                source_name_col=source_name_col,
                reference_name_col=reference_name_col,
                source_id_col=source_id_col,
                reference_id_col=reference_id_col,
                source_id_lookup=source_id_lookup,
                default_status="Needs Review",
                default_reason="NEEDS_REVIEW",
            )
            metrics = {
                "n_source": len(source_df),
                "n_reference": len(reference_df),
                "matches_found": 0,
                "matches_by_status": {"Needs Review": len(rows)},
                "gateway_available": False,
                "gateway_error": str(exc),
            }
            return pd.DataFrame(rows), metrics

        gateway_mappings = self._build_gateway_mappings(
            source_df, reference_df, mappings
        )

        async def _run_match():
            gateway = await MatchGateway.create()
            return await gateway.match(
                source_df,
                reference_df,
                threshold=threshold,
                mappings=gateway_mappings,
            )

        try:
            result = self._run_async(_run_match())
            matches_df = result.matches if result else pd.DataFrame()
        except Exception as exc:
            self.logger.warning("MatchGateway match failed: %s", exc)
            rows = self._build_gateway_rows(
                matches_df=pd.DataFrame(),
                source_df=source_df,
                reference_df=reference_df,
                threshold=threshold,
                best_only=best_only,
                source_name_col=source_name_col,
                reference_name_col=reference_name_col,
                source_id_col=source_id_col,
                reference_id_col=reference_id_col,
                source_id_lookup=source_id_lookup,
                default_status="Needs Review",
                default_reason="NEEDS_REVIEW",
            )
            metrics = {
                "n_source": len(source_df),
                "n_reference": len(reference_df),
                "matches_found": 0,
                "matches_by_status": {"Needs Review": len(rows)},
                "gateway_available": False,
                "gateway_error": str(exc),
            }
            return pd.DataFrame(rows), metrics

        rows = self._build_gateway_rows(
            matches_df=matches_df,
            source_df=source_df,
            reference_df=reference_df,
            threshold=threshold,
            best_only=best_only,
            source_name_col=source_name_col,
            reference_name_col=reference_name_col,
            source_id_col=source_id_col,
            reference_id_col=reference_id_col,
            source_id_lookup=source_id_lookup,
        )
        output_df = pd.DataFrame(rows)
        matches_by_status = (
            output_df["match_status"].value_counts().to_dict()
            if not output_df.empty and "match_status" in output_df.columns
            else {}
        )
        metrics = {
            "n_source": len(source_df),
            "n_reference": len(reference_df),
            "matches_found": int(
                matches_by_status.get("Account Match", 0)
                if matches_by_status
                else 0
            ),
            "matches_by_status": matches_by_status,
            "gateway_available": True,
        }
        if result and getattr(result, "stats", None) is not None:
            metrics["gateway_stats"] = result.stats
        if result and getattr(result, "config_used", None) is not None:
            metrics["gateway_threshold"] = getattr(result.config_used, "threshold", None)
        return output_df, metrics

    def execute(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        mappings: List[MappingConfig],
        anchors: Optional[Dict[str, Any]] = None,
        blocks: Optional[Dict[str, Any]] = None,
        threshold: float = 0.6,  # Lowered from 0.8 to match UI default
        best_only: bool = True,
        limit: Optional[int] = None,
        strip_b2c: bool = True,
        progress_callback: Optional[Callable[[int, int], None]] = None,
        source_id_column: Optional[str] = None,
        reference_id_column: Optional[str] = None,
    ) -> Tuple[pd.DataFrame, Dict[str, Any]]:
        if source_df is None or source_df.empty:
            raise ManualMatchServiceError("Source dataframe is empty")
        if reference_df is None or reference_df.empty:
            raise ManualMatchServiceError("Reference dataframe is empty")
        if not mappings:
            raise ManualMatchServiceError("At least one mapping must be provided")

        threshold = _normalize_threshold(threshold)
        source_df = source_df.copy()
        reference_df = reference_df.copy()
        self.strip_b2c = strip_b2c

        if limit is not None and limit > 0:
            source_df = source_df.head(int(limit))

        source_name_col = _select_display_column(source_df, ["company name", "name"])
        reference_name_col = _select_display_column(
            reference_df, ["account name", "name"]
        )
        source_id_col, reference_id_col, source_id_lookup = self._resolve_id_columns(
            source_df,
            reference_df,
            source_id_column,
            reference_id_column,
        )

        if _requires_gateway(mappings):
            return self._execute_with_gateway(
                source_df=source_df,
                reference_df=reference_df,
                mappings=mappings,
                threshold=threshold,
                best_only=best_only,
                source_name_col=source_name_col,
                reference_name_col=reference_name_col,
                source_id_col=source_id_col,
                reference_id_col=reference_id_col,
                source_id_lookup=source_id_lookup,
            )

        # BLOCKING GUARD: Prevent cartesian explosions with auto-blocking intelligence
        auto_blocking_result = self._enforce_blocking_guard(
            source_df, reference_df, blocks, mappings
        )

        engine_block_info: Optional[Dict[str, Any]] = None
        engine_strategy_info: Optional[Dict[str, Any]] = None
        if auto_blocking_result:
            if blocks is None:
                blocks = {}
            engine_block_info = auto_blocking_result.get("engine_block_info")
            strategy_info = auto_blocking_result.get("strategy") or {}
            engine_strategy_info = strategy_info
            blocks["engine_strategy"] = strategy_info
            blocks["engine_block_source_column"] = auto_blocking_result.get(
                "source_column"
            )
            blocks["engine_block_reference_column"] = auto_blocking_result.get(
                "reference_column"
            )
            blocks["engine_quality_score"] = auto_blocking_result.get("quality_score")

            engine_strategy_key = strategy_info.get("engine_block_strategy")
            if engine_strategy_key in ("domain", "email_domain", "domain_email"):
                blocks["domain"] = True
                blocks.pop("state", None)
            elif engine_strategy_key in ("state", "geo_state"):
                blocks["state"] = True
                blocks.pop("domain", None)
            else:
                blocks.pop("domain", None)
                blocks.pop("state", None)

            self.logger.debug(
                "Engine blocking strategy: %s", engine_strategy_key or "unknown"
            )

        runtime_mappings = self._prepare_mappings(source_df, reference_df, mappings)

        block_domain = bool(blocks and blocks.get("domain"))
        block_state = bool(blocks and blocks.get("state"))
        anchor_kind = (anchors or {}).get("require_exact_on") if anchors else None

        domain_mapping = (
            self._find_domain_mapping(runtime_mappings)
            if block_domain or self._requires_domain(runtime_mappings)
            else None
        )
        all_domain_mappings = (
            self._find_all_domain_mappings(runtime_mappings) if block_domain else []
        )
        state_mapping = (
            self._find_geo_mapping(runtime_mappings, "state") if block_state else None
        )
        anchor_mapping = self._resolve_anchor(runtime_mappings, anchor_kind)

        domain_index = (
            self._build_multi_domain_index(reference_df, all_domain_mappings)
            if all_domain_mappings and block_domain
            else None
        )
        state_index = (
            self._build_state_index(reference_df, state_mapping)
            if state_mapping and block_state
            else None
        )

        row_indices = list(source_df.index)
        chunk_results: List[Dict[str, Any]] = []
        binary_alert_samples: List[Tuple[str, str]] = []

        if row_indices:
            # Manual match jobs should use multiple cores regardless of runtime policy
            # The GSHEETS/SERVERLESS profiles throttle to max_workers=1, but manual jobs
            # benefit from parallelization since they run server-side, not in browser
            worker_count = 1
            try:
                from fmatch.core.engine import RUNTIME_AWARE

                if RUNTIME_AWARE:
                    from fmatch.core.engine_runtime import get_runtime_manager

                    runtime_manager = get_runtime_manager()
                    policy_workers = getattr(runtime_manager.policy, "max_workers", 1)

                    # Override throttled policies for server-side manual jobs
                    if policy_workers <= 1:
                        worker_count = max(1, min(16, os.cpu_count() or 1))
                        self.logger.info(
                            f"[PARALLEL] Runtime policy has max_workers={policy_workers}, overriding to {worker_count} for server-side manual job"
                        )
                    else:
                        worker_count = policy_workers
                        self.logger.info(
                            f"[PARALLEL] Using runtime policy max_workers={worker_count}"
                        )
                else:
                    worker_count = max(1, os.cpu_count() or 1)
                    self.logger.info(
                        f"[PARALLEL] RUNTIME_AWARE=False, using os.cpu_count()={worker_count}"
                    )
            except Exception as e:
                worker_count = max(1, os.cpu_count() or 1)
                self.logger.info(
                    f"[PARALLEL] Runtime import failed ({e}), using os.cpu_count()={worker_count}"
                )

            worker_count = max(1, min(worker_count, len(row_indices)))
            self.logger.info(
                f"[PARALLEL] Final worker_count={worker_count} (row_count={len(row_indices)})"
            )

            if worker_count <= 1 or len(row_indices) < 64:
                self.logger.info(
                    f"[PARALLEL] Using single-threaded path (worker_count={worker_count}, rows={len(row_indices)})"
                )
                chunk_results.append(
                    self._process_rows_chunk(
                        row_indices,
                        source_df,
                        reference_df,
                        runtime_mappings,
                        domain_mapping,
                        all_domain_mappings,
                        domain_index,
                        state_mapping,
                        state_index,
                        anchor_mapping,
                        anchor_kind,
                        engine_block_info,
                        best_only,
                        threshold,
                        source_name_col,
                        reference_name_col,
                        source_id_col,
                        reference_id_col,
                        source_id_lookup,
                    )
                )
            else:
                chunk_size = max(1, math.ceil(len(row_indices) / worker_count))
                chunks = [
                    row_indices[i : i + chunk_size]
                    for i in range(0, len(row_indices), chunk_size)
                ]
                context = {
                    "logger_name": getattr(self.logger, "name", __name__),
                    "strip_b2c": getattr(self, "strip_b2c", True),
                    "domain_registry_config": _extract_registry_config(
                        getattr(self, "domain_registry", None)
                    ),
                    "source_df": source_df,
                    "reference_df": reference_df,
                    "runtime_mappings": runtime_mappings,
                    "domain_mapping": domain_mapping,
                    "all_domain_mappings": all_domain_mappings,
                    "domain_index": domain_index,
                    "state_mapping": state_mapping,
                    "state_index": state_index,
                    "anchor_mapping": anchor_mapping,
                    "anchor_kind": anchor_kind,
                    "engine_block_info": engine_block_info,
                    "best_only": best_only,
                    "threshold": threshold,
                    "source_name_col": source_name_col,
                    "reference_name_col": reference_name_col,
                    "source_id_col": source_id_col,
                    "reference_id_col": reference_id_col,
                    "source_id_lookup": source_id_lookup,
                }
                self.logger.info(
                    f"[PARALLEL] Using ProcessPoolExecutor with {worker_count} workers, {len(chunks)} chunks of ~{chunk_size} rows each"
                )

                # Track progress for incremental updates
                total_rows = len(row_indices)
                processed_rows = 0

                with ProcessPoolExecutor(
                    max_workers=worker_count,
                    initializer=_init_process_pool_worker,
                    initargs=(context,),
                ) as executor:
                    for chunk_idx, chunk_result in enumerate(
                        executor.map(_process_rows_chunk_in_worker, chunks)
                    ):
                        chunk_results.append(chunk_result)

                        # Update progress after each chunk completes
                        processed_rows += len(chunks[chunk_idx])
                        if progress_callback:
                            try:
                                progress_callback(processed_rows, total_rows)
                            except Exception as e:
                                self.logger.warning(f"[PROGRESS] Callback failed: {e}")

            result_rows: List[Dict[str, Any]] = []
            total_candidates = 0
            blocked_rows = 0
            engine_blocked_rows = 0
            engine_residual_rows = 0
            blocked_zero_but_has_alt = 0
            saturation_hits = 0
            all_scores: List[float] = []
            scores_by_status = {
                "Account Match": [],
                "Possible Match": [],
                "No Match": [],
            }
            reasons_counter: Dict[str, int] = {}
            candidates_per_row_list: List[int] = []
            binary_alerts_total = 0

            for chunk_result in chunk_results:
                result_rows.extend(chunk_result["rows"])
                total_candidates += chunk_result["total_candidates"]
                blocked_rows += chunk_result["blocked_rows"]
                engine_blocked_rows += chunk_result["engine_blocked_rows"]
                engine_residual_rows += chunk_result["engine_residual_rows"]
                blocked_zero_but_has_alt += chunk_result["blocked_zero_but_has_alt"]
                saturation_hits += chunk_result["saturation_hits"]
                all_scores.extend(chunk_result["all_scores"])
                for status, values in chunk_result["scores_by_status"].items():
                    scores_by_status[status].extend(values)
                for reason, count in chunk_result["reasons_counter"].items():
                    reasons_counter[reason] = reasons_counter.get(reason, 0) + count
                candidates_per_row_list.extend(chunk_result["candidates_per_row"])
                binary_alerts_total += chunk_result["binary_alerts"]
                for sample in chunk_result["binary_alert_samples"]:
                    if len(binary_alert_samples) < 5:
                        binary_alert_samples.append(sample)

            if binary_alerts_total > 0:
                self._binary_score_alerts = binary_alerts_total
                for src_sample, ref_sample in binary_alert_samples:
                    self.logger.warning(
                        "[BINARY_SCORE_ALERT] domain=1.0 name=0.0 src_name='%s' ref_name='%s'",
                        src_sample,
                        ref_sample,
                    )
            elif hasattr(self, "_binary_score_alerts"):
                delattr(self, "_binary_score_alerts")

        matches_found = sum(
            1 for row in result_rows if row["match_status"] == "Account Match"
        )

        # Calculate diagnostic metrics for data quality
        source_blank_name_count = 0
        source_has_email_domain_count = 0
        source_has_website_count = 0
        source_domains = set()
        reference_domains = set()

        if source_name_col:
            source_blank_name_count = sum(
                1
                for _, row in source_df.iterrows()
                if not str(row.get(source_name_col, "")).strip()
            )

        # Collect domains from ALL domain mappings (not just first)
        email_rows_seen = set()
        website_rows_seen = set()
        for dm in all_domain_mappings:
            for row_name, domain_val in dm.source_series.items():
                if domain_val:
                    source_domains.add(domain_val)
                    # Count unique rows per type (avoid double-counting)
                    if (
                        dm.config.algo == "email_domain"
                        and row_name not in email_rows_seen
                    ):
                        source_has_email_domain_count += 1
                        email_rows_seen.add(row_name)
                    elif (
                        dm.config.algo == "domain" and row_name not in website_rows_seen
                    ):
                        source_has_website_count += 1
                        website_rows_seen.add(row_name)

            for row_name, domain_val in dm.reference_series.items():
                if domain_val:
                    reference_domains.add(domain_val)

        # Upper bound based on unified domain index (any domain column match)
        domain_overlap_upper_bound = len(source_domains & reference_domains)

        # Calculate percentiles
        import numpy as np

        def percentiles(data):
            if not data:
                return {"p50": 0, "p90": 0, "p99": 0, "max": 0}
            arr = np.array(data)
            return {
                "p50": float(np.percentile(arr, 50)),
                "p90": float(np.percentile(arr, 90)),
                "p99": float(np.percentile(arr, 99)),
                "max": float(np.max(arr)),
            }

        # Build comprehensive metrics
        metrics = {
            # Basic counts
            "n_source": int(len(source_df)),
            "n_reference": int(len(reference_df)),
            "rows_evaluated": int(len(source_df)),
            # Matches by status
            "matches_by_status": {
                "Account Match": len(scores_by_status["Account Match"]),
                "Possible Match": len(scores_by_status["Possible Match"]),
                "No Match": len(scores_by_status["No Match"]),
            },
            "matches_found": matches_found,
            "account_matches": len(scores_by_status["Account Match"]),
            "possible_matches": len(scores_by_status["Possible Match"]),
            # Score distributions by status
            "score_stats_by_status": {
                "Account Match": percentiles(scores_by_status["Account Match"]),
                "Possible Match": percentiles(scores_by_status["Possible Match"]),
                "No Match": percentiles(scores_by_status["No Match"]),
            },
            # Overall score distribution
            "score_stats": percentiles(all_scores),
            # Candidates per row
            "candidates_per_row": percentiles(candidates_per_row_list),
            "avg_candidates_per_row": (total_candidates / len(source_df))
            if len(source_df)
            else 0,
            # Top 10 reasons
            "top_reasons": sorted(
                reasons_counter.items(), key=lambda x: x[1], reverse=True
            )[:10],
            # Other stats
            "blocked_rows": int(blocked_rows),
            "blocked_zero_but_has_alt_domain": int(blocked_zero_but_has_alt),
            "engine_block_quality_score": blocks.get("engine_quality_score")
            if engine_block_info
            else None,
            "engine_guardrail_triggered": (
                engine_strategy_info.get("guardrail_triggered")
                if engine_strategy_info
                else None
            ),
            "engine_block_strategy_details": engine_strategy_info,
            "engine_block_strategy": (
                engine_strategy_info.get("engine_block_strategy")
                if engine_strategy_info
                else None
            ),
            "engine_projected_pairs": (
                engine_strategy_info.get("projected_candidate_pairs")
                if engine_strategy_info
                else None
            ),
            "engine_blocked_rows": int(engine_blocked_rows),
            "engine_residual_rows": int(engine_residual_rows),
            "engine_residual_limit": int(engine_block_info.get("residual_limit"))
            if engine_block_info and engine_block_info.get("residual_limit") is not None
            else None,
            "manual_mode": True,
            "score_saturation_pct": int((saturation_hits / len(source_df)) * 100)
            if len(source_df)
            else 0,
            "b2c_stripped_count": int(getattr(self, "_b2c_stripped_count", 0)),
            "domain_normalized_count": int(
                getattr(self, "_domain_normalized_count", 0)
            ),
            "names_normalized_count": int(getattr(self, "_names_normalized_count", 0)),
            # Diagnostic metrics for data quality
            "source_blank_name_count": int(source_blank_name_count),
            "source_has_email_domain_count": int(source_has_email_domain_count),
            "source_has_website_count": int(source_has_website_count),
            "domain_overlap_upper_bound": int(domain_overlap_upper_bound),
            "rows_processed": int(len(source_df)),
        }

        # Sanity checks and warnings
        sanity_warnings = []

        # Check 1: No "No Match" with score >= 0.95
        high_score_no_match = [s for s in scores_by_status["No Match"] if s >= 0.95]
        if high_score_no_match:
            sanity_warnings.append(
                f"WARNING: {len(high_score_no_match)} 'No Match' rows with score >= 0.95"
            )

        # Check 2: >=95% of Account Matches have domain or high name score
        account_with_strong_signal = sum(
            1
            for reason_list in [
                r["reasons"].split("|")
                for r in result_rows
                if r["match_status"] == "Account Match"
            ]
            for reason in reason_list
            if "M:DOMAIN" in reason or "M:EMAIL_DOMAIN" in reason or "M:NAME" in reason
        )
        if matches_found > 0:
            strong_signal_pct = (account_with_strong_signal / matches_found) * 100
            if strong_signal_pct < 95:
                sanity_warnings.append(
                    f"WARNING: Only {strong_signal_pct:.1f}% of Account Matches have strong signals (DOM/NAME)"
                )

        if sanity_warnings:
            metrics["sanity_warnings"] = sanity_warnings
            for warning in sanity_warnings:
                self.logger.warning(warning)

        # Log comprehensive metrics
        self.logger.info(
            f"[MANUAL MATCH METRICS] n_source={metrics['n_source']}, n_reference={metrics['n_reference']}"
        )
        self.logger.info(
            f"[MANUAL MATCH METRICS] Matches by status: {metrics['matches_by_status']}"
        )
        self.logger.info(
            f"[MANUAL MATCH METRICS] Score P50/P90/P99: {metrics['score_stats']['p50']:.3f}/{metrics['score_stats']['p90']:.3f}/{metrics['score_stats']['p99']:.3f}"
        )
        self.logger.info(
            f"[MANUAL MATCH METRICS] Candidates per row P50/P90/P99: {metrics['candidates_per_row']['p50']:.0f}/{metrics['candidates_per_row']['p90']:.0f}/{metrics['candidates_per_row']['p99']:.0f}"
        )
        if engine_strategy_info:
            self.logger.info(
                "[MANUAL MATCH METRICS] Engine strategy: %s (blocked_rows=%s, residual_rows=%s, residual_limit=%s, projected_pairs=%s)",
                engine_strategy_info.get("engine_block_strategy"),
                engine_blocked_rows,
                engine_residual_rows,
                engine_block_info.get("residual_limit") if engine_block_info else None,
                engine_strategy_info.get("projected_candidate_pairs"),
            )
        self.logger.info(
            f"[MANUAL MATCH METRICS] Top 5 reasons: {metrics['top_reasons'][:5]}"
        )

        # Alert on binary scoring pattern (indicates normalization issues)
        if hasattr(self, "_binary_score_alerts") and self._binary_score_alerts > 0:
            pct = (
                (self._binary_score_alerts / metrics["n_source"]) * 100
                if metrics["n_source"] > 0
                else 0
            )
            self.logger.warning(
                f"[BINARY_SCORE_ALERT] Found {self._binary_score_alerts} cases ({pct:.1f}%) where domain=1.0 but name=0.0. "
                "This may indicate tokenization issues (apostrophes, FKA fragments, etc.)"
            )

        result_df = pd.DataFrame(result_rows)
        return result_df, metrics

    # --- internal helpers ---

    def _prepare_mappings(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        mappings: List[MappingConfig],
    ) -> List[_RuntimeMapping]:
        # CHANGED: Allow MULTIPLE domain mappings for blocking and scoring
        # For blocking: check all domain sources (email_domain, domain, website) to maximize candidate coverage
        # For scoring: use adaptive weights (only weight features where data exists)
        domain_mappings = []
        other_mappings = []

        for mapping in mappings:
            algo_lower = (mapping.algo or "").lower()
            if algo_lower in ("email_domain", "email", "domain", "website"):
                domain_mappings.append(mapping)
            else:
                other_mappings.append(mapping)

        # Add all domain mappings (for blocking and scoring)
        # Adaptive weight normalization will handle scoring correctly
        filtered_mappings = other_mappings + domain_mappings
        if len(domain_mappings) > 1:
            self.logger.info(
                f"[MULTI-DOMAIN] Using {len(domain_mappings)} domain mappings for blocking + adaptive scoring"
            )

        # Now process the filtered mappings
        prepared: List[_RuntimeMapping] = []
        for mapping in filtered_mappings:
            weight = _normalize_weight(mapping.weight)
            if weight <= 0:
                continue
            if mapping.source not in source_df.columns:
                self.logger.warning(
                    "Mapping source column '%s' not found", mapping.source
                )
                source_series = pd.Series([""] * len(source_df))
            else:
                source_series = source_df[mapping.source]
            if mapping.reference not in reference_df.columns:
                self.logger.warning(
                    "Mapping reference column '%s' not found", mapping.reference
                )
                reference_series = pd.Series([""] * len(reference_df))
            else:
                reference_series = reference_df[mapping.reference]

            opts = mapping.opts or MappingOptions()
            score_func, src_processed, ref_processed = self._build_mapping_series(
                mapping.algo, source_series, reference_series, opts
            )
            prepared.append(
                _RuntimeMapping(
                    config=MappingConfig(
                        source=mapping.source,
                        reference=mapping.reference,
                        algo=mapping.algo.lower(),
                        weight=weight,
                        opts=opts,
                    ),
                    source_series=src_processed,
                    reference_series=ref_processed,
                    score_func=score_func,
                    raw_source=source_series.fillna(""),
                    raw_reference=reference_series.fillna(""),
                )
            )
        if not prepared:
            raise ManualMatchServiceError(
                "All mappings were filtered out due to missing columns or zero weights"
            )
        return prepared

    def _build_mapping_series(
        self,
        algo: str,
        source_series: pd.Series,
        reference_series: pd.Series,
        opts: MappingOptions,
    ) -> Tuple[Callable[[str, str], float], pd.Series, pd.Series]:
        algo = (algo or "").lower()
        strip_b2c = getattr(self, "strip_b2c", True)

        if algo in ("domain", "website"):
            # Extract domain and count B2C strippings
            results = source_series.apply(lambda x: _extract_domain(x, strip_b2c))
            src_domains = results.apply(
                lambda x: x[0] if isinstance(x, tuple) else None
            )
            src_b2c_count = sum(1 for x in results if isinstance(x, tuple) and x[1])
            src_normalized_count = sum(1 for x in src_domains if x)

            results = reference_series.apply(lambda x: _extract_domain(x, strip_b2c))
            ref_domains = results.apply(
                lambda x: x[0] if isinstance(x, tuple) else None
            )
            ref_b2c_count = sum(1 for x in results if isinstance(x, tuple) and x[1])
            ref_normalized_count = sum(1 for x in ref_domains if x)

            # Store B2C count in instance for metrics
            if not hasattr(self, "_b2c_stripped_count"):
                self._b2c_stripped_count = 0
            self._b2c_stripped_count += src_b2c_count + ref_b2c_count

            # Store domain normalized count
            if not hasattr(self, "_domain_normalized_count"):
                self._domain_normalized_count = 0
            self._domain_normalized_count += src_normalized_count + ref_normalized_count
            return _exact_match_score, src_domains.fillna(""), ref_domains.fillna("")

        if algo in ("email_domain", "email"):
            # Extract email domain and count B2C strippings
            results = source_series.apply(lambda x: _extract_email_domain(x, strip_b2c))
            src_domains = results.apply(
                lambda x: x[0] if isinstance(x, tuple) else None
            )
            src_b2c_count = sum(1 for x in results if isinstance(x, tuple) and x[1])
            src_normalized_count = sum(1 for x in src_domains if x)

            results = reference_series.apply(
                lambda x: _extract_email_domain(x, strip_b2c)
            )
            ref_domains = results.apply(
                lambda x: x[0] if isinstance(x, tuple) else None
            )
            ref_b2c_count = sum(1 for x in results if isinstance(x, tuple) and x[1])
            ref_normalized_count = sum(1 for x in ref_domains if x)

            # Store B2C count in instance for metrics
            if not hasattr(self, "_b2c_stripped_count"):
                self._b2c_stripped_count = 0
            self._b2c_stripped_count += src_b2c_count + ref_b2c_count

            # Store domain normalized count
            if not hasattr(self, "_domain_normalized_count"):
                self._domain_normalized_count = 0
            self._domain_normalized_count += src_normalized_count + ref_normalized_count
            return _exact_match_score, src_domains.fillna(""), ref_domains.fillna("")

        # For name/company matching, use enhanced company name normalization
        if algo in ("token_set", "name", "fuzzy"):
            raise ManualMatchServiceError(
                f"Local fuzzy matching ('{algo}') is disabled; use MatchGateway."
            )

        # For other algos, use standard text normalization
        normalized_src = source_series.fillna("").map(
            lambda v: _normalize_text(v, opts)
        )
        normalized_ref = reference_series.fillna("").map(
            lambda v: _normalize_text(v, opts)
        )

        if algo == "exact":
            return _exact_match_score, normalized_src, normalized_ref
        if algo in ("jaro", "jaro_winkler", "jarowinkler"):
            raise ManualMatchServiceError(
                f"Local fuzzy matching ('{algo}') is disabled; use MatchGateway."
            )
        if algo == "contains":
            return _contains_score, normalized_src, normalized_ref

        self.logger.warning("Unknown algorithm '%s', defaulting to exact", algo)
        return _exact_match_score, normalized_src, normalized_ref

    def _find_domain_mapping(
        self, mappings: List[_RuntimeMapping]
    ) -> Optional[_RuntimeMapping]:
        """Return first domain mapping for backward compatibility (used for metrics)"""
        for m in mappings:
            if m.config.algo in ("domain", "email_domain"):
                return m
        return None

    def _find_all_domain_mappings(
        self, mappings: List[_RuntimeMapping]
    ) -> List[_RuntimeMapping]:
        """Return ALL domain mappings for blocking (supports multiple domain columns)"""
        return [m for m in mappings if m.config.algo in ("domain", "email_domain")]

    def _requires_domain(self, mappings: List[_RuntimeMapping]) -> bool:
        return any(m.config.algo in ("domain", "email_domain") for m in mappings)

    def _find_geo_mapping(
        self, mappings: List[_RuntimeMapping], token: str
    ) -> Optional[_RuntimeMapping]:
        token = token.lower()
        for m in mappings:
            if token in m.config.source.lower() and token in m.config.reference.lower():
                return m
        for m in mappings:
            if token in m.config.source.lower():
                return m
        return None

    def _resolve_anchor(
        self, mappings: List[_RuntimeMapping], anchor_kind: Optional[str]
    ) -> Optional[_RuntimeMapping]:
        if not anchor_kind:
            return None
        key = str(anchor_kind).strip().lower()
        if key == "country":
            return self._find_geo_mapping(mappings, "country")
        if key == "state":
            return self._find_geo_mapping(mappings, "state")
        return None

    def _build_domain_index(
        self, reference_df: pd.DataFrame, mapping: _RuntimeMapping
    ) -> Dict[str, List[int]]:
        """Build domain index from a single domain mapping (legacy/backward compat)"""
        index: Dict[str, List[int]] = {}
        for idx, value in mapping.reference_series.items():
            if not value:
                continue
            index.setdefault(value, []).append(idx)
        return index

    def _build_multi_domain_index(
        self, reference_df: pd.DataFrame, mappings: List[_RuntimeMapping]
    ) -> Dict[str, List[int]]:
        """Build unified domain index from ALL domain mappings (supports multiple domain columns)"""
        index: Dict[str, List[int]] = {}
        for mapping in mappings:
            for idx, value in mapping.reference_series.items():
                if not value:
                    continue
                # Add this reference row to the index for this domain value
                if value not in index:
                    index[value] = []
                if (
                    idx not in index[value]
                ):  # Avoid duplicates if same domain in multiple columns
                    index[value].append(idx)
        return index

    def _prepare_engine_block_index(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        strategy: Dict[str, Any],
    ) -> Dict[str, Any]:
        """Build lookup structures for engine-selected blocking strategy."""
        src_col = strategy.get("source_block_col")
        ref_col = strategy.get("ref_block_col") or src_col
        preproc = strategy.get("preproc", "alnum")
        key_len = int(strategy.get("block_key_len", 3) or 3)
        if not src_col or src_col not in source_df.columns:
            raise ManualMatchServiceError(
                f"Auto-blocking selected column '{src_col}' which is not present in source data."
            )
        if ref_col and ref_col not in reference_df.columns:
            if src_col in reference_df.columns:
                ref_col = src_col
            else:
                raise ManualMatchServiceError(
                    f"Auto-blocking selected column '{ref_col}' which is not present in reference data."
                )
        ruleset_name = strategy.get("ruleset", "default")
        from fmatch.core.engine import build_key

        src_series = source_df[src_col]
        src_keys, _ = build_key(src_series, preproc, key_len, ruleset_name)
        src_keys = src_keys.fillna("").astype(str)
        ref_series = (
            reference_df[ref_col]
            if ref_col in reference_df.columns
            else reference_df[src_col]
        )
        ref_keys, _ = build_key(ref_series, preproc, key_len, ruleset_name)
        ref_keys = ref_keys.fillna("").astype(str)
        label_to_pos = {label: pos for pos, label in enumerate(reference_df.index)}
        ref_index: Dict[str, List[int]] = {}
        for label, key in ref_keys.items():
            if not key:
                continue
            pos = label_to_pos.get(label)
            if pos is None:
                continue
            ref_index.setdefault(key, []).append(pos)
        residual_info = strategy.get("residual_handling") or {}
        residual_limit = None
        if residual_info.get("mode") == "cap_candidates_per_row":
            try:
                residual_limit = int(residual_info.get("limit", 0) or 0)
            except (TypeError, ValueError):
                residual_limit = None
            if residual_limit is not None and residual_limit <= 0:
                residual_limit = None
        return {
            "strategy": strategy,
            "source_keys": src_keys,
            "ref_index": ref_index,
            "residual_limit": residual_limit,
            "reference_positions": list(range(len(reference_df))),
        }

    def _build_state_index(
        self, reference_df: pd.DataFrame, mapping: _RuntimeMapping
    ) -> Dict[str, List[int]]:
        index: Dict[str, List[int]] = {}
        for idx, value in mapping.reference_series.items():
            if not value:
                continue
            index.setdefault(value, []).append(idx)
        return index

    def _candidate_indices(
        self,
        reference_df: pd.DataFrame,
        domain_index: Optional[Dict[str, List[int]]],
        state_index: Optional[Dict[str, List[int]]],
        source_row: pd.Series,
        all_domain_mappings: List[_RuntimeMapping],
        state_mapping: Optional[_RuntimeMapping],
        engine_block_info: Optional[Dict[str, Any]] = None,
    ) -> Tuple[List[int], Dict[str, bool]]:
        """Return candidate reference indices for a source row."""
        engine_meta = {
            "engine_used": bool(engine_block_info),
            "engine_residual": False,
            "engine_had_key": False,
            "domain_candidates": False,
            "domain_intersection": False,
            "full_scan": False,
        }
        candidates: List[int] = []

        domain_candidate_set: Optional[set] = None
        domain_candidates: List[int] = []
        if domain_index is not None and all_domain_mappings:
            candidate_set = set()
            for domain_mapping in all_domain_mappings:
                domain_value = domain_mapping.source_series.get(source_row.name)
                if domain_value and domain_value in domain_index:
                    candidate_set.update(domain_index[domain_value])
            if candidate_set:
                domain_candidate_set = candidate_set
                domain_candidates = sorted(candidate_set)
                engine_meta["domain_candidates"] = True

        if engine_block_info:
            source_keys = engine_block_info.get("source_keys")
            ref_index = engine_block_info.get("ref_index", {})
            key = ""
            if source_keys is not None:
                key = str(source_keys.get(source_row.name, ""))
            if key:
                key_candidates = ref_index.get(key)
                if key_candidates:
                    candidates = list(key_candidates)
                    engine_meta["engine_had_key"] = True

        if domain_candidate_set:
            if candidates:
                filtered = [idx for idx in candidates if idx in domain_candidate_set]
                if filtered:
                    candidates = filtered
                    engine_meta["domain_intersection"] = True
                else:
                    candidates = domain_candidates
            else:
                candidates = domain_candidates

        if engine_block_info and not candidates:
            residual_limit = engine_block_info.get("residual_limit")
            if residual_limit:
                candidates = self._engine_residual_candidates(
                    engine_block_info,
                    source_row.name,
                    residual_limit,
                )
                if candidates:
                    engine_meta["engine_residual"] = True
            if not candidates:
                fallback_limit = min(
                    250, len(engine_block_info.get("reference_positions", []))
                )
                if fallback_limit:
                    candidates = self._engine_residual_candidates(
                        engine_block_info,
                        source_row.name,
                        fallback_limit,
                    )
                    if candidates:
                        engine_meta["engine_residual"] = True

        if not candidates:
            candidates = (
                domain_candidates
                if domain_candidates
                else list(range(len(reference_df)))
            )
            if not domain_candidates:
                engine_meta["full_scan"] = True

        if state_index is not None and state_mapping is not None and candidates:
            state_value = state_mapping.source_series.get(source_row.name)
            if state_value:
                candidates = [
                    idx
                    for idx in candidates
                    if state_mapping.reference_series.get(idx) == state_value
                ]

        return candidates, engine_meta

    def _engine_residual_candidates(
        self,
        engine_block_info: Dict[str, Any],
        source_row_name: Any,
        limit: int,
    ) -> List[int]:
        positions = engine_block_info.get("reference_positions") or []
        if not positions:
            return []
        limit = min(max(int(limit), 0), len(positions))
        if limit == 0:
            return []
        seed = hash((source_row_name, limit, len(positions))) & 0xFFFFFFFF
        start = seed % len(positions)
        result: List[int] = []
        for i in range(limit):
            pos = (start + i) % len(positions)
            result.append(positions[pos])
        return result

    def _process_rows_chunk(
        self,
        chunk_indices: List[Any],
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        runtime_mappings: List[_RuntimeMapping],
        domain_mapping: Optional[_RuntimeMapping],
        all_domain_mappings: List[_RuntimeMapping],
        domain_index: Optional[Dict[str, List[int]]],
        state_mapping: Optional[_RuntimeMapping],
        state_index: Optional[Dict[str, List[int]]],
        anchor_mapping: Optional[_RuntimeMapping],
        anchor_kind: Optional[str],
        engine_block_info: Optional[Dict[str, Any]],
        best_only: bool,
        threshold: float,
        source_name_col: Optional[str],
        reference_name_col: Optional[str],
        source_id_col: Optional[str],
        reference_id_col: Optional[str],
        source_id_lookup: Optional[Dict[Any, str]] = None,
    ) -> Dict[str, Any]:
        result_rows: List[Dict[str, Any]] = []
        total_candidates = 0
        blocked_rows = 0
        engine_blocked_rows = 0
        engine_residual_rows = 0
        blocked_zero_but_has_alt = 0
        saturation_hits = 0
        all_scores: List[float] = []
        scores_by_status = {"Account Match": [], "Possible Match": [], "No Match": []}
        reasons_counter: Dict[str, int] = {}
        candidates_per_row_list: List[int] = []
        binary_alerts = 0
        binary_alert_samples: List[Tuple[str, str]] = []

        trace_enabled = getattr(self, "_trace_match_perf", False)
        trace_stats: Optional[Dict[str, Any]] = None
        trace_chunk_start = 0.0
        if trace_enabled:
            trace_chunk_start = time.perf_counter()
            trace_stats = {
                "rows": 0,
                "domain_rows": 0,
                "domain_intersection_rows": 0,
                "engine_key_rows": 0,
                "engine_residual_rows": 0,
                "full_scan_rows": 0,
                "total_candidates": 0,
                "max_candidates": 0,
            }

        if not chunk_indices:
            return {
                "rows": result_rows,
                "total_candidates": 0,
                "blocked_rows": 0,
                "engine_blocked_rows": 0,
                "engine_residual_rows": 0,
                "blocked_zero_but_has_alt": 0,
                "saturation_hits": 0,
                "all_scores": [],
                "scores_by_status": scores_by_status,
                "reasons_counter": {},
                "candidates_per_row": [],
                "binary_alerts": 0,
                "binary_alert_samples": [],
            }

        source_chunk = source_df.loc[chunk_indices]

        for row_idx, source_row in source_chunk.iterrows():
            candidates, candidate_meta = self._candidate_indices(
                reference_df,
                domain_index,
                state_index,
                source_row,
                all_domain_mappings,
                state_mapping,
                engine_block_info=engine_block_info,
            )

            if (
                domain_index is not None
                and len(candidates) > 0
                and len(all_domain_mappings) > 1
            ):
                first_domain_value = all_domain_mappings[0].source_series.get(
                    source_row.name
                )
                first_domain_has_candidates = (
                    first_domain_value and first_domain_value in domain_index
                )
                if not first_domain_has_candidates:
                    blocked_zero_but_has_alt += 1

            if candidate_meta.get("engine_used"):
                if candidate_meta.get("engine_had_key"):
                    engine_blocked_rows += 1
                if candidate_meta.get("engine_residual"):
                    engine_residual_rows += 1
            elif domain_index is not None:
                blocked_rows += 1

            if trace_stats is not None:
                trace_stats["rows"] += 1
                candidate_count = len(candidates)
                trace_stats["total_candidates"] += candidate_count
                if candidate_count > trace_stats["max_candidates"]:
                    trace_stats["max_candidates"] = candidate_count
                if candidate_meta.get("domain_candidates"):
                    trace_stats["domain_rows"] += 1
                if candidate_meta.get("domain_intersection"):
                    trace_stats["domain_intersection_rows"] += 1
                if candidate_meta.get("engine_had_key"):
                    trace_stats["engine_key_rows"] += 1
                if candidate_meta.get("engine_residual"):
                    trace_stats["engine_residual_rows"] += 1
                if candidate_meta.get("full_scan"):
                    trace_stats["full_scan_rows"] += 1

            total_candidates += len(candidates)
            candidates_per_row_list.append(len(candidates))
            candidate_results: List[_CandidateResult] = []

            for ref_idx in candidates:
                candidate = self._score_candidate(
                    runtime_mappings,
                    source_row,
                    reference_df.iloc[ref_idx],
                    threshold,
                    domain_mapping,
                    anchor_mapping,
                    anchor_kind,
                )
                candidate.reference_index = ref_idx
                if candidate.binary_alert:
                    binary_alerts += 1
                    if candidate.binary_alert_sample and len(binary_alert_samples) < 5:
                        binary_alert_samples.append(candidate.binary_alert_sample)
                candidate_results.append(candidate)

            if not candidate_results:
                candidate_results = []

            candidate_results.sort(key=lambda c: c.final_score, reverse=True)
            has_blocking = (
                bool(domain_index) or bool(state_index) or engine_block_info is not None
            )
            evaluation_limit = 100 if has_blocking else 250
            candidate_results = candidate_results[:evaluation_limit]

            best_candidates = self._select_candidates(
                candidate_results, best_only, threshold
            )
            if any(c.final_score >= threshold for c in candidate_results):
                saturation_hits += 1

            if not best_candidates:
                best_candidates = [self._no_match_result(threshold)]

            for rank, candidate in enumerate(best_candidates):
                all_scores.append(candidate.final_score)
                scores_by_status[candidate.status].append(candidate.final_score)
                for reason in candidate.reasons:
                    reasons_counter[reason] = reasons_counter.get(reason, 0) + 1
                ref_row = (
                    reference_df.iloc[candidate.reference_index]
                    if candidate.reference_index is not None
                    else None
                )
                result_rows.append(
                    self._build_output_row(
                        row_idx,
                        rank,
                        source_row,
                        ref_row,
                        candidate,
                        source_name_col,
                        reference_name_col,
                        source_id_col,
                        reference_id_col,
                        domain_mapping,
                        source_id_lookup=source_id_lookup,
                    )
                )

        result_payload = {
            "rows": result_rows,
            "total_candidates": total_candidates,
            "blocked_rows": blocked_rows,
            "engine_blocked_rows": engine_blocked_rows,
            "engine_residual_rows": engine_residual_rows,
            "blocked_zero_but_has_alt": blocked_zero_but_has_alt,
            "saturation_hits": saturation_hits,
            "all_scores": all_scores,
            "scores_by_status": scores_by_status,
            "reasons_counter": reasons_counter,
            "candidates_per_row": candidates_per_row_list,
            "binary_alerts": binary_alerts,
            "binary_alert_samples": binary_alert_samples,
        }

        if trace_stats is not None and trace_stats["rows"]:
            elapsed = time.perf_counter() - trace_chunk_start
            avg_candidates = trace_stats["total_candidates"] / max(
                trace_stats["rows"], 1
            )
            domain_ratio = trace_stats["domain_rows"] / trace_stats["rows"]
            engine_ratio = trace_stats["engine_key_rows"] / trace_stats["rows"]
            residual_ratio = trace_stats["engine_residual_rows"] / trace_stats["rows"]
            full_scan_ratio = trace_stats["full_scan_rows"] / trace_stats["rows"]
            chunk_start_idx = chunk_indices[0] if chunk_indices else None
            chunk_end_idx = chunk_indices[-1] if chunk_indices else None
            self.logger.info(
                "[TRACE] match_chunk rows=%d elapsed=%.2fs avg_candidates=%.1f max_candidates=%d "
                "domain_rows=%d (%.1f%%) domain_intersections=%d engine_key_rows=%d (%.1f%%) "
                "engine_residual_rows=%d (%.1f%%) full_scan_rows=%d (%.1f%%) chunk_range=%s-%s",
                trace_stats["rows"],
                elapsed,
                avg_candidates,
                trace_stats["max_candidates"],
                trace_stats["domain_rows"],
                domain_ratio * 100.0,
                trace_stats["domain_intersection_rows"],
                trace_stats["engine_key_rows"],
                engine_ratio * 100.0,
                trace_stats["engine_residual_rows"],
                residual_ratio * 100.0,
                trace_stats["full_scan_rows"],
                full_scan_ratio * 100.0,
                chunk_start_idx,
                chunk_end_idx,
            )

        return result_payload

    def _score_candidate(
        self,
        runtime_mappings: List[_RuntimeMapping],
        source_row: pd.Series,
        reference_row: pd.Series,
        threshold: float,
        domain_mapping: Optional[_RuntimeMapping],
        anchor_mapping: Optional[_RuntimeMapping],
        anchor_kind: Optional[str],
    ) -> _CandidateResult:
        total_weighted_score = 0.0
        active_weight_sum = 0.0
        domain_score = 0.0
        name_score = 0.0
        reasons: List[
            str
        ] = []  # Reason codes: M:DOMAIN, M:EMAIL_DOMAIN, M:NAME90, M:DOMAIN_BOOST, M:DOMAIN_STRONG, M:STRONG_SIGNAL, M:B2C_FALLBACK, etc.
        mapping_details: List[Dict[str, Any]] = []
        binary_alert = False
        binary_alert_sample: Optional[Tuple[str, str]] = None
        matched_family_id: Optional[str] = (
            None  # Track family ID when family match detected
        )
        source_name_for_rules = ""
        reference_name_for_rules = ""

        anchor_matched = True
        if anchor_mapping is not None:
            src_anchor = anchor_mapping.source_series.get(source_row.name)
            ref_anchor = anchor_mapping.reference_series.get(reference_row.name)
            anchor_matched = bool(src_anchor) and src_anchor == ref_anchor

        for mapping in runtime_mappings:
            src_val = mapping.source_series.get(source_row.name, "")
            ref_val = mapping.reference_series.get(reference_row.name, "")
            raw_src = mapping.raw_source.get(source_row.name, "")
            raw_ref = mapping.raw_reference.get(reference_row.name, "")

            # Compute initial score
            if src_val and ref_val:
                score = mapping.score_func(src_val, ref_val)
            else:
                score = 0.0

            # OVERRIDE with domain family relationship if applicable
            if (
                (
                    mapping is domain_mapping
                    or mapping.config.algo in ("domain", "email_domain")
                )
                and src_val
                and ref_val
                and self.domain_registry
            ):
                try:
                    relation, family_id = self.domain_registry.relation(
                        src_val, ref_val
                    )
                    if relation == DomainRelation.EXACT:
                        score = 1.0
                        if family_id:
                            matched_family_id = family_id
                    elif relation == DomainRelation.FAMILY:
                        score = 0.95  # High confidence but slightly below exact match
                        if family_id:
                            matched_family_id = family_id
                    # relation == DomainRelation.NONE keeps score from score_func (usually 0.0)
                except Exception as e:
                    # Fall back to score_func if domain_registry fails
                    self.logger.debug(
                        f"Domain family lookup failed: {e}, using exact match fallback"
                    )
                    # score is already set from score_func above

            # NOW accumulate weighted score with the corrected value
            if src_val and ref_val:
                total_weighted_score += mapping.config.weight * score
                active_weight_sum += mapping.config.weight

            if mapping is domain_mapping or mapping.config.algo in (
                "domain",
                "email_domain",
            ):
                domain_score = max(domain_score, score)
                if score >= 1.0:
                    if mapping.config.algo == "email_domain":
                        reasons.append("M:EMAIL_DOMAIN")
                    else:
                        reasons.append("M:DOMAIN")
                elif score >= 0.95:
                    # Family match - add FAM reason code
                    reasons.append("FAM")
            elif mapping.config.algo in ("token_set", "jaro", "name", "fuzzy"):
                name_score = max(name_score, score)
                chip = _compute_reason_band(score)
                if chip:
                    reasons.append(chip)
                if not source_name_for_rules:
                    source_name_for_rules = mapping.raw_source.get(
                        source_row.name, ""
                    ) or mapping.source_series.get(source_row.name, "")
                if not reference_name_for_rules:
                    reference_name_for_rules = mapping.raw_reference.get(
                        reference_row.name, ""
                    ) or mapping.reference_series.get(reference_row.name, "")
            elif mapping.config.algo == "exact" and score >= 1.0:
                reasons.append("M:EXACT")
            elif mapping.config.algo == "contains" and score >= 1.0:
                reasons.append("M:CONTAINS")

            mapping_details.append(
                {
                    "source": mapping.config.source,
                    "reference": mapping.config.reference,
                    "algo": mapping.config.algo,
                    "weight": mapping.config.weight,
                    "score": round(float(score), 4),
                    "source_value": raw_src,
                    "reference_value": raw_ref,
                }
            )

        # Normalize score by active weights only (ignore missing features)
        if active_weight_sum > 0:
            total_score = total_weighted_score / active_weight_sum
        else:
            total_score = 0.0

        total_score = max(0.0, min(total_score, 1.0))

        # Enhanced policy: Accept strong single signals
        # Policy 1: Perfect domain + decent name (>=0.55) = Account Match
        # Policy 2: Perfect name alone (>=0.90) = Account Match
        # Policy 3: Use >= not > for threshold (exact threshold matches count)
        # Policy 4: Domain mismatch guardrail for high name scores
        status = "No Match"

        # Check for domain mismatch when relying on name alone
        # Consider both exact matches AND family relationships
        has_domain_mismatch = False
        source_company_name = _normalize_company_name(source_name_for_rules)
        reference_company_name = _normalize_company_name(reference_name_for_rules)

        if source_company_name and _is_junk_company_name(source_company_name):
            explanation = {
                "threshold": threshold,
                "total_score": 0.0,
                "domain_score": 0.0,
                "name_score": round(name_score, 4),
                "reason": "JUNK_SOURCE_NAME",
                "source_value": source_company_name,
            }
            return _CandidateResult(
                reference_index=None,
                final_score=0.0,
                domain_score=0.0,
                name_score=name_score,
                status="No Match",
                reasons=["JUNK_SOURCE_NAME"],
                explanation=explanation,
                binary_alert=False,
                binary_alert_sample=None,
                matched_family_id=None,
            )

        if domain_mapping is not None:
            src_domain_val = domain_mapping.source_series.get(source_row.name, "")
            ref_domain_val = domain_mapping.reference_series.get(reference_row.name, "")
            # Both have non-empty domains
            if src_domain_val and ref_domain_val:
                # Check if they're exact match or same family
                if src_domain_val != ref_domain_val:
                    # Not an exact match - check family relationship
                    if self.domain_registry:
                        try:
                            relation, _ = self.domain_registry.relation(
                                src_domain_val, ref_domain_val
                            )
                            # Mismatch only if they're not in the same family
                            if relation == DomainRelation.NONE:
                                has_domain_mismatch = True
                        except Exception:
                            # If family check fails, treat as mismatch
                            has_domain_mismatch = True
                    else:
                        # No registry available - treat non-exact as mismatch
                        has_domain_mismatch = True

        # Strong signal policies with domain boost
        # Account Match requires strong name match to avoid false merges (shared corporate domains)
        # Possible Match allows weaker name for user review

        # EXACT domain (1.0) policies
        perfect_domain_decent_name = (
            domain_score >= 1.0 and name_score >= 0.55
        )  # Conservative: avoid false merges
        perfect_domain_weak_name = (
            domain_score >= 1.0 and name_score >= 0.30
        )  # New: weak name → Possible Match for review

        # FAMILY domain (0.95) policies - slightly higher name requirement
        family_domain_decent_name = (
            domain_score >= 0.95 and domain_score < 1.0 and name_score >= 0.60
        )  # Account Match
        family_domain_weak_name = (
            domain_score >= 0.95 and domain_score < 1.0 and name_score >= 0.35
        )  # Possible Match

        # Alert on binary scoring pattern: domain=1.0 but name=0.0 (potential normalization bug)
        if domain_score >= 1.0 and name_score == 0.0:
            binary_alert = True
            src_name_val = ""
            ref_name_val = ""
            for mapping in runtime_mappings:
                if mapping.config.algo in ("token_set", "name", "fuzzy"):
                    src_name_val = mapping.source_series.get(source_row.name, "")
                    ref_name_val = mapping.reference_series.get(reference_row.name, "")
                    break
            binary_alert_sample = (str(src_name_val)[:100], str(ref_name_val)[:100])

        # Apply domain mismatch guardrail: if domains mismatch, require higher name score
        if has_domain_mismatch:
            perfect_name = name_score >= 0.94  # Stricter threshold to avoid homonyms
        else:
            perfect_name = name_score >= 0.90

        short_source_token = "".join(ch for ch in source_company_name if ch.isalnum())
        if domain_score == 0.0 and short_source_token and len(short_source_token) <= 3:
            explanation = {
                "threshold": threshold,
                "total_score": 0.0,
                "domain_score": 0.0,
                "name_score": round(name_score, 4),
                "reason": "SHORT_ACRONYM_NO_DOMAIN",
                "source_value": source_company_name,
            }
            return _CandidateResult(
                reference_index=None,
                final_score=0.0,
                domain_score=0.0,
                name_score=name_score,
                status="No Match",
                reasons=["SHORT_ACRONYM_NO_DOMAIN"],
                explanation=explanation,
                binary_alert=False,
                binary_alert_sample=None,
                matched_family_id=None,
            )

        name_domain_conflict = False
        if (
            domain_mapping is not None
            and domain_score >= 1.0
            and source_company_name
            and reference_company_name
            and name_score < 0.45
            and not _has_name_token_overlap(source_company_name, reference_company_name)
        ):
            name_domain_conflict = True

        asymmetric_tokens = False
        if (
            source_company_name
            and reference_company_name
            and name_score >= 0.90
            and domain_score == 0.0
        ):
            source_tokens = _tokenize_company_name(source_company_name)
            reference_tokens = _tokenize_company_name(reference_company_name)
            if source_tokens and reference_tokens:
                source_set = set(source_tokens)
                reference_set = set(reference_tokens)
                if source_set and reference_set:
                    if (
                        source_set.issuperset(reference_set)
                        and source_set != reference_set
                    ):
                        asymmetric_tokens = True

        if asymmetric_tokens:
            explanation = {
                "threshold": threshold,
                "total_score": 0.0,
                "domain_score": 0.0,
                "name_score": round(name_score, 4),
                "reason": "NAME_TOKEN_SUBSET",
                "source_value": source_company_name,
                "reference_value": reference_company_name,
            }
            return _CandidateResult(
                reference_index=None,
                final_score=0.0,
                domain_score=0.0,
                name_score=name_score,
                status="No Match",
                reasons=["NAME_TOKEN_SUBSET"],
                explanation=explanation,
                binary_alert=False,
                binary_alert_sample=None,
                matched_family_id=None,
            )

        # Add B2C_FALLBACK reason when domain is missing but name carries match
        b2c_fallback = (
            domain_mapping is not None and domain_score == 0.0 and name_score >= 0.90
        )

        if anchor_matched:
            if domain_mapping is not None:
                # Domain is mapped - require domain signal or strong name
                if (
                    domain_score >= 1.0
                    and total_score >= threshold
                    and name_score >= 0.55
                ) or perfect_domain_decent_name:
                    status = "Account Match"
                    if perfect_domain_decent_name and total_score < threshold:
                        reasons.append("M:DOMAIN_BOOST")
                    if b2c_fallback:
                        reasons.append("M:B2C_FALLBACK")
                elif family_domain_decent_name:
                    # Family domain with decent name = Account Match
                    # This is the KEY fix - allow family matches to reach Account Match status
                    status = "Account Match"
                    reasons.append("M:FAMILY_BOOST")
                elif perfect_domain_weak_name:
                    # Perfect domain with very weak name match = Possible Match
                    # Catches cases like org structure differences or abbreviations
                    status = "Possible Match"
                    reasons.append("M:DOMAIN_STRONG")
                elif family_domain_weak_name:
                    # Family domain with weak name = Possible Match
                    status = "Possible Match"
                    reasons.append("M:FAMILY_POSSIBLE")
                elif total_score >= max(threshold - 0.10, 0.0):
                    status = "Possible Match"
                    if b2c_fallback:
                        reasons.append("M:B2C_FALLBACK")
                elif perfect_name and not has_domain_mismatch:
                    # Perfect name can still trigger Possible even if below threshold
                    status = "Possible Match"
                    reasons.append("M:STRONG_NAME")
                    if b2c_fallback:
                        reasons.append("M:B2C_FALLBACK")
            else:
                # No domain mapping - rely on name and other features
                if total_score >= threshold or perfect_name:
                    status = "Account Match"
                    if perfect_name and total_score < threshold:
                        reasons.append("M:STRONG_SIGNAL")
                elif total_score >= max(threshold - 0.10, 0.0):
                    status = "Possible Match"

        # POSSIBLE by strong name when domain is absent/inconclusive
        if status == "No Match":
            name_only_strong = name_score >= 0.85
            domain_inconclusive = (domain_mapping is None) or (domain_score == 0.0)
            if (
                name_only_strong
                and domain_inconclusive
                and total_score >= max(threshold - 0.10, 0.0)
            ):
                status = "Possible Match"
                reasons.append("M:NAME_POSSIBLE")

        if status == "Account Match" and name_domain_conflict:
            status = "Possible Match"
            reasons.append("DOMAIN_NAME_CONFLICT")
        elif name_domain_conflict:
            reasons.append("DOMAIN_NAME_CONFLICT")

        if anchor_mapping is not None and anchor_matched and anchor_kind:
            reasons.append(f"ANCHOR:{str(anchor_kind).upper()}")

        reasons = self._dedupe_reasons(reasons)

        explanation = {
            "threshold": threshold,
            "total_score": round(total_score, 4),
            "domain_score": round(domain_score, 4),
            "name_score": round(name_score, 4),
            "mappings": mapping_details,
        }

        return _CandidateResult(
            reference_index=getattr(reference_row, "name", None),
            final_score=total_score,
            domain_score=domain_score,
            name_score=name_score,
            status=status,
            reasons=reasons,
            explanation=explanation,
            binary_alert=binary_alert,
            binary_alert_sample=binary_alert_sample,
            matched_family_id=matched_family_id,
        )

    def _select_candidates(
        self,
        candidates: List[_CandidateResult],
        best_only: bool,
        threshold: float,
    ) -> List[_CandidateResult]:
        if not candidates:
            return []
        candidates.sort(
            key=lambda c: (c.final_score, c.domain_score, c.name_score), reverse=True
        )
        if best_only:
            return [candidates[0]]
        usable = [c for c in candidates if c.final_score >= max(threshold - 0.10, 0.0)]
        return usable or [candidates[0]]

    def _no_match_result(self, threshold: float) -> _CandidateResult:
        return _CandidateResult(
            reference_index=None,
            final_score=0.0,
            domain_score=0.0,
            name_score=0.0,
            status="No Match",
            reasons=[],
            explanation={"threshold": threshold, "total_score": 0.0, "mappings": []},
        )

    def _build_output_row(
        self,
        source_index: int,
        rank: int,
        source_row: pd.Series,
        reference_row: Optional[pd.Series],
        candidate: _CandidateResult,
        source_name_col: Optional[str],
        reference_name_col: Optional[str],
        source_id_col: Optional[str],
        reference_id_col: Optional[str],
        domain_mapping: Optional[_RuntimeMapping],
        source_id_lookup: Optional[Dict[Any, str]] = None,
    ) -> Dict[str, Any]:
        source_name = (
            str(source_row.get(source_name_col, "")) if source_name_col else ""
        )
        reference_name = ""
        reference_id = ""
        matched_domain = ""
        if reference_row is not None:
            if reference_name_col:
                reference_name = str(reference_row.get(reference_name_col, ""))
            if reference_id_col:
                reference_id = str(reference_row.get(reference_id_col, ""))
            if domain_mapping is not None:
                matched_domain = str(
                    domain_mapping.reference_series.get(reference_row.name, "") or ""
                )

        explanation_json = json.dumps(candidate.explanation)[:10000]

        # Add B2C explanation if B2C_FALLBACK reason is present
        b2c_explanation = ""
        if "M:B2C_FALLBACK" in candidate.reasons:
            b2c_explanation = "Personal email domain (gmail.com, yahoo.com, etc.) was filtered to improve match accuracy"

        try:
            source_row_index = int(source_index)
        except (TypeError, ValueError):
            source_row_index = source_index

        reference_row_index = candidate.reference_index
        if reference_row_index is not None:
            try:
                reference_row_index = int(reference_row_index)
            except (TypeError, ValueError):
                pass

        source_identifier = ""
        if source_id_lookup is not None:
            source_identifier = source_id_lookup.get(source_index, "")
        if not source_identifier and source_id_col:
            raw_source_id = source_row.get(source_id_col)
            if raw_source_id is not None and not pd.isna(raw_source_id):
                source_identifier = str(raw_source_id).strip()
        if not source_identifier:
            source_identifier = str(source_index + 1)

        result = {
            "source_id": source_identifier,
            "source_name": source_name,
            "candidate_rank": rank + 1,
            "reference_id": reference_id,
            "reference_name": reference_name,
            "match_score": int(round(candidate.final_score * 100)),
            "match_status": candidate.status,
            "reasons": "|".join(candidate.reasons),
            "matched_domain": matched_domain,
            "matched_family": candidate.matched_family_id or "",
            "b2c_explanation": b2c_explanation,
            "source_row_index": source_row_index,
            "reference_row_index": reference_row_index,
            "explanation": explanation_json,
        }
        if source_identifier:
            result["source_key"] = source_identifier
            result["source_record_id"] = source_identifier
            result["source_primary_id"] = source_identifier
        return result

    def _enforce_blocking_guard(
        self,
        source_df: pd.DataFrame,
        reference_df: pd.DataFrame,
        blocks: Optional[Dict[str, Any]],
        mappings: List[MappingConfig],
    ) -> Optional[Dict[str, Any]]:
        """
        Prevent cartesian explosions with intelligent auto-blocking.

        When comparisons exceed HARD_CAP_COMPS (8M), automatically calls the desktop
        engine's blocking intelligence to find the best blocking column. If no suitable
        column exists, raises user-friendly error with actionable guidance.

        Returns:
            Updated blocks dict if auto-blocking was applied, None otherwise

        Raises:
            ManualMatchServiceError: If comparisons exceed limit and no valid blocking strategy exists
        """
        from fmatch.core.engine import (
            HARD_CAP_COMPS,
            determine_optimal_blocking_strategy,
            BlockingMode,
            BlockingGuardrailError,
        )

        N_src = len(source_df)
        N_ref = len(reference_df)
        estimated_pairs = N_src * N_ref
        explicit_domain_block = bool(blocks and blocks.get("domain"))
        explicit_state_block = bool(blocks and blocks.get("state"))
        skip_engine_analysis = explicit_domain_block or explicit_state_block

        # Check if we're under the hard cap (8M comparisons)
        if estimated_pairs < HARD_CAP_COMPS:
            self.logger.debug(
                f"Blocking guard: {estimated_pairs:,} estimated pairs < {HARD_CAP_COMPS:,} limit. Proceeding."
            )
            return None

        if explicit_domain_block or explicit_state_block:
            self.logger.info(
                "Blocking guard: %s estimated pairs with explicit blocking "
                "(domain=%s, state=%s). Validating coverage before skipping engine auto-blocking.",
                f"{estimated_pairs:,}",
                explicit_domain_block,
                explicit_state_block,
            )

        # First attempt manual domain blocking if viable to avoid engine fallback.
        def _valid_mask(series: pd.Series) -> pd.Series:
            if series is None:
                return pd.Series([], dtype=bool)
            placeholders = {
                "",
                "n/a",
                "na",
                "none",
                "null",
                "-",
                "--",
                "n\\a",
                "tbd",
                "unknown",
                "unspecified",
            }
            str_series = series.fillna("").astype(str).str.strip()
            lower_series = str_series.str.lower()
            mask = str_series != ""
            mask &= ~lower_series.isin(placeholders)
            return mask

        def _build_manual_prefix_fallback(
            src_df: pd.DataFrame,
            ref_df: pd.DataFrame,
            candidate_mappings: List[Dict[str, Any]],
        ) -> Optional[Tuple[Dict[str, Any], Dict[str, Any]]]:
            if not candidate_mappings:
                return None

            def _prefix_series(series: pd.Series) -> pd.Series:
                normalized = series.fillna("").astype(str).str.lower()
                normalized = normalized.str.replace(r"[^a-z0-9]", "", regex=True)
                return normalized.str[:3]

            label_to_pos = {label: pos for pos, label in enumerate(ref_df.index)}

            for mapping in candidate_mappings:
                src_col = mapping.get("source")
                ref_col = mapping.get("ref")
                if not src_col or not ref_col:
                    continue
                if src_col not in src_df.columns or ref_col not in ref_df.columns:
                    continue

                src_series = src_df[src_col]
                ref_series = ref_df[ref_col]

                src_keys = _prefix_series(src_series)
                ref_keys = _prefix_series(ref_series)

                src_valid = src_keys != ""
                ref_valid = ref_keys != ""

                src_coverage = float(src_valid.mean()) if len(src_keys) else 0.0
                ref_coverage = float(ref_valid.mean()) if len(ref_keys) else 0.0

                if ref_coverage < 0.01 or src_coverage < 0.01:
                    continue

                ref_index: Dict[str, List[int]] = {}
                total_entries = 0
                for label, key in ref_keys[ref_valid].items():
                    if not key:
                        continue
                    pos = label_to_pos.get(label)
                    if pos is None:
                        continue
                    ref_index.setdefault(key, []).append(pos)
                    total_entries += 1

                if not ref_index:
                    continue

                avg_bucket = total_entries / max(len(ref_index), 1)
                src_count = int(src_valid.sum())
                estimated_after = (
                    int(src_count * avg_bucket) if src_count and avg_bucket else None
                )

                residual_limit = (
                    max(50, min(250, int(avg_bucket * 4))) if avg_bucket else 50
                )
                source_keys = {idx: (val or "") for idx, val in src_keys.items()}

                engine_block_info = {
                    "source_keys": source_keys,
                    "ref_index": ref_index,
                    "residual_limit": residual_limit,
                    "reference_positions": list(range(len(ref_df))),
                }

                meta = {
                    "source_column": src_col,
                    "reference_column": ref_col,
                    "src_coverage": src_coverage,
                    "ref_coverage": ref_coverage,
                    "avg_bucket": avg_bucket,
                    "estimated_after": estimated_after,
                }
                return engine_block_info, meta

            return None

        domain_candidates: List[Tuple[float, MappingConfig, Dict[str, Any]]] = []
        domain_configs = [
            m for m in mappings if (m.algo or "").lower() in ("domain", "email_domain")
        ]

        MIN_SRC_COVERAGE = 0.02
        MIN_REF_COVERAGE = 0.02
        MIN_REF_UNIQUE = 25

        for cfg in domain_configs:
            if (
                cfg.source not in source_df.columns
                or cfg.reference not in reference_df.columns
            ):
                continue

            src_series = source_df[cfg.source]
            ref_series = reference_df[cfg.reference]

            src_mask = _valid_mask(src_series)
            ref_mask = _valid_mask(ref_series)

            if src_mask.empty or ref_mask.empty:
                continue

            src_valid = src_series[src_mask].astype(str).str.strip()
            ref_valid = ref_series[ref_mask].astype(str).str.strip()

            src_coverage = src_mask.mean()
            ref_coverage = ref_mask.mean()
            ref_unique = ref_valid.nunique(dropna=True)

            if (
                src_coverage >= MIN_SRC_COVERAGE
                and ref_coverage >= MIN_REF_COVERAGE
                and ref_unique >= MIN_REF_UNIQUE
            ):
                src_unique = src_valid.nunique(dropna=True)
                src_count = int(src_mask.sum())
                ref_count = int(ref_mask.sum())
                avg_ref_bucket = ref_count / ref_unique if ref_unique else None
                estimated_after = (
                    int(src_count * avg_ref_bucket) if avg_ref_bucket else None
                )
                reduction_ratio = (
                    (estimated_after / estimated_pairs) if estimated_after else None
                )
                score = min(src_coverage, ref_coverage) * math.log1p(ref_unique)
                domain_candidates.append(
                    (
                        score,
                        cfg,
                        {
                            "src_coverage": src_coverage,
                            "ref_coverage": ref_coverage,
                            "src_unique": src_unique,
                            "ref_unique": ref_unique,
                            "src_count": src_count,
                            "ref_count": ref_count,
                            "estimated_after": estimated_after,
                            "reduction_ratio": reduction_ratio,
                        },
                    )
                )

        if domain_candidates:
            domain_candidates.sort(key=lambda item: item[0], reverse=True)
            _, best_cfg, stats = domain_candidates[0]

            self.logger.info(
                "Blocking guard: leveraging manual domain blocking %s -> %s "
                "(source coverage %.1f%%, reference coverage %.1f%%, unique reference domains %d)",
                best_cfg.source,
                best_cfg.reference,
                stats["src_coverage"] * 100,
                stats["ref_coverage"] * 100,
                stats["ref_unique"],
            )

            estimated_after = stats["estimated_after"]
            if estimated_after:
                self.logger.info(
                    "Blocking guard: estimated comparisons after domain blocking ~%s "
                    "(%.1f%% of original)",
                    f"{estimated_after:,}",
                    stats["reduction_ratio"] * 100 if stats["reduction_ratio"] else 0.0,
                )
            else:
                self.logger.info(
                    "Blocking guard: unable to estimate comparisons after manual domain blocking (insufficient metrics)"
                )

            src_mask_best = _valid_mask(source_df[best_cfg.source])
            no_domain_mask = ~src_mask_best
            no_domain_ratio = (
                float(no_domain_mask.mean()) if len(src_mask_best) else 0.0
            )

            fallback_strategy = None
            engine_block_info = None

            # Attempt secondary blocking for rows without domains to avoid cartesian fallback.
            if no_domain_ratio > 0.0:
                non_domain_fraction = no_domain_ratio * 100
                self.logger.info(
                    "Blocking guard: %.1f%% of source rows lack %s; attempting secondary blocking strategy",
                    non_domain_fraction,
                    best_cfg.source,
                )

                fallback_mappings = [
                    {"source": m.source, "ref": m.reference}
                    for m in mappings
                    if (m.algo or "").lower() not in ("domain", "email_domain")
                    and m.source in source_df.columns
                    and m.reference in reference_df.columns
                ]

                # Guarantee at least a name-based mapping for fallback if possible.
                if not fallback_mappings:
                    default_source = "Company / Account"
                    default_ref = "Account Name"
                    if (
                        default_source in source_df.columns
                        and default_ref in reference_df.columns
                    ):
                        fallback_mappings = [
                            {"source": default_source, "ref": default_ref}
                        ]

                if fallback_mappings:
                    if skip_engine_analysis:
                        manual_fallback = _build_manual_prefix_fallback(
                            source_df,
                            reference_df,
                            fallback_mappings,
                        )
                        if manual_fallback:
                            engine_block_info, fallback_meta = manual_fallback
                            estimated_pairs_after = fallback_meta.get("estimated_after")
                            reduction_ratio = (
                                (estimated_pairs_after / estimated_pairs)
                                if estimated_pairs_after and estimated_pairs
                                else None
                            )
                            fallback_strategy = {
                                "engine_block_strategy": "manual_prefix",
                                "preproc": "prefix3",
                                "source_block_col": fallback_meta.get("source_column"),
                                "ref_block_col": fallback_meta.get("reference_column"),
                                "coverage_source": fallback_meta.get("src_coverage"),
                                "coverage_reference": fallback_meta.get("ref_coverage"),
                                "avg_bucket": fallback_meta.get("avg_bucket"),
                                "estimated_pairs_after": estimated_pairs_after,
                                "reduction_ratio": reduction_ratio,
                                "apply_blocking": True,
                            }
                            reduction_phrase = "unknown reduction"
                            if isinstance(estimated_pairs_after, (int, float)):
                                reduction_phrase = f"{estimated_pairs_after:,}"
                                if isinstance(reduction_ratio, (int, float)):
                                    reduction_phrase = f"{estimated_pairs_after:,} ({reduction_ratio * 100:.1f}% reduction)"
                            self.logger.info(
                                "Blocking guard: manual fallback prefix blocking %s -> %s "
                                "(coverage src=%.1f%%, ref=%.1f%%, avg bucket=%.1f, estimated pairs %s)",
                                fallback_meta.get("source_column"),
                                fallback_meta.get("reference_column"),
                                fallback_meta.get("src_coverage", 0.0) * 100,
                                fallback_meta.get("ref_coverage", 0.0) * 100,
                                fallback_meta.get("avg_bucket", 0.0) or 0.0,
                                reduction_phrase,
                            )
                        else:
                            self.logger.info(
                                "Blocking guard: unable to build manual fallback blocking from helper mappings; continuing with domain-only blocking."
                            )
                    else:
                        try:
                            from fmatch.core.engine import (
                                determine_optimal_blocking_strategy,
                            )

                            # Provide trimmed DataFrames containing only fallback columns to steer the engine.
                            fallback_source_cols = {
                                m["source"] for m in fallback_mappings
                            }
                            fallback_ref_cols = {m["ref"] for m in fallback_mappings}

                            source_subset = source_df[list(fallback_source_cols)].copy()
                            reference_subset = reference_df[
                                list(fallback_ref_cols)
                            ].copy()

                            fallback_strategy = determine_optimal_blocking_strategy(
                                src_df=source_subset,
                                ref_df=reference_subset,
                                mappings=fallback_mappings,
                                mode=BlockingMode.MATCH,
                                ruleset="default",
                            )

                            if fallback_strategy.get("apply_blocking"):
                                blocking_col_src = fallback_strategy.get(
                                    "source_block_col"
                                )
                                blocking_col_ref = fallback_strategy.get(
                                    "ref_block_col"
                                )

                                # Reject fallback if it loops back to the primary domain column to avoid the earlier issue.
                                domain_col_set = {
                                    best_cfg.source,
                                    "domain_for_blocking",
                                    "domain_extracted",
                                    "Apollo Domain",
                                    "Domain",
                                    "Website",
                                }
                                if (
                                    blocking_col_src
                                    and blocking_col_src in domain_col_set
                                ):
                                    self.logger.info(
                                        "Blocking guard: fallback strategy chose domain column '%s'; ignoring fallback.",
                                        blocking_col_src,
                                    )
                                    fallback_strategy = None
                                else:
                                    engine_block_info = (
                                        self._prepare_engine_block_index(
                                            source_df,
                                            reference_df,
                                            fallback_strategy,
                                        )
                                    )

                                    reduction_phrase = "unknown reduction"
                                    estimated_pairs_after = fallback_strategy.get(
                                        "estimated_pairs_after"
                                    )
                                    reduction_ratio = fallback_strategy.get(
                                        "reduction_ratio"
                                    )
                                    if isinstance(estimated_pairs_after, (int, float)):
                                        reduction_phrase = (
                                            f"{estimated_pairs_after:,.0f} ("
                                            f"{(reduction_ratio * 100):.1f}% reduction)"
                                            if isinstance(reduction_ratio, (int, float))
                                            else f"{estimated_pairs_after:,.0f}"
                                        )
                                    self.logger.info(
                                        "Blocking guard: secondary blocking %s -> %s with preproc=%s "
                                        "(fallback coverage target %.1f%% of rows, estimated pairs %s)",
                                        blocking_col_src,
                                        blocking_col_ref,
                                        fallback_strategy.get("preproc", "unknown"),
                                        non_domain_fraction,
                                        reduction_phrase,
                                    )
                            else:
                                reason = fallback_strategy.get(
                                    "reason", "no suitable column"
                                )
                                self.logger.info(
                                    "Blocking guard: fallback strategy was unable to apply blocking (%s)",
                                    reason,
                                )
                        except Exception as exc:
                            self.logger.warning(
                                "Blocking guard: fallback blocking strategy failed (%s). Continuing with domain-only blocking.",
                                exc,
                            )

            manual_strategy = {
                "engine_block_strategy": "domain",
                "is_manual_override": True,
                "source_block_col": best_cfg.source,
                "ref_block_col": best_cfg.reference,
                "preproc": "manual_domain",
                "coverage_source": stats["src_coverage"],
                "coverage_reference": stats["ref_coverage"],
                "unique_source": stats["src_unique"],
                "unique_reference": stats["ref_unique"],
                "estimated_pairs_before": estimated_pairs,
                "estimated_pairs_after": estimated_after,
                "reduction_ratio": stats["reduction_ratio"],
                "fallback_strategy": fallback_strategy,
            }

            return {
                "auto": True,
                "source_column": best_cfg.source,
                "reference_column": best_cfg.reference,
                "quality_score": float(
                    min(stats["src_coverage"], stats["ref_coverage"])
                ),
                "strategy": manual_strategy,
                "engine_block_info": engine_block_info,
            }

        if skip_engine_analysis:
            self.logger.info(
                "Blocking guard: explicit blocking supplied; skipping engine auto-blocking analysis."
            )
            return None

        # CRITICAL: Never trust user blocking configuration blindly
        # ALWAYS delegate to engine to validate and potentially apply fallbacks
        # Even if user enabled domain blocking, the engine needs to verify it will work

        block_domain = bool(blocks and blocks.get("domain"))
        block_state = bool(blocks and blocks.get("state"))

        # Log user intent but don't bypass engine
        if block_domain or block_state:
            self.logger.info(
                f"Blocking guard: {estimated_pairs:,} pairs estimated. User requested blocking "
                f"(domain={block_domain}, state={block_state}). Delegating to engine for validation..."
            )
        else:
            self.logger.info(
                f"Blocking guard: {estimated_pairs:,} pairs estimated, no user blocking config. "
                f"Delegating to engine for strategy..."
            )

        # ALWAYS try engine-based blocking intelligence
        multiplier = estimated_pairs / HARD_CAP_COMPS

        self.logger.info(
            f"Blocking guard: {estimated_pairs:,} comparisons ({multiplier:.1f}x over {HARD_CAP_COMPS:,} limit). "
            f"Attempting auto-blocking..."
        )

        try:
            # Convert MappingConfig to desktop engine format
            engine_mappings = [
                {"source": m.source, "ref": m.reference} for m in mappings
            ]

            # Call desktop engine's blocking intelligence
            strategy = determine_optimal_blocking_strategy(
                src_df=source_df,
                ref_df=reference_df,
                mappings=engine_mappings,
                mode=BlockingMode.MATCH,
                ruleset="default",
            )

            engine_block_info: Optional[Dict[str, Any]] = None

            # Check if engine recommends blocking
            if strategy.get("apply_blocking"):
                engine_block_info = self._prepare_engine_block_index(
                    source_df, reference_df, strategy
                )
                blocking_col_src = strategy.get("source_block_col")
                blocking_col_ref = strategy.get("ref_block_col")
                quality_score = strategy.get("quality_score", 0)
                estimated_after = strategy.get("estimated_pairs_after")
                reduction_ratio = strategy.get("reduction_ratio", 0)

                # Build telemetry message (handle missing metrics gracefully)
                if isinstance(estimated_after, (int, float)):
                    after_display = f"{int(estimated_after):,}"
                    reduction_phrase = (
                        f"reduces {estimated_pairs:,} -> {after_display} comparisons"
                    )
                else:
                    after_display = "unknown"
                    reduction_phrase = f"reduces {estimated_pairs:,} comparisons"
                if isinstance(reduction_ratio, (int, float)):
                    reduction_pct_display = (
                        f"{float(reduction_ratio)*100:.1f}% reduction"
                    )
                else:
                    reduction_pct_display = "unknown reduction"
                telemetry = f"{reduction_phrase}, {reduction_pct_display}"

                # Log strategy type and guardrail info
                engine_strategy = strategy.get("engine_block_strategy", "unknown")
                is_fallback = strategy.get("is_fallback", False)
                guardrail_triggered = strategy.get("guardrail_triggered")
                preproc = strategy.get("preproc", "unknown")

                strategy_desc = f"strategy={engine_strategy}"
                if is_fallback:
                    strategy_desc += " (FALLBACK)"
                if guardrail_triggered:
                    strategy_desc += f" guardrail={guardrail_triggered}"
                if preproc == "name_prefix":
                    blocked_pct = strategy.get("blocked_fraction", 0) * 100
                    avg_bucket = strategy.get("avg_bucket_size", 0)
                    strategy_desc += f" name_prefix(blocked={blocked_pct:.0f}%, avg_bucket={avg_bucket:.0f})"

                self.logger.info(
                    "Auto-blocking ENABLED: %s <-> %s (quality: %.2f, %s, %s)",
                    blocking_col_src,
                    blocking_col_ref,
                    quality_score,
                    telemetry,
                    strategy_desc,
                )

                # Return updated blocks configuration
                # Note: The actual blocking will be handled by existing logic in execute()
                # We're just signaling that auto-blocking found a good column
                return {
                    "auto": True,
                    "source_column": blocking_col_src,
                    "reference_column": blocking_col_ref,
                    "quality_score": quality_score,
                    "strategy": strategy,
                    "engine_block_info": engine_block_info,
                }
            else:
                # Engine says no suitable blocking column exists
                reason = strategy.get("reason", "No suitable blocking column found")
                self.logger.warning(f"Auto-blocking failed: {reason}")

                # Raise user-friendly error
                error_msg = (
                    f"Dataset too large: {estimated_pairs:,} comparisons "
                    f"({N_src:,} source x {N_ref:,} reference).\n\n"
                    f"No suitable blocking column found in your data.\n\n"
                    f"Recommended actions:\n"
                    f"1. Add email or website columns to enable domain-based matching\n"
                    f"2. Test with a smaller sample first (try limit: 1000)\n"
                    f"3. Add geographic columns (city/state) if matching by location\n\n"
                    f"With 1,000 rows: ~{N_ref:,} comparisons (completes in seconds)\n"
                    f"Current size would take: ~{self._estimate_runtime(estimated_pairs)}"
                )

                self.logger.error(error_msg)
                raise ManualMatchServiceError(error_msg)

        except BlockingGuardrailError as e:
            # Engine raised hard guardrail - even name-prefix fallback exceeded limits
            self.logger.error(f"Blocking guardrail triggered: {e}")

            error_msg = (
                f"Dataset too large for safe processing.\n\n"
                f"{str(e)}\n\n"
                f"Recommended actions:\n"
                f"1. Reduce dataset size (try limit: 1000 or 10000)\n"
                f"2. Add email/domain columns to enable efficient blocking\n"
                f"3. Filter data by region/category before matching\n\n"
                f"Current: {N_src:,} x {N_ref:,} = {estimated_pairs:,} comparisons\n"
                f"With 1,000 rows: ~{N_ref:,} comparisons (completes in seconds)"
            )
            raise ManualMatchServiceError(error_msg)

        except Exception as e:
            if isinstance(e, ManualMatchServiceError):
                raise

            # Desktop engine threw an unexpected error
            self.logger.error(f"Auto-blocking engine error: {e}", exc_info=True)

            # Fall back to simple error message
            error_msg = (
                f"Dataset too large: {estimated_pairs:,} comparisons "
                f"({N_src:,} source x {N_ref:,} reference).\n\n"
                f"Unable to automatically optimize blocking strategy.\n\n"
                f"Recommended actions:\n"
                f"1. Test with smaller sample first (limit: 1000)\n"
                f"2. Add email/website columns for better matching\n\n"
                f"Current size would take: ~{self._estimate_runtime(estimated_pairs)}"
            )

            raise ManualMatchServiceError(error_msg)

    def _estimate_runtime(self, pairs: int) -> str:
        """Estimate wall-clock runtime based on ~6000 pairs/second throughput."""
        seconds = pairs / 6000
        if seconds < 120:
            return f"{int(seconds)}s"
        elif seconds < 7200:
            return f"{int(seconds / 60)}min"
        else:
            return f"{seconds / 3600:.1f}hrs"

    def _dedupe_reasons(self, reasons: List[str]) -> List[str]:
        ordered: List[str] = []
        seen = set()
        for chip in reasons:
            if chip and chip not in seen:
                ordered.append(chip)
                seen.add(chip)
        return ordered


def execute(
    source_df: pd.DataFrame,
    reference_df: pd.DataFrame,
    mappings: List[MappingConfig],
    anchors: Optional[Dict[str, Any]] = None,
    blocks: Optional[Dict[str, Any]] = None,
    threshold: float = 0.6,  # Lowered from 0.8 to match UI default
    best_only: bool = True,
    strip_b2c: bool = True,
    limit: Optional[int] = None,
    logger: Optional[logging.Logger] = None,
    progress_callback: Optional[Callable[[int, int], None]] = None,
    source_id_column: Optional[str] = None,
    reference_id_column: Optional[str] = None,
    domain_registry: Optional[DuckDBDomainRegistry] = None,
) -> Tuple[pd.DataFrame, Dict[str, Any]]:
    """Convenience function to execute manual matching."""
    service = ManualMatchService(logger=logger, domain_registry=domain_registry)
    return service.execute(
        source_df=source_df,
        reference_df=reference_df,
        mappings=mappings,
        anchors=anchors,
        blocks=blocks,
        threshold=threshold,
        best_only=best_only,
        strip_b2c=strip_b2c,
        limit=limit,
        progress_callback=progress_callback,
        source_id_column=source_id_column,
        reference_id_column=reference_id_column,
    )
