# -*- coding: utf-8 -*-
"""Core fuzzy-matching engine logic."""

from __future__ import annotations  # MUST be first real line (PEP 236)

# stdlib imports
from contextlib import closing  # NEW - helps close the short RW connection
import cProfile

import logging
import hashlib

log = logging.getLogger(__name__)
from enum import Enum  # NEW - For BlockingMode
import multiprocessing as mp
import pickle
import base64  # Added pickle, base64
import sqlite3
import functools
import os
from functools import lru_cache  # <--- Import lru_cache and wraps
from fmatch.utils_normalize import to_domain, apply_transforms, apply_transforms_list

# import hashlib
import re
import threading
import time  # Added time for RunStats
from pathlib import Path
from typing import (
    Any,
    Dict,
    List,
    Optional,
    Union,  # Ensure Union is imported
    Tuple,
    Callable,
    Generator,
    Literal,
    Sequence,
)
from dataclasses import dataclass, field
from collections import defaultdict  # Added defaultdict
import uuid  # Added uuid
from dataclasses import asdict
import math
from datetime import datetime, date
import tempfile  # For memory-mapped files
from pyarrow import parquet as pq

# Ã¢â€â‚¬Ã¢â€â‚¬ third-party Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
import asteval
import duckdb  # Import BEFORE we reference its types
from duckdb import (
    DuckDBPyConnection,  # Added for stream_candidate_pairs
    ConnectionException,
    Error as DuckDBError,
)  # Removed manual os.makedirs and duckdb.connect
import numpy as np
import pandas as pd

_pd = pd
# --------------------------------------------------------------------------- #
#  Optional dependency: psutil
#  (If it's missing we silently fall back to 'scale'-mode.)
# --------------------------------------------------------------------------- #
try:
    import psutil  # noqa: F401
except ImportError:  # pragma: no cover
    DuckDBPyConnection = (
        Any  # Placeholder if duckdb is not available, though it should be
    )
    psutil = None
import pyarrow as pa
import pyarrow.feather as feather
import yaml

# Ã¢â€â‚¬Ã¢â€â‚¬ Runtime-aware resource management Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
try:
    from .engine_runtime import (
        get_runtime_manager,
        get_max_workers,
        get_rf_threads,
        should_use_memory_mapped as runtime_should_use_mm,
        get_processing_mode as runtime_get_processing_mode,
        apply_degradation,
        ConcurrencyModel,
    )

    RUNTIME_AWARE = True
    runtime_manager = get_runtime_manager()
    log.info("[engine] Runtime-aware resource management enabled")
except ImportError as e:
    log.warning(f"[engine] Runtime management not available: {e}")
    RUNTIME_AWARE = False
    runtime_manager = None
from rapidfuzz import fuzz as rf_fuzz
from rapidfuzz import process as rf_process
from rapidfuzz.distance import Jaro, JaroWinkler
from fmatch.core.b2c_engine_integration import (
    cleanse_single_domain,
)

# from unidecode import unidecode  # REPLACED with script-aware normalization
from ..preprocessing.rules.text_normalizers import normalize_text, _strip_accents_latin
import unicodedata as _ud

try:
    import chardet  # For detect_encoding

    CHARDET_AVAILABLE = True
except ImportError:  # pragma: no cover - optional dependency
    chardet = None  # type: ignore
    CHARDET_AVAILABLE = False
    log.warning(
        "chardet not available; defaulting to UTF-8 for CSV encoding detection"
    )

try:
    from fmatch.profiling.exotic_data_detector import ExoticDataDetector, EncodingFixer
except ImportError:
    log.warning("ExoticDataDetector not available - exotic data handling disabled")
    ExoticDataDetector = None
    EncodingFixer = None

from .progress_event import ProgressPhase
from .engine_runstats import RunStats
from fmatch.core.blocking_metrics import (
    calculate_raw_signals,
    normalize_signals,
    quality_score,
    _calculate_gini,
    _calculate_entropy,
)
from fmatch.core.blocking.guardrails import BlockGuardrails

from fmatch.core.utils import sanitize_column_name

_sanitize_name = sanitize_column_name

from fmatch.core.algorithm_selector import (
    MultiAlgorithmScorer,
    EnhancedMapping,
    EnhancedMatchConfig,
    EnhancedDedupeConfig,
)

from fmatch.engine_algorithm_integration import (
    enhance_strategy_with_algorithm_rules,
)


def _enhanced_mapping_is_domain(mapping: EnhancedMapping) -> bool:
    """Determine whether a mapping targets domain/url data."""
    if not mapping:
        return False
    transforms = getattr(mapping, "transforms", []) or []
    if any(t == "to_domain" for t in transforms):
        return True
    source_name = (mapping.source or "").lower()
    ref_name = (mapping.ref or mapping.source or "").lower()
    domain_tokens = ("domain", "website", "url", "hostname", "host")
    return any(token in source_name for token in domain_tokens) or any(
        token in ref_name for token in domain_tokens
    )


# Memory-mapped table globals for workers
_GLOBAL_SRC_TBL = None
_GLOBAL_REF_TBL = None
_GLOBAL_CONFIG = None
_GLOBAL_SCORE_CACHE = None

# Legacy DataFrame globals (kept for compatibility)
_GLOBAL_SRC_DF: pd.DataFrame | None = None
_GLOBAL_REF_DF: pd.DataFrame | None = None

# Group index caches (worker-local)
_GLOBAL_SRC_GROUPS = None
_GLOBAL_REF_GROUPS = None
_GLOBAL_BLOCK_KEY_NAME = None  # Store the key name to avoid drift

# Track slow blocks for sampling
_slow_block_counter = 0
_slow_block_lock = threading.Lock()


def _inc_slow_block_counter():
    global _slow_block_counter
    with _slow_block_lock:
        _slow_block_counter += 1
        return _slow_block_counter


class ColumnStatsCache:
    """Lightweight cache for column-level metadata reused across mapping and blocking."""

    def __init__(self):
        self._column_stats: dict[tuple[int, str], dict] = {}
        self._processed_headers: dict[str, str] = {}
        self._header_tokens: dict[str, set[str]] = {}
        self._detected_types: dict[tuple[int, str], str] = {}
        self._normalized_overlap: dict[tuple[tuple[int, str], str], np.ndarray] = {}
        self._variant_outputs: dict[
            tuple[tuple[int, str | None], str, str], tuple[pd.Series, int]
        ] = {}

    @staticmethod
    def remember_series(series: pd.Series, parent_df_id: int) -> pd.Series:
        try:
            series.attrs["_fmatch_parent_df_id"] = parent_df_id
        except Exception:
            pass
        return series

    @staticmethod
    def _series_key(
        series: pd.Series, header: str | None = None
    ) -> tuple[int, str | None]:
        parent_id = series.attrs.get("_fmatch_parent_df_id")
        if parent_id is None:
            parent_id = id(series)
        label = header if header is not None else getattr(series, "name", None)
        return parent_id, label

    @staticmethod
    def _df_key(df: pd.DataFrame, column: str) -> tuple[int, str]:
        return id(df), column

    def get_series_metrics(self, series: pd.Series, header: str, compute_fn) -> dict:
        key = self._series_key(series, header)
        if key not in self._column_stats:
            remembered = self.remember_series(
                series, series.attrs.get("_fmatch_parent_df_id", id(series))
            )
            self._column_stats[key] = compute_fn(remembered)
        return self._column_stats[key]

    def cache_series_metrics(
        self, series: pd.Series, header: str, metrics: dict
    ) -> None:
        key = self._series_key(series, header)
        self._column_stats[key] = metrics

    def get_column_stats(self, df: pd.DataFrame, column: str, compute_fn) -> dict:
        key = self._df_key(df, column)
        if key not in self._column_stats:
            series = self.remember_series(df[column], id(df))
            self._column_stats[key] = compute_fn(series)
        return self._column_stats[key]

    def get_processed_header(self, header: str) -> str:
        key = header or ""
        if key not in self._processed_headers:
            self._processed_headers[key] = _process_col_name_for_fuzzy(header)
        return self._processed_headers[key]

    def get_header_tokens(self, header: str) -> set[str]:
        key = header or ""
        if key not in self._header_tokens:
            tokens = set(re.findall(r"[a-z]+", str(header or "").lower()))
            self._header_tokens[key] = tokens
        return self._header_tokens[key]

    def get_detected_type(self, series: pd.Series, header: str, compute_fn) -> str:
        key = self._series_key(series, header)
        if key not in self._detected_types:
            self._detected_types[key] = compute_fn(header, series)
        return self._detected_types[key]

    def get_normalized_overlap(
        self, series: pd.Series, detected_type: str, normalizer
    ) -> np.ndarray:
        key = (self._series_key(series), detected_type)
        if key not in self._normalized_overlap:
            self._normalized_overlap[key] = normalizer(series, detected_type)
        return self._normalized_overlap[key]

    def get_variant_output(
        self, series: pd.Series, variant: str, k_len: int, ruleset: str, builder
    ) -> tuple[pd.Series, int]:
        key = (self._series_key(series), variant, ruleset or "")
        cached = self._variant_outputs.get(key)
        if cached:
            cached_series, max_k = cached
            if k_len <= max_k:
                if k_len == max_k:
                    return cached_series.copy(deep=False), max_k
                return cached_series.str.slice(0, k_len), max_k

        generated_series, eff_k = builder()
        generated_series = generated_series.copy(deep=False)

        if cached:
            cached_series, cached_max = cached
            if cached_max > eff_k:
                self._variant_outputs[key] = (cached_series, cached_max)
                if k_len <= cached_max:
                    if k_len == cached_max:
                        return cached_series.copy(deep=False), cached_max
                    return cached_series.str.slice(0, k_len), cached_max
                return cached_series.str.slice(0, min(k_len, cached_max)), cached_max

        self._variant_outputs[key] = (generated_series, eff_k)
        if k_len < eff_k:
            return generated_series.str.slice(0, k_len), eff_k
        return generated_series.copy(deep=False), eff_k

    def clear(self) -> None:
        self._column_stats.clear()
        self._processed_headers.clear()
        self._header_tokens.clear()
        self._detected_types.clear()
        self._normalized_overlap.clear()
        self._variant_outputs.clear()


# Compatibility shim for _rf_threads used by pool initializers
def _rf_threads(worker_processes: int) -> int:
    """
    Calculate optimal RapidFuzz threads per worker.
    Compatibility shim that works with both runtime-aware and legacy modes.
    FM_RF_THREADS=0 or unset means 'auto'.
    FM_RF_THREADS>0 means use that exact value.
    """
    if RUNTIME_AWARE:
        requested = get_rf_threads()
        if requested and requested > 0:
            return max(1, requested)
        return 1

    env = os.getenv("FM_RF_THREADS")
    try:
        v = int(env) if env is not None else 0
    except ValueError:
        v = 0

    if v > 0:
        return v

    return 1


_WARNED_HIGH_WORKERS = False


def _cleanup_worker_cache():
    """Cleanup function called when worker process exits."""
    global _GLOBAL_SCORE_CACHE
    if _GLOBAL_SCORE_CACHE is not None:
        try:
            _GLOBAL_SCORE_CACHE.flush()
            _GLOBAL_SCORE_CACHE.close()
            log.debug(
                f"[Worker {os.getpid()}] ScoreCache flushed and closed on shutdown"
            )
        except Exception as e:
            log.warning(
                f"[Worker {os.getpid()}] Failed to flush/close cache on shutdown: {e}"
            )
        finally:
            # Clear reference to prevent any accidental reuse after close
            _GLOBAL_SCORE_CACHE = None


def _add_prefixed_row(record: dict, prefix: str, row_data, columns) -> None:
    """Add row data to record with prefix, filtering out temp columns."""
    # Handle both tuple/list (indexed) and Series/dict (keyed) data
    if hasattr(row_data, "__getitem__"):
        for i, col in enumerate(columns):
            if not col.startswith("_temp_"):
                # Try both indexed and keyed access
                try:
                    value = (
                        row_data[i]
                        if isinstance(row_data, (tuple, list))
                        else row_data[col]
                    )
                    record[f"{prefix}_{sanitize_column_name(col)}"] = value
                except (IndexError, KeyError):
                    pass


def _pool_init(src_df: pd.DataFrame, ref_df: pd.DataFrame, config: MatchConfig):
    """
    OPTIMIZED: Pre-compute group indices with category optimization.
    """
    global _GLOBAL_SRC_DF, _GLOBAL_REF_DF, _GLOBAL_CONFIG, _GLOBAL_SCORE_CACHE
    global _RF_WORKERS, RAPIDFUZZ_THREAD_COUNT
    global _GLOBAL_SRC_GROUPS, _GLOBAL_REF_GROUPS, _GLOBAL_BLOCK_KEY_NAME

    # Store block key name to avoid drift
    _GLOBAL_BLOCK_KEY_NAME = getattr(config, "block_col", "_ACTIVE_BLOCK_KEY_")

    # Reset indices for clean slicing
    _GLOBAL_SRC_DF = src_df.reset_index(drop=True)
    _GLOBAL_REF_DF = ref_df.reset_index(drop=True)
    _GLOBAL_CONFIG = config

    # OPTIMIZATION: Convert block keys to category for faster groupby
    if _GLOBAL_BLOCK_KEY_NAME in _GLOBAL_SRC_DF.columns:
        # Convert to category if it's object/string for memory and speed
        if _GLOBAL_SRC_DF[_GLOBAL_BLOCK_KEY_NAME].dtype == "object":
            _GLOBAL_SRC_DF[_GLOBAL_BLOCK_KEY_NAME] = _GLOBAL_SRC_DF[
                _GLOBAL_BLOCK_KEY_NAME
            ].astype("category")

        _GLOBAL_SRC_GROUPS = _GLOBAL_SRC_DF.groupby(
            _GLOBAL_BLOCK_KEY_NAME, observed=True
        ).indices
        log.info(
            f"[Worker {os.getpid()}] Cached {len(_GLOBAL_SRC_GROUPS)} source block indices"
        )
    else:
        _GLOBAL_SRC_GROUPS = {}

    if _GLOBAL_BLOCK_KEY_NAME in _GLOBAL_REF_DF.columns:
        if _GLOBAL_REF_DF[_GLOBAL_BLOCK_KEY_NAME].dtype == "object":
            _GLOBAL_REF_DF[_GLOBAL_BLOCK_KEY_NAME] = _GLOBAL_REF_DF[
                _GLOBAL_BLOCK_KEY_NAME
            ].astype("category")

        _GLOBAL_REF_GROUPS = _GLOBAL_REF_DF.groupby(
            _GLOBAL_BLOCK_KEY_NAME, observed=True
        ).indices
        log.info(
            f"[Worker {os.getpid()}] Cached {len(_GLOBAL_REF_GROUPS)} reference block indices"
        )
    else:
        _GLOBAL_REF_GROUPS = {}

    # Set RapidFuzz threads
    procs = max(1, getattr(config, "max_workers", _auto_worker_count()))
    _RF_WORKERS = _rf_threads(procs)
    RAPIDFUZZ_THREAD_COUNT = _RF_WORKERS

    # CRITICAL: Set the environment variable that RapidFuzz actually reads
    os.environ["RAPIDFUZZ_THREAD_COUNT"] = str(RAPIDFUZZ_THREAD_COUNT)
    log.info(
        f"[Worker {os.getpid()}] Initialized with RF threads = {RAPIDFUZZ_THREAD_COUNT} (env var set)"
    )

    # Build and store enhanced mappings
    if (
        hasattr(config, "enable_multi_algo")
        and config.enable_multi_algo
        and config.maps
    ):
        enhanced_maps = []
        for m in config.maps:
            enhanced_maps.append(
                EnhancedMapping(
                    source=m.get("source"),
                    ref=m.get("ref"),
                    weight=m.get("weight", 1.0),
                    preferred_algo=m.get("preferred_algo"),
                    algo_confidence=m.get("algo_confidence", 0.0),
                    transforms=m.get("transforms") or [],
                    preprocessing=m.get("preprocessing"),
                )
            )
        _GLOBAL_CONFIG.enhanced_maps = enhanced_maps
    else:
        _GLOBAL_CONFIG.enhanced_maps = []

    # CRITICAL: Default to NO cache unless explicitly requested
    if getattr(config, "processing_mode", "scale") == "speed":
        _GLOBAL_SCORE_CACHE = None
        log.debug(f"[Worker {os.getpid()}] Speed mode - cache disabled")
        return

    # Only initialize cache if explicit path provided
    if (
        hasattr(config, "cache_path")
        and config.cache_path
        and config.cache_path != ":memory:"
    ):
        try:
            from fmatch.core.cache.score_cache import ScoreCache, ScoreCacheConfig

            cache_config = ScoreCacheConfig(db_path=config.cache_path)
            _GLOBAL_SCORE_CACHE = ScoreCache(
                cache_config, is_worker=True, read_only=True
            )
            log.info(
                f"[Worker {os.getpid()}] ScoreCache explicitly enabled at {config.cache_path}"
            )

            import atexit

            atexit.register(_cleanup_worker_cache)
        except Exception as e:
            _GLOBAL_SCORE_CACHE = None
            log.debug(
                f"[Worker {os.getpid()}] Cache init failed (proceeding without): {e}"
            )
    else:
        _GLOBAL_SCORE_CACHE = None
        log.debug(
            f"[Worker {os.getpid()}] No explicit cache path - proceeding without cache"
        )


def _pool_init_mm(src_path: str, ref_path: str, cfg: MatchConfig):
    """
    Lightweight initializer that opens memory-mapped Arrow tables.
    Each worker opens read-only views - no copying!
    """
    global \
        _GLOBAL_SRC_TBL, \
        _GLOBAL_REF_TBL, \
        _GLOBAL_CONFIG, \
        _GLOBAL_SCORE_CACHE, \
        _RF_WORKERS, \
        RAPIDFUZZ_THREAD_COUNT

    # Open memory-mapped Arrow tables (zero-copy, shared read-only)
    _GLOBAL_SRC_TBL = feather.read_table(src_path, memory_map=True)
    _GLOBAL_REF_TBL = feather.read_table(ref_path, memory_map=True)
    _GLOBAL_CONFIG = cfg

    # CRITICAL: Set RapidFuzz threads properly for this worker
    procs = max(1, getattr(cfg, "max_workers", _auto_worker_count()))
    _RF_WORKERS = _rf_threads(procs)
    RAPIDFUZZ_THREAD_COUNT = _RF_WORKERS  # CRITICAL: Set both!

    # CRITICAL: Set the environment variable that RapidFuzz actually reads
    os.environ["RAPIDFUZZ_THREAD_COUNT"] = str(RAPIDFUZZ_THREAD_COUNT)
    log.info(
        f"[Worker {os.getpid()}] Initialized with memory-mapped tables, RF threads = {RAPIDFUZZ_THREAD_COUNT} (env var set)"
    )

    # Initialize score cache if needed (for scale mode)
    if (
        getattr(cfg, "processing_mode", "scale") == "scale"
        and hasattr(cfg, "cache_path")
        and cfg.cache_path
    ):
        from fmatch.core.cache.score_cache import ScoreCache, ScoreCacheConfig

        try:
            cache_config = ScoreCacheConfig(db_path=cfg.cache_path)
            _GLOBAL_SCORE_CACHE = ScoreCache(
                cache_config, is_worker=True, read_only=True
            )
            log.info(f"[Worker {os.getpid()}] ScoreCache initialized (in-memory)")
        except Exception as e:
            _GLOBAL_SCORE_CACHE = None
            log.warning(f"[Worker {os.getpid()}] Failed to initialize ScoreCache: {e}")


def _process_one_key_mm(task_data: tuple) -> Union[list[dict], tuple[list[dict], dict]]:
    """
    Process a single block using row indices to fetch data from memory-mapped tables.
    Receives: (source_indices, ref_indices, block_info)
    """
    try:
        s_indices, r_indices, block_info = task_data

        if not s_indices or not r_indices:
            progress = {
                "block_key": block_info.get("key"),
                "blocks": 1,
                "items": 0,
                "comparisons": 0,
                "matches": 0,
            }
            return [], progress

        # Materialize only this small block from the memory-mapped tables
        # Use Arrow arrays directly to avoid conversion overhead
        s_arr = (
            pa.array(s_indices) if not isinstance(s_indices, pa.Array) else s_indices
        )
        r_arr = (
            pa.array(r_indices) if not isinstance(r_indices, pa.Array) else r_indices
        )
        src_block = _GLOBAL_SRC_TBL.take(s_arr).to_pandas()
        ref_block = _GLOBAL_REF_TBL.take(r_arr).to_pandas()

        # Determine block key for logging
        is_soft_widen = block_info.get("is_soft_widen", False)
        if is_soft_widen:
            block_log_key = f"{block_info.get('src_key')}->{block_info.get('ref_key')}"
        else:
            block_log_key = str(block_info.get("key"))

        # Instrument block processing
        pid = os.getpid()
        t_start = time.perf_counter()
        ops_count = len(src_block) * len(ref_block)

        _trace_block(
            f"[Worker {pid}] Starting block '{block_log_key}' "
            f"src={len(src_block)} ref={len(ref_block)} ops={ops_count}"
        )

        # Calculate total weight
        total_weight = (
            sum(float(m.get("weight", 1.0)) for m in (_GLOBAL_CONFIG.maps or []))
            or 0.01
        )

        # Build enhanced mappings from config
        enhanced_mappings = []
        if getattr(_GLOBAL_CONFIG, "enable_multi_algo", False) and _GLOBAL_CONFIG.maps:
            for m in _GLOBAL_CONFIG.maps:
                enhanced_mappings.append(
                    EnhancedMapping(
                        source=m.get("source"),
                        ref=m.get("ref"),
                        weight=m.get("weight", 1.0),
                        preferred_algo=m.get("preferred_algo"),
                        algo_confidence=m.get("algo_confidence", 0.0),
                        transforms=m.get("transforms") or [],
                        preprocessing=m.get("preprocessing"),
                    )
                )

        # Process the block using existing logic
        matches = process_block(
            src_block_df=src_block,
            ref_block_df=ref_block,
            block_key_val=block_log_key,
            block_idx=block_info.get("idx", 0),
            src_id_col=_GLOBAL_CONFIG.src_id_col,
            ref_id_col=_GLOBAL_CONFIG.ref_id_col,
            threshold=block_info.get("threshold", _GLOBAL_CONFIG.threshold),
            algo_key_str=_GLOBAL_CONFIG.algorithm,
            maps_for_algo=_GLOBAL_CONFIG.maps,
            enable_multi_algo=getattr(_GLOBAL_CONFIG, "enable_multi_algo", False),
            scorer=None,
            enhanced_mappings=enhanced_mappings,  # Pass actual enhanced mappings
            algo_callable=ALGORITHM_MAP.get(_GLOBAL_CONFIG.algorithm),
            coverage_gate=getattr(_GLOBAL_CONFIG, "coverage_gate", 0.60),
            MAX_TOTAL_WEIGHT=total_weight,
            pid=os.getpid(),
        )

        # Log completion
        t_elapsed = time.perf_counter() - t_start
        safe_elapsed = t_elapsed if t_elapsed > 0 else 1e-9
        throughput_mops = ops_count / safe_elapsed / 1e6

        _trace_block(
            f"[Worker {pid}] Finished block '{block_log_key}' in {t_elapsed:.2f}s "
            f"matches={len(matches)} throughput={throughput_mops:.2f}M ops/s",
            elapsed=t_elapsed,
        )

        # Log slow blocks
        if t_elapsed > 3.0:
            log.warning(
                f"[SLOW BLOCK] '{block_log_key}' took {t_elapsed:.2f}s for {ops_count} ops"
            )

        progress = {
            "block_key": block_log_key,
            "blocks": 1,
            "items": len(src_block),
            "comparisons": ops_count,
            "matches": len(matches) if matches else 0,
        }

        return matches, progress

    except Exception as e:
        log.error(f"[Worker {os.getpid()}] Failed to process block: {e}")
        return [], None


def _process_one_key(block_info: dict) -> Union[list[dict], tuple[list[dict], dict]]:
    """
    OPTIMIZED: Fast block processing with pre-computed indices.
    """
    global _slow_block_counter

    if _GLOBAL_SRC_DF is None or _GLOBAL_REF_DF is None or _GLOBAL_CONFIG is None:
        log.error(f"[Worker {os.getpid()}] Global data not initialized!")
        return []

    # Determine block keys
    is_soft_widen = block_info.get("is_soft_widen", False)
    if is_soft_widen:
        src_key = block_info.get("src_key")
        ref_key = block_info.get("ref_key")
        block_log_key = f"{src_key}->{ref_key}"
    else:
        src_key = ref_key = block_info.get("key")
        block_log_key = src_key

    if not src_key or not ref_key:
        progress = {
            "block_key": block_info.get("key"),
            "blocks": 1,
            "items": 0,
            "comparisons": 0,
            "matches": 0,
        }
        return [], progress

    # Use pre-computed indices - O(1) lookup!
    src_indices = _GLOBAL_SRC_GROUPS.get(src_key)
    ref_indices = _GLOBAL_REF_GROUPS.get(ref_key)

    if src_indices is None or ref_indices is None:
        progress = {
            "block_key": block_log_key,
            "blocks": 1,
            "items": 0,
            "comparisons": 0,
            "matches": 0,
        }
        return [], progress

    # GUARD: Dedupe can't compare singletons; match can score 1xN blocks.
    mode = (getattr(_GLOBAL_CONFIG, "mode", "match") or "match").lower()
    min_rows = MIN_BLOCK_SIZE if mode in ("dedupe", "duplicate") else 1
    if len(src_indices) < min_rows or len(ref_indices) < min_rows:
        log.debug(
            f"[Worker {os.getpid()}] Skipping tiny block '{block_log_key}' "
            f"(src={len(src_indices)}, ref={len(ref_indices)})"
        )
        progress = {
            "block_key": block_log_key,
            "blocks": 1,
            "items": 0,
            "comparisons": 0,
            "matches": 0,
        }
        return [], progress

    # Direct slice with pre-computed indices - no DataFrame scan!
    src_block = _GLOBAL_SRC_DF.iloc[src_indices]
    ref_block = _GLOBAL_REF_DF.iloc[ref_indices]

    pid = os.getpid()
    start_time = time.perf_counter()
    ops_count = len(src_block) * len(ref_block)

    _trace_block(
        f"[Worker {pid}] Starting block '{block_log_key}' "
        f"src={len(src_block)} ref={len(ref_block)} ops={ops_count}"
    )

    # Use enhanced mappings from config if already built
    enhanced_mappings = getattr(_GLOBAL_CONFIG, "enhanced_maps", [])

    # Process block using the existing robust logic
    effective_threshold = block_info.get("threshold", _GLOBAL_CONFIG.threshold)
    matches = process_block(
        src_block,
        ref_block,
        block_log_key,
        0,
        _GLOBAL_CONFIG.src_id_col,
        _GLOBAL_CONFIG.ref_id_col,
        effective_threshold,
        _GLOBAL_CONFIG.algorithm,
        _GLOBAL_CONFIG.maps,
        getattr(_GLOBAL_CONFIG, "enable_multi_algo", False),
        None,
        enhanced_mappings,  # Use pre-built enhanced mappings
        ALGORITHM_MAP.get(_GLOBAL_CONFIG.algorithm),
        getattr(_GLOBAL_CONFIG, "coverage_gate", 0.60),
        sum(float(m.get("weight", 1.0)) for m in _GLOBAL_CONFIG.maps)
        if _GLOBAL_CONFIG.maps
        else 0.01,
        pid,
    )

    # Performance logging with sampling
    elapsed = time.perf_counter() - start_time
    safe_elapsed = elapsed if elapsed > 0 else 1e-9
    throughput_mops = ops_count / safe_elapsed / 1e6

    _trace_block(
        f"[Worker {pid}] Finished block '{block_log_key}' in {elapsed:.3f}s "
        f"matches={len(matches)} throughput={throughput_mops:.1f}M ops/s",
        elapsed=elapsed,
    )

    # SAMPLED slow block warnings
    if (
        ops_count >= SLOW_BLOCK_MIN_OPS
        and throughput_mops < SLOW_BLOCK_THROUGHPUT_THRESHOLD
    ):
        counter = _inc_slow_block_counter()
        if counter % int(1 / SLOW_BLOCK_SAMPLE_RATE) == 0:  # Sample 1 in 5
            log.warning(
                f"[SLOW BLOCK SAMPLE] '{block_log_key}' took {elapsed:.2f}s "
                f"for {ops_count} ops ({throughput_mops:.2f}M ops/s)"
            )

    progress = {
        "block_key": block_log_key,
        "blocks": 1,
        "items": len(src_block),
        "comparisons": ops_count,
        "matches": len(matches) if matches else 0,
    }

    if hasattr(_GLOBAL_CONFIG, "collect_metrics") and _GLOBAL_CONFIG.collect_metrics:
        progress["metrics"] = {
            "block_key": str(block_log_key),
            "src_len": len(src_block),
            "ref_len": len(ref_block),
            "comparisons": ops_count,
            "matches_found": len(matches),
            "processing_time_ms": elapsed * 1000,
            "worker_pid": pid,
        }

    return matches, progress


# Add this function near the top of engine.py
def _auto_worker_count(
    mem_per_worker_mb: int = 300, env_var: str = "FM_CPU_LIMIT"
) -> int:
    """Calculate optimal worker count based on available resources."""
    # 1 - explicit override
    if env_var in os.environ:
        try:
            return max(1, int(os.environ[env_var]))
        except ValueError:
            log.warning("%s is not an int - ignoring", env_var)

    # 2 - Calculate based on available memory and CPU
    if psutil:
        vm = psutil.virtual_memory()
        available_mb = vm.available / (1024 * 1024)
        cpu_count = os.cpu_count() or 1

        # Conservative thresholds
        if available_mb < 4096:  # < 4GB available
            return min(2, cpu_count)
        elif available_mb < 8192:  # < 8GB available
            return min(3, cpu_count)
        elif available_mb < 16384:  # < 16GB available
            return min(4, cpu_count)
        else:
            # For larger systems, cap at 6 to avoid oversubscription
            mem_based_workers = int(available_mb / (mem_per_worker_mb * 2))
            return min(6, mem_based_workers, cpu_count)

    # Fallback
    return min(4, os.cpu_count() or 1)


class BlockProcessingMonitor:
    def __init__(self, total_blocks: int):
        self.total = total_blocks
        self.completed = 0
        self.start_time = time.perf_counter()
        self.lock = threading.Lock()

    def update(self, blocks_done: int = 1):
        with self.lock:
            self.completed += blocks_done
            elapsed = time.perf_counter() - self.start_time
            rate = self.completed / elapsed if elapsed > 0 else 0
            eta = (self.total - self.completed) / rate if rate > 0 else 0

            if self.completed % 10 == 0 or self.completed == self.total:
                log.info(
                    f"Block progress: {self.completed}/{self.total} "
                    f"({self.completed/self.total*100:.1f}%) "
                    f"Rate: {rate:.1f} blocks/s, ETA: {eta:.1f}s"
                )


def _match_in_parallel(
    cfg: MatchConfig,
    src_df: pd.DataFrame,
    ref_df: pd.DataFrame,
    all_blocks: list[dict],
    run_stats: Optional[RunStats],
    progress_q: Optional[mp.Queue],
) -> list[dict]:
    """
    OPTIMIZED: Smart chunking and worker management.
    """
    num_workers = min(
        getattr(cfg, "max_workers", _auto_worker_count()), _auto_worker_count()
    )

    if not all_blocks:
        return []

    total_ops = sum(b["metric"] for b in all_blocks)
    total_blocks = len(all_blocks)

    # Don't parallelize tiny workloads
    if total_ops < PARALLEL_OPS_THRESHOLD or total_blocks < 50:
        log.info(
            f"Small workload ({total_ops:,} ops, {total_blocks} blocks) - sequential processing"
        )
        # _process_one_key relies on globals normally set in mp.Pool initializers.
        # In the sequential fallback we must initialize them explicitly.
        _pool_init(src_df, ref_df, cfg)
        all_matches = []
        processed_blocks = 0
        for block_info in all_blocks:
            result = _process_one_key(block_info)
            if isinstance(result, tuple):
                matches_from_block, progress = result
            else:
                matches_from_block, progress = result, None

            if matches_from_block:
                all_matches.extend(matches_from_block)

            if progress:
                blocks_delta = int(progress.get("blocks", 1) or 1)
                processed_blocks += blocks_delta
                if run_stats:
                    run_stats.update_blocks(
                        blocks_processed=processed_blocks,
                        total_blocks=len(all_blocks),
                        items_in_chunk=progress.get("items", 0),
                        comparisons_in_chunk=progress.get("comparisons", 0),
                        progress_q=progress_q,
                    )
                _emit_progress(
                    progress.get("block_key"),
                    progress.get("items", 0),
                    progress.get("comparisons", 0),
                    progress.get("matches", 0),
                    blocks_delta=blocks_delta,
                )
            else:
                processed_blocks += 1
                if run_stats:
                    run_stats.update_blocks(
                        blocks_processed=processed_blocks,
                        total_blocks=len(all_blocks),
                        items_in_chunk=block_info.get("src_len", 0),
                        comparisons_in_chunk=block_info.get("metric", 0),
                        progress_q=progress_q,
                    )
                _emit_progress(
                    block_info.get("key"),
                    block_info.get("src_len", 0),
                    block_info.get("metric", 0),
                    0,
                )

        _emit_progress(None, 0, 0, 0, blocks_delta=0, flush=True)
        return all_matches

    # Adaptive worker scaling
    if total_blocks < 100:
        num_workers = min(2, num_workers)
    elif total_blocks < 500:
        num_workers = min(3, num_workers)
    else:
        num_workers = min(4, num_workers)  # Cap at 4 for stability

    # OPTIMIZED chunksize calculation
    if total_blocks < 2 * num_workers:
        chunksize = 1  # Keep all workers busy with tiny queue
    else:
        chunksize = max(1, min(32, total_blocks // (num_workers * 2)))  # Cap at 32

    log.info(
        f"Parallel processing: {total_blocks} blocks, {num_workers} workers, "
        f"chunksize={chunksize}, {total_ops:,} total ops"
    )

    all_matches = []
    processed_blocks = 0

    if run_stats:
        run_stats.emit_heartbeat(
            f"Processing {total_blocks} blocks with {num_workers} workers..."
        )

    # Regular pool without memory mapping
    with mp.Pool(
        processes=num_workers, initializer=_pool_init, initargs=(src_df, ref_df, cfg)
    ) as pool:
        for block_result in pool.imap_unordered(
            _process_one_key, all_blocks, chunksize=chunksize
        ):
            if isinstance(block_result, tuple):
                matches_from_block, progress = block_result
            else:
                matches_from_block, progress = block_result, None

            if matches_from_block:
                all_matches.extend(matches_from_block)

            if progress:
                blocks_delta = int(progress.get("blocks", 1) or 1)
                processed_blocks += blocks_delta
                if run_stats:
                    run_stats.update_blocks(
                        blocks_processed=processed_blocks,
                        total_blocks=total_blocks,
                        items_in_chunk=progress.get("items", 0),
                        comparisons_in_chunk=progress.get("comparisons", 0),
                        progress_q=progress_q,
                    )
                _emit_progress(
                    progress.get("block_key"),
                    progress.get("items", 0),
                    progress.get("comparisons", 0),
                    progress.get("matches", 0),
                    blocks_delta=blocks_delta,
                )
            else:
                processed_blocks += 1
                if run_stats:
                    run_stats.update_blocks(
                        blocks_processed=processed_blocks,
                        total_blocks=total_blocks,
                        items_in_chunk=0,
                        comparisons_in_chunk=0,
                        progress_q=progress_q,
                    )
                _emit_progress(None, 0, 0, 0, blocks_delta=1)

    _emit_progress(None, 0, 0, 0, blocks_delta=0, flush=True)
    log.info(f"Parallel processing complete. Total matches: {len(all_matches)}")
    return all_matches


def _match_in_parallel_mm(
    cfg: MatchConfig,
    src_df: pd.DataFrame,
    ref_df: pd.DataFrame,
    all_blocks: list[dict],
    run_stats: Optional[RunStats],
    progress_q: Optional[mp.Queue],
) -> list[dict]:
    """
    Distribute block matching using memory-mapped data sharing.
    Only sends row indices (lists of ints) to workers - no DataFrame pickling!
    """
    num_workers = min(getattr(cfg, "max_workers", CDIST_WORKERS), CDIST_WORKERS)
    if not all_blocks:
        return []

    # NEW: Adaptive worker scaling
    if len(all_blocks) < 50:
        log.info(f"Only {len(all_blocks)} blocks - using sequential processing")
        # Sequential fallback: initialize globals for _process_one_key.
        # This avoids silent empty results when MM is chosen but block-count is small.
        _pool_init(src_df, ref_df, cfg)
        all_matches = []
        processed_blocks = 0
        for block_info in all_blocks:
            result = _process_one_key(block_info)
            if isinstance(result, tuple):
                matches, progress = result
            else:
                matches, progress = result, None

            if matches:
                all_matches.extend(matches)

            if progress:
                blocks_delta = int(progress.get("blocks", 1) or 1)
                processed_blocks += blocks_delta
                if run_stats:
                    run_stats.update_blocks(
                        blocks_processed=processed_blocks,
                        total_blocks=len(all_blocks),
                        items_in_chunk=progress.get("items", 0),
                        comparisons_in_chunk=progress.get("comparisons", 0),
                        progress_q=progress_q,
                    )
                _emit_progress(
                    progress.get("block_key"),
                    progress.get("items", 0),
                    progress.get("comparisons", 0),
                    progress.get("matches", 0),
                    blocks_delta=blocks_delta,
                )
            else:
                processed_blocks += 1
                if run_stats:
                    run_stats.update_blocks(
                        blocks_processed=processed_blocks,
                        total_blocks=len(all_blocks),
                        items_in_chunk=block_info.get("src_len", 0),
                        comparisons_in_chunk=block_info.get("metric", 0),
                        progress_q=progress_q,
                    )
                _emit_progress(
                    block_info.get("key"),
                    block_info.get("src_len", 0),
                    block_info.get("metric", 0),
                    0,
                )

        _emit_progress(None, 0, 0, 0, blocks_delta=0, flush=True)
        return all_matches

    # Scale workers based on block count
    if len(all_blocks) < 100:
        num_workers = min(2, num_workers)
    elif len(all_blocks) < 500:
        num_workers = min(4, num_workers)

    # Stage data to memory-mapped files once
    log.info("Staging data for memory-mapped access...")
    src_temp = tempfile.NamedTemporaryFile(delete=False, suffix=".feather")
    ref_temp = tempfile.NamedTemporaryFile(delete=False, suffix=".feather")
    src_temp.close()
    ref_temp.close()

    try:
        # CRITICAL: Reset indices to ensure they match Arrow row positions
        src_df_mm = src_df.reset_index(drop=True)
        ref_df_mm = ref_df.reset_index(drop=True)

        # Write the RESET DataFrames to feather format WITHOUT compression for faster memory-mapping
        src_table = pa.Table.from_pandas(src_df_mm, preserve_index=False)
        ref_table = pa.Table.from_pandas(ref_df_mm, preserve_index=False)
        feather.write_feather(
            src_table, src_temp.name, compression=None
        )  # NO COMPRESSION
        feather.write_feather(
            ref_table, ref_temp.name, compression=None
        )  # NO COMPRESSION

        # Pre-compute row indices from the SAME reset DataFrames
        log.info("Pre-computing block indices...")
        src_idx_map = src_df_mm.groupby(_ACTIVE_BLOCK_KEY_, observed=True).indices
        ref_idx_map = ref_df_mm.groupby(_ACTIVE_BLOCK_KEY_, observed=True).indices

        # Generate lightweight tasks (just indices)
        def task_generator():
            for block_info in all_blocks:
                is_soft_widen = block_info.get("is_soft_widen", False)
                src_key = (
                    block_info.get("src_key")
                    if is_soft_widen
                    else block_info.get("key")
                )
                ref_key = (
                    block_info.get("ref_key")
                    if is_soft_widen
                    else block_info.get("key")
                )

                # Get indices for this block
                s_indices = src_idx_map.get(src_key)
                r_indices = ref_idx_map.get(ref_key)

                if s_indices is not None and r_indices is not None:
                    # Convert to list for pickling (small!)
                    yield (s_indices.tolist(), r_indices.tolist(), block_info)

        all_matches = []
        processed_blocks = 0
        chunksize = max(1, min(32, len(all_blocks) // (num_workers * 4)))

        log.info(
            f"Starting parallel processing: {len(all_blocks)} blocks across {num_workers} workers"
        )

        if run_stats:
            run_stats.emit_heartbeat(
                f"Initializing {num_workers} workers with memory-mapped data..."
            )

        # Create pool with memory-mapped initialization
        with mp.Pool(
            processes=num_workers,
            initializer=_pool_init_mm,
            initargs=(src_temp.name, ref_temp.name, cfg),
        ) as pool:
            if run_stats:
                run_stats.emit_heartbeat(f"Processing {len(all_blocks)} blocks...")

            # Stream tasks without materializing the full list
            for result in pool.imap_unordered(
                _process_one_key_mm, task_generator(), chunksize=chunksize
            ):
                if isinstance(result, tuple):
                    matches, progress = result
                else:
                    matches, progress = result, None

                if matches:
                    all_matches.extend(matches)

                if progress:
                    blocks_delta = int(progress.get("blocks", 1) or 1)
                    processed_blocks += blocks_delta
                    if run_stats:
                        run_stats.update_blocks(
                            blocks_processed=processed_blocks,
                            total_blocks=len(all_blocks),
                            items_in_chunk=progress.get("items", 0),
                            comparisons_in_chunk=progress.get("comparisons", 0),
                            progress_q=progress_q,
                        )
                    _emit_progress(
                        progress.get("block_key"),
                        progress.get("items", 0),
                        progress.get("comparisons", 0),
                        progress.get("matches", 0),
                        blocks_delta=blocks_delta,
                    )
                else:
                    processed_blocks += 1
                    if run_stats:
                        run_stats.update_blocks(
                            blocks_processed=processed_blocks,
                            total_blocks=len(all_blocks),
                            items_in_chunk=0,
                            comparisons_in_chunk=0,
                            progress_q=progress_q,
                        )
                    _emit_progress(None, 0, 0, 0, blocks_delta=1)

        _emit_progress(None, 0, 0, 0, blocks_delta=0, flush=True)
        log.info(f"Parallel processing complete. Total matches: {len(all_matches)}")
        return all_matches

    finally:
        # FIX: Better Windows cleanup with garbage collection
        import gc

        # Force garbage collection to release memory-mapped references
        gc.collect()
        time.sleep(0.5)  # Give Windows time to release handles

        for filepath in [src_temp.name, ref_temp.name]:
            if os.path.exists(filepath):
                for attempt in range(10):  # More attempts
                    try:
                        os.remove(filepath)
                        break
                    except (PermissionError, OSError) as e:
                        if attempt == 9:
                            log.warning(
                                f"Failed to clean up {filepath} after 10 attempts: {e}"
                            )
                        else:
                            time.sleep(0.5)  # Longer wait between attempts


# Feature flag for algorithm fitness
ENABLE_ALGORITHM_FITNESS = False  # Will be enabled via config


@dataclass
class EngineConfig:
    """Configuration for engine behavior."""

    enable_exotic_detection: bool = True
    enable_algorithm_fitness: bool = True
    enable_composite_keys: bool = False
    max_composite_columns: int = 2
    exotic_detection_sample_size: int = 512
    algorithm_fitness_sample_size: int = 200
    b2b_mode: bool = True


# Global config instance
ENGINE_CONFIG = EngineConfig()


def configure_engine(**kwargs):
    """Update engine configuration."""
    global ENGINE_CONFIG
    for key, value in kwargs.items():
        if hasattr(ENGINE_CONFIG, key):
            setattr(ENGINE_CONFIG, key, value)

    # Update feature flags
    global ENABLE_ALGORITHM_FITNESS
    ENABLE_ALGORITHM_FITNESS = ENGINE_CONFIG.enable_algorithm_fitness

    log.info(f"Engine configured: {ENGINE_CONFIG}")


def _quick_algo_fitness(
    sample_pairs: list[tuple[str, str]], algorithm: str, column_profile: dict
) -> float:
    """
    Quick fitness check for an algorithm on sample pairs.
    Returns a normalized score [0, 1].
    """
    if not sample_pairs or not algorithm:
        return 0.5  # Neutral score

    # Get the algorithm function
    algo_func = ALGORITHM_MAP.get(algorithm)
    if not algo_func:
        return 0.5

    # Special handling for certain algorithms based on data profile
    script = column_profile.get("script", "unknown")
    semantic_type = column_profile.get("semantic_type")
    avg_length = column_profile.get("avg_length", 10)

    # Quick heuristic rules (will be replaced by YAML rules later)
    if algorithm == "Exact" and semantic_type == "product_code":
        return 0.9  # Excellent fit

    if algorithm in ["WRatio", "TokenSetRatio"] and avg_length < 10:
        return 0.3  # Poor fit for short strings

    # Calculate separation power
    scores = []
    for s1, s2 in sample_pairs[:100]:  # Limit to 100 for speed
        try:
            score = algo_func(s1, s2)
            scores.append(score)
        except:
            scores.append(0)

    if not scores:
        return 0.5

    # Calculate fitness based on score distribution
    scores_array = np.array(scores)

    # Good algorithms should have:
    # 1. High variance (good discrimination)
    # 2. Not all scores at extremes
    # 3. Reasonable mean

    variance = np.var(scores_array)
    mean_score = np.mean(scores_array)

    # Normalize variance (0-2500 is typical range)
    variance_score = min(variance / 2500, 1.0)

    # Penalize if all scores are very high or very low
    extreme_ratio = np.sum((scores_array < 10) | (scores_array > 90)) / len(
        scores_array
    )
    distribution_score = 1.0 - extreme_ratio

    # Combine metrics
    fitness = variance_score * 0.5 + distribution_score * 0.5

    # Apply semantic adjustments
    if semantic_type == "email" and algorithm in ["JaroWinkler", "Jaro"]:
        fitness *= 1.2  # Boost for good fit
    elif semantic_type == "product_code" and algorithm == "WRatio":
        fitness *= 0.7  # Penalty for poor fit

    return min(max(fitness, 0.0), 1.0)


def _quick_column_quality_score(series: pd.Series) -> float:
    """Quick heuristic for column suitability (0..1)."""
    if series is None or series.empty:
        return 0.0

    non_null = series.dropna()
    if non_null.empty:
        return 0.0

    non_null_ratio = len(non_null) / len(series)
    unique_ratio = non_null.nunique() / max(len(non_null), 1)

    return max(0.0, min(1.0, 0.5 * non_null_ratio + 0.5 * unique_ratio))


def _sample_block_pairs(
    src_keys: pd.Series,
    ref_keys: Optional[pd.Series],
    src_values: pd.Series,  # NEW: pass the actual column values
    ref_values: Optional[pd.Series] = None,  # NEW
    n: int = 100,
) -> list[tuple[str, str]]:
    """Sample pairs from blocking keys for algorithm fitness evaluation."""
    pairs = []

    if ref_keys is None:
        # Dedupe mode - sample from same series
        if len(src_keys) < 2:
            return []

        # Get some similar pairs (same block)
        block_counts = src_keys.value_counts()
        for block, count in block_counts.head(10).items():
            if count > 1 and str(block).strip():  # Skip empty blocks
                block_indices = src_keys[src_keys == block].index[: min(count, 5)]
                for i in range(len(block_indices)):
                    for j in range(i + 1, len(block_indices)):
                        # Get actual values, not indices!
                        val_i = src_values.loc[block_indices[i]]
                        val_j = src_values.loc[block_indices[j]]
                        pairs.append((str(val_i), str(val_j)))

    else:
        # Match mode - sample across series
        common_blocks = set(src_keys.unique()) & set(ref_keys.unique())
        for block in list(common_blocks)[:10]:
            if str(block).strip():  # Skip empty blocks
                src_indices = src_keys[src_keys == block].index[:5]
                ref_indices = ref_keys[ref_keys == block].index[:5]
                for s_idx in src_indices:
                    for r_idx in ref_indices:
                        # Get actual values!
                        val_s = src_values.loc[s_idx]
                        val_r = ref_values.loc[r_idx]
                        pairs.append((str(val_s), str(val_r)))

    return pairs[:n]


# Ã¢â€â‚¬Ã¢â€â‚¬ local Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬

# --- Moved DEFAULT_NORMALIZATION_RULES and _collapse_ws earlier to resolve import errors ---
_RE_COLLAPSE_WS = re.compile(r"\s+")
CLEANING_REGEX_COMPILED = re.compile(r"[^0-9a-z]+")


def _collapse_ws(text: str) -> str:
    """Collapse runs of whitespace into a single space."""
    return _RE_COLLAPSE_WS.sub(" ", text.strip())


DEFAULT_NORMALIZATION_RULES: dict[str, callable] = {
    "lower": str.lower,
    "strip": str.strip,
    "collapse_ws": _collapse_ws,
}
# --- End moved block ---

# --- Pre-compiled Regex for Preprocessing Variants ---
_RE_EMAIL_DOMAIN = re.compile(r"@([^@\s]+)$")
_RE_PHONE_DIGITS = re.compile(r"[^\d]")
_RE_PHONE_US_CLEAN = re.compile(r"^1(\d{10})$")
_RE_URL_PROTOCOL = re.compile(r"^https?://")
_RE_URL_WWW = re.compile(r"^www\.")
_RE_URL_DOMAIN = re.compile(r"^([^/]+)")
_RE_URL_TLD = re.compile(r"\.[^.]+$")
_RE_COMPANY_SUFFIXES = re.compile(
    r"\b(incorporated|inc|llc|ltd|limited|corp|corporation|company|co|"
    r"group|holdings|international|intl|plc|gmbh|sa|ag|nv|bv|pty|"
    r"partners|partnership|lp|llp|pllc|pc|pa|sc|"
    r"pvt|private|public|"
    r"associates|consulting|consultants|services|solutions|"
    r"tech|technologies|technology|labs|laboratory|systems|software|"
    r"global|worldwide|usa|america|europe|asia|"
    r"financial|finance|capital|investments|advisors|"
    r"medical|healthcare|health|pharma|"
    r"media|digital|interactive|creative|design|"
    r"retail|wholesale|distribution|supply|"
    r"research|development|innovations|"
    r"the|and|&|of)\b\.?",
    re.IGNORECASE,
)

# Moved from embedded in the file
_RE_PUNCT = re.compile(r"[^\w\s]")
_RE_FUZZY_CLEAN = re.compile(r"[\s/_&-]+")
PHASE3_VECTORIZED = True
_NAME_PREFIX_KEYWORDS = [
    "company",
    "account",
    "name",
    "organization",
    "organisation",
    "business",
    "vendor",
    "supplier",
    "customer",
    "client",
    "partner",
    "firm",
    "brand",
]
_NAME_PREFIX_ALIAS_SPLIT = re.compile(
    r"\b(?:aka|fka|dba|formerly known as)\b", re.IGNORECASE
)
_NAME_PREFIX_SECOND_TOKEN_LEN = 2
_NAME_PREFIX_BUCKET_TARGET = 100
_NAME_PREFIX_BUCKET_RANGE = (50, 200)
_NAME_PREFIX_STOPWORDS = {"and", "the"}


def _strip_company_alias(text: str) -> str:
    if not text:
        return ""
    parts = _NAME_PREFIX_ALIAS_SPLIT.split(text)
    return parts[0] if parts else text


def _normalize_company_tokens(raw: Any) -> list[str]:
    if raw is None:
        return []
    if not isinstance(raw, str):
        raw = str(raw)
    raw = raw.strip()
    if not raw:
        return []
    if EncodingFixer is not None:
        normalized = EncodingFixer.safe_unicode_normalize(raw)
    else:
        # Optional dependency: keep behavior deterministic without crashing.
        normalized = _ud.normalize("NFKC", raw)
    normalized = _strip_company_alias(normalized)
    normalized = _ud.normalize("NFKD", normalized)
    normalized = "".join(ch for ch in normalized if not _ud.combining(ch))
    normalized = normalized.lower().replace("&", " and ")
    normalized = _RE_COMPANY_SUFFIXES.sub(" ", normalized)
    normalized = _RE_PUNCT.sub(" ", normalized)
    tokens = [
        tok for tok in normalized.split() if tok and tok not in _NAME_PREFIX_STOPWORDS
    ]
    return tokens


def _render_name_prefix(
    tokens: list[str], prefix_len: int, second_len: int = _NAME_PREFIX_SECOND_TOKEN_LEN
) -> str:
    if not tokens:
        return ""
    first = tokens[0][:prefix_len]
    second = tokens[1][:second_len] if len(tokens) > 1 and second_len > 0 else ""
    return f"{first}{second}".strip()


def _build_name_prefix_keys(
    series: pd.Series, prefix_len: int, second_len: int = _NAME_PREFIX_SECOND_TOKEN_LEN
) -> pd.Series:
    tokens = series.fillna("").map(_normalize_company_tokens)
    keys = tokens.map(lambda toks: _render_name_prefix(toks, prefix_len, second_len))
    return keys.astype("string", copy=False)


# Domain caching optimization


@lru_cache(maxsize=100_000)
def _cached_domain_clean(domain: str) -> Optional[str]:
    """
    Cached domain cleaning to avoid repeated regex and Bloom filter lookups.
    This correctly calls the single-value cleaning function.
    """
    return cleanse_single_domain(domain)


def clean_domain_with_cache(value: Any) -> Optional[str]:
    """Type-safe wrapper for domain cleaning that leverages an LRU cache."""
    if not isinstance(value, str) or not value:
        return None
    return _cached_domain_clean(value)


# +++ ADD THIS FUNCTION DEFINITION HERE +++
def detect_encoding(path: str | Path, read_bytes: int = 16_384) -> str:
    """
    Reads a chunk from the file, asks chardet to guess the codec,
    and falls back to 'utf-8'. Kept for GUI compatibility.
    """
    # Ensure chardet is imported at the top of engine.py: import chardet
    # Ensure Path is imported from pathlib: from pathlib import Path
    # Ensure log is defined (e.g., log = logging.getLogger(__name__))
    try:
        with open(path, "rb") as fh:  # type: ignore
            raw = fh.read(read_bytes)

        if CHARDET_AVAILABLE and chardet is not None and raw:
            guess = chardet.detect(raw)  # type: ignore
            enc = (guess or {}).get("encoding") or "utf-8"
        else:
            enc = "utf-8"

        if enc.lower() == "ascii":  # ASCII is a subset of UTF-8
            return "utf-8"
        return enc.lower()
    except Exception as exc:
        log.warning(
            "detect_encoding(%s) failed Ã¢â‚¬â€œ defaulting to utf-8: %s", path, exc
        )
        return "utf-8"


# +++ END OF detect_encoding FUNCTION +++
def validate_dataframe(
    df: pd.DataFrame, required_cols: list[str] | None = None
) -> pd.DataFrame:
    """
    Legacy no-op validator kept for GUI back-compat.
    Warns if required columns are missing.
    """
    if required_cols:
        # Check actual columns first
        missing = [c for c in required_cols if c not in df.columns]
        if missing:
            log.warning(
                "validate_dataframe: frame is missing expected columns %s - proceeding anyway.",
                missing,
            )
        # Check for columns potentially used as index (sometimes causes issues)
        if isinstance(df.index, pd.MultiIndex):
            index_cols = df.index.names
        else:
            index_cols = [df.index.name]
        missing_in_index = [
            c for c in required_cols if c in index_cols and c not in df.columns
        ]
        if missing_in_index:
            log.warning(
                "validate_dataframe: frame might be using expected columns %s as index - proceeding anyway.",
                missing_in_index,
            )
    return df


# Ã¢â€â‚¬Ã¢â€â‚¬ local Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
from fmatch.preprocessing.core.column_detector import (
    ColumnDetector,
)  # <-- ADD THIS LINE

# Add after other imports
try:
    from fmatch.core.telemetry import TelemetryCollector

    _TELEMETRY_AVAILABLE = True
except ImportError:
    _TELEMETRY_AVAILABLE = False
    TelemetryCollector = None

# Initialize global telemetry collector
_TELEMETRY = TelemetryCollector() if _TELEMETRY_AVAILABLE else None


# --------------------------------------------------------------------------- #
# Globals & constants
# --------------------------------------------------------------------------- #

# Runtime-aware or fallback configuration
policy = getattr(runtime_manager, "policy", None) if runtime_manager else None

if RUNTIME_AWARE and runtime_manager and policy:
    # Use runtime manager for all resource decisions
    max_workers = policy.max_workers or max(os.cpu_count() - 1, 1)
    rf_threads = policy.rf_threads_per_proc or 1
    # NOTE: mm_* thresholds can be None to explicitly disable MM in some runtimes
    mm_rows_threshold = policy.mm_rows_threshold
    mm_ops_threshold = policy.mm_ops_threshold

    CPU_LIMIT = max_workers
    CDIST_WORKERS = max_workers
    _RF_WORKERS = rf_threads
    SPEED_MAX_ROWS = policy.speed_max_rows or 1_000_000
    # Used only in legacy/non-runtime-aware paths; keep it defined for logging.
    ADAPTIVE_PARALLEL_THRESHOLD = (
        int(mm_rows_threshold // 3)
        if isinstance(mm_rows_threshold, int) and mm_rows_threshold > 0
        else 500_000
    )
    PARALLEL_OPS_THRESHOLD = 400_000  # Keep as is
    MEMORY_MAPPED_OPS_THRESHOLD = (
        int(mm_ops_threshold)
        if isinstance(mm_ops_threshold, int) and mm_ops_threshold > 0
        else 50_000_000
    )

    mm_threshold_display = (
        f"{mm_rows_threshold:,}"
        if isinstance(mm_rows_threshold, int) and mm_rows_threshold > 0
        else "disabled"
    )

    log.info(
        "[engine] Using runtime policy: workers=%s, rf_threads=%s, speed_max=%s, "
        "mm_threshold=%s",
        CDIST_WORKERS,
        _RF_WORKERS,
        f"{SPEED_MAX_ROWS:,}",
        mm_threshold_display,
    )
else:
    # Fallback to original logic
    CPU_LIMIT = int(os.getenv("FM_CPU_LIMIT", str(os.cpu_count() - 1)))
    CDIST_WORKERS = _auto_worker_count()

    # RapidFuzz threading configuration
    _RF_WORKERS = _rf_threads(CDIST_WORKERS)
    SPEED_MAX_ROWS = int(os.getenv("FM_SPEED_MAX_ROWS", "1_000_000"))
    # Performance thresholds - defaults (can be overridden via env)
    ADAPTIVE_PARALLEL_THRESHOLD = int(os.getenv("FM_ADAPTIVE_PARALLEL_THRESHOLD", "500_000"))
    PARALLEL_OPS_THRESHOLD = int(os.getenv("FM_PARALLEL_OPS_THRESHOLD", "400_000"))
    # Align with engine_runtime policy/env names where possible
    MEMORY_MAPPED_OPS_THRESHOLD = int(os.getenv("FM_MM_OPS_THRESHOLD", "50_000_000"))
RAPIDFUZZ_THREAD_COUNT = _RF_WORKERS  # For logging only

# CRITICAL FIX: Actually set the environment variable that RapidFuzz reads
os.environ["RAPIDFUZZ_THREAD_COUNT"] = str(_RF_WORKERS)
log.info(f"Set RAPIDFUZZ_THREAD_COUNT environment variable to {_RF_WORKERS}")

# Diagnostic tracing (off by default)
TRACE_BLOCKS = os.getenv("FM_TRACE_BLOCKS", "0") == "1"
TRACE_CDIST = os.getenv("FM_TRACE_CDIST", "0") == "1"

_trace_block_counter = 0


def _trace_block(msg: str, elapsed: float = None):
    """Trace block operations. Only logs slow blocks or every 100th block to reduce noise."""
    if not TRACE_BLOCKS:
        return

    global _trace_block_counter
    _trace_block_counter += 1

    # Log slow blocks or sample periodically
    if elapsed and elapsed > 1.0:  # Always log slow blocks
        log.info(f"[TRACE-BLOCK-SLOW] {msg}")
    elif _trace_block_counter % 100 == 0:  # Log every 100th block
        log.info(f"[TRACE-BLOCK] {msg}")


def _trace_cdist(msg: str):
    if TRACE_CDIST:
        log.info(f"[TRACE-CDIST] {msg}")


_PROGRESS_MIN_INTERVAL = 0.02
_progress_last_emit = 0.0
_progress_accum = {
    "blocks": 0,
    "items": 0,
    "comparisons": 0,
    "matches": 0,
    "last_block": None,
}
_LAST_DROP_LOG = 0.0
_DROP_LOG_INTERVAL = 60.0  # seconds


def _emit_progress(
    block_key,
    items_delta,
    comparisons_delta,
    matches_delta,
    blocks_delta=1,
    flush=False,
):
    """Emit progress events with throttling and batching."""
    q = getattr(_GLOBAL_CONFIG, "progress_q", None)
    if not q:
        return

    global _progress_last_emit, _progress_accum

    acc = _progress_accum
    acc["blocks"] += int(blocks_delta or 0)
    acc["items"] += int(items_delta or 0)
    acc["comparisons"] += int(comparisons_delta or 0)
    acc["matches"] += int(matches_delta or 0)
    if block_key is not None:
        acc["last_block"] = block_key

    now = time.perf_counter()
    if not flush and (now - _progress_last_emit) < _PROGRESS_MIN_INTERVAL:
        return

    if (
        acc["blocks"] <= 0
        and acc["items"] <= 0
        and acc["comparisons"] <= 0
        and acc["matches"] <= 0
    ):
        if flush:
            _progress_last_emit = now
        return

    payload = {
        "type": "run",
        "block_key": str(acc["last_block"]) if acc["last_block"] is not None else None,
        "items_delta": int(acc["items"]),
        "comparisons_delta": int(acc["comparisons"]),
        "matches_delta": int(acc["matches"]),
        "blocks_delta": int(acc["blocks"]),
        "message": None,
    }
    try:
        q.put_nowait(payload)
    except Exception:
        global _LAST_DROP_LOG
        now2 = time.perf_counter()
        if now2 - _LAST_DROP_LOG >= _DROP_LOG_INTERVAL:
            log.debug("progress queue full; dropping event")
            _LAST_DROP_LOG = now2

    _progress_accum = {
        "blocks": 0,
        "items": 0,
        "comparisons": 0,
        "matches": 0,
        "last_block": None,
    }
    _progress_last_emit = now


def _get_score_cutoff(mapping: dict, threshold: float) -> int:
    """Calculate appropriate cutoff for early termination - conservative to avoid breaking coverage gate."""
    t = max(0.0, float(threshold or 0))
    col = (mapping.get("source") or "").lower()
    is_domain = "domain" in col or "website" in col

    # Check if high weight (protect recall)
    if mapping.get("weight", 1.0) >= 2.0:
        return 0  # Disable cutoff for high-weight mappings

    # Conservative cutoffs to avoid breaking coverage gate
    if is_domain:
        return int(max(0, min(t - 10, 80)))

    algo = (mapping.get("preferred_algo") or "WRatio").lower()
    if "exact" in algo:
        return int(max(0, min(t - 5, 85)))
    if "jaro" in algo:
        return int(max(70, min(100, t - 20)))
    # WRatio, TokenSet, etc.
    return int(max(60, min(100, t - 25)))


# Add warning for high-core, low-memory systems
if CDIST_WORKERS > 8 and psutil and not _WARNED_HIGH_WORKERS:
    available_gb = psutil.virtual_memory().available / (1024**3)
    if available_gb < 16:
        log.warning(
            f"High worker count ({CDIST_WORKERS}) with {available_gb:.1f}GB available RAM. Consider FM_CPU_LIMIT=4"
        )
        _WARNED_HIGH_WORKERS = True


DEFAULT_BLOCK_COL = (
    "block_key"  # Keep as a fallback name IF needed, but _ACTIVE_BLOCK_KEY_ is primary
)
_ACTIVE_BLOCK_KEY_ = "_active_block_key_"
_RULESET_LOCK = threading.Lock()
AVAILABLE_RULESETS: dict[str, dict] = {}  # type: ignore
DEFAULT_ALGORITHM_KEY = "WRatio"
DEFAULT_THRESHOLD = 80
DEFAULT_DUP_BLOCK_LIMIT = 2000
DELTA_THRESHOLD_PROGRESS_Q = 5000  # Or 500, choose a new value
SMALL_BLOCK_DEDUPE_PAIRS_THRESHOLD = (
    200  # Force more blocks up to ~60x60 to use the fast path
)
SMALL_BLOCK_OPS_THRESHOLD = 10000  # Threshold for using fast path
# Soft guardrails: blocks above these limits are sharded/sampled, not skipped.
MAX_BLOCK_SIZE = 5000
MAX_BLOCK_OPERATIONS = 500_000


def _should_skip_block(
    block_key_val: str,
    num_src: int,
    num_ref: int,
    *,
    max_size: int,
    max_ops: int,
    is_dedup: bool = False,
) -> bool:
    """Return True only for pathological block sizes that should be dropped outright."""
    if num_src <= 0 or num_ref <= 0:
        return True

    # Keep skip behavior for extreme outliers only.
    guard = BlockGuardrails()
    hard_max_rows = max(int(max_size or 0), int(guard.max_block_rows * 2))
    hard_max_ops = max(int(max_ops or 0), int(guard.max_pairs * 10))

    if num_src > hard_max_rows or num_ref > hard_max_rows:
        log.warning(
            f"Block '{block_key_val}' oversized ({num_src}x{num_ref}). Skipping."
        )
        return True

    estimated_ops = (
        (num_src * (num_src - 1)) // 2 if is_dedup else (num_src * num_ref)
    )
    if estimated_ops > hard_max_ops:
        log.warning(
            f"Block '{block_key_val}' too large ({estimated_ops:,} ops). Skipping."
        )
        return True

    return False

# --------------------------------------------------------------------
#  Temporary stub so functions defined below can compile
#  It will be overwritten after `reload_rulesets()` runs later.
# --------------------------------------------------------------------
DEFAULT_RULESET_NAME: str = "default"
_WORKER_DB_CONN_CACHE = (
    threading.local()
)  # <<< Use threading.local for worker connections
_RULESET_CACHE = {}

# at top of file
_INITIALISED = False

# CDIST Values
# Threshold for dedupe small blocks (N*(N-1)/2 comparisons)


# --------------------------------------------------------------------------- #
#  Automatic speed-mode detection
# --------------------------------------------------------------------------- #
# Performance thresholds are configured above (runtime policy or env defaults).
# Do not overwrite them here, or runtime-aware settings become dead code.
MIN_BLOCK_SIZE = 2  # Skip singleton blocks
SPEED_RAM_FRACTION = 0.70  # leave at least 30 % of free RAM untouched

# Slow block logging control
SLOW_BLOCK_SAMPLE_RATE = 0.2  # Log 1 in 5 slow blocks
SLOW_BLOCK_MIN_OPS = 5000  # Only warn for blocks with meaningful work
SLOW_BLOCK_THROUGHPUT_THRESHOLD = 1.0  # M ops/s threshold

# ----------------------------------------------------------------------
# ensure __all__ exists before we start extending it
# ----------------------------------------------------------------------
if "__all__" not in globals():
    __all__: list[str] = []


class BlockingWeightProfile:
    """Configurable weight profiles for different data matching scenarios."""

    PROFILES = {
        "high_precision": {
            "name": "High Precision (B2B Sales)",
            "description": "Minimize false positives, ideal for sales outreach",
            "weights": {
                "rr": 0.05,  # Less emphasis on reduction
                "h": 0.15,  # Moderate entropy
                "g": 0.40,  # High emphasis on balanced distribution
                "p1": 0.30,  # Strong penalty for large blocks
                "s1": 0.05,  # Low singleton penalty
                "k": 0.05,  # Key length less important
                "af": 0.00,  # Algorithm fitness (disabled by default)
            },
        },
        "high_recall": {
            "name": "High Recall (Data Enrichment)",
            "description": "Catch all possible matches, ideal for enrichment",
            "weights": {
                "rr": 0.20,  # More emphasis on reduction
                "h": 0.25,  # Higher entropy weight
                "g": 0.20,  # Less emphasis on balance
                "p1": 0.20,  # Lower penalty for large blocks
                "s1": 0.10,  # Moderate singleton penalty
                "k": 0.05,
                "af": 0.00,  # Algorithm fitness (disabled by default)
            },
        },
        "balanced": {
            "name": "Balanced (General Purpose)",
            "description": "Good for most use cases",
            "weights": {
                "rr": 0.09,
                "h": 0.18,
                "g": 0.27,
                "p1": 0.225,
                "s1": 0.09,
                "k": 0.045,
                "af": 0.10,  # Algorithm fitness (enabled by default)
            },
        },
        "messy_data": {
            "name": "Messy Data (Data Cleansing)",
            "description": "For inconsistent, dirty data",
            "weights": {
                "rr": 0.15,  # Moderate reduction
                "h": 0.30,  # High entropy (find diverse blocks)
                "g": 0.25,  # Good distribution
                "p1": 0.20,  # Some large block tolerance
                "s1": 0.05,  # Few singletons expected
                "k": 0.05,
                "af": 0.00,  # Algorithm fitness (disabled by default)
            },
        },
        "exotic_aware": {
            "name": "Exotic Data Aware",
            "description": "For non-Western or mixed-script data",
            "weights": {
                "rr": 0.10,
                "h": 0.15,
                "g": 0.25,
                "p1": 0.20,
                "s1": 0.10,
                "k": 0.05,
                "af": 0.15,  # Algorithm fitness enabled
            },
        },
        "custom": {
            "name": "Custom",
            "description": "User-defined weights",
            "weights": {
                "rr": 0.10,
                "h": 0.20,
                "g": 0.30,
                "p1": 0.25,
                "s1": 0.10,
                "k": 0.05,
                "af": 0.00,  # Algorithm fitness (disabled by default)
            },
        },
    }

    @classmethod
    def get_weights(cls, profile_name: str) -> dict:
        """Get weights for a specific profile."""
        profile = cls.PROFILES.get(profile_name, cls.PROFILES["balanced"])
        return profile["weights"]

    @classmethod
    def validate_weights(cls, weights: dict) -> tuple[bool, str]:
        """Validate custom weights."""
        required_keys = {"rr", "h", "g", "p1", "s1", "k", "af"}

        # Check all keys present
        if set(weights.keys()) != required_keys:
            missing = required_keys - set(weights.keys())
            extra = set(weights.keys()) - required_keys
            msg = []
            if missing:
                msg.append(f"Missing: {missing}")
            if extra:
                msg.append(f"Extra: {extra}")
            return False, "; ".join(msg)

        # Check all values are numbers
        for k, v in weights.items():
            if not isinstance(v, (int, float)):
                return False, f"Non-numeric value for {k}: {v}"
            if not 0 <= v <= 1:
                return False, f"Value for {k} outside [0,1]: {v}"

        # Check sum is reasonable (0.8 to 1.2)
        total = sum(weights.values())
        if not 0.8 <= total <= 1.2:
            return False, f"Weights sum to {total:.2f}, should be ~1.0"

        return True, "Valid"


class MemoryManager:
    def __init__(self):
        if psutil is not None:
            self.available_memory = psutil.virtual_memory().available
        else:
            self.available_memory = int(
                os.getenv("FM_AVAILABLE_MEM_BYTES", "2000000000")
            )
        self.safety_factor = getattr(self, "safety_factor", 0.7)

    def calculate_safe_chunk_size(self, row_size_bytes: int) -> int:
        safe_memory = self.available_memory * self.safety_factor
        return int(safe_memory / row_size_bytes)

    def should_use_chunking(self, estimated_bytes):
        """Determine if chunking is needed based on memory requirements"""
        # Simple implementation for now
        return estimated_bytes > self.available_memory * 0.8


class BlockingAnalysisMonitor:
    """Enhanced progress monitoring for blocking analysis."""

    def __init__(self, total_strategies: int, callback=None):
        self.total = total_strategies
        self.current = 0
        self.start_time = time.perf_counter()
        self.best_score = -1
        self.best_strategy = None
        self.callback = callback
        self.strategies_evaluated = []

    def update(self, strategy_info: dict, score: float):
        """Update progress with strategy evaluation results."""
        self.current += 1
        elapsed = time.perf_counter() - self.start_time

        # Track strategy
        self.strategies_evaluated.append(
            {"strategy": strategy_info, "score": score, "timestamp": elapsed}
        )

        # Update best
        if score > self.best_score:
            self.best_score = score
            self.best_strategy = strategy_info.copy()
            log.info(
                f"New best strategy: {strategy_info['src_col']} "
                f"({strategy_info['preproc']}, k={strategy_info['k_len']}) "
                f"score={score:.3f}"
            )

        # Calculate metrics
        rate = self.current / elapsed if elapsed > 0 else 0
        eta = (self.total - self.current) / rate if rate > 0 else 0

        # Progress info
        progress_data = {
            "current": self.current,
            "total": self.total,
            "percent": (self.current / self.total) * 100,
            "elapsed": elapsed,
            "eta": eta,
            "rate": rate,
            "best_score": self.best_score,
            "best_strategy": self.best_strategy,
            "current_strategy": strategy_info,
            "current_score": score,
        }

        # Log periodically
        if self.current % 10 == 0 or score > self.best_score:
            log.info(
                f"Blocking analysis: {self.current}/{self.total} "
                f"({progress_data['percent']:.1f}%) "
                f"Rate: {rate:.1f}/sec, ETA: {eta:.1f}s, "
                f"Best: {self.best_score:.3f}"
            )

        # Call callback if provided
        if self.callback:
            self.callback(progress_data)

    def get_summary(self) -> dict:
        """Get analysis summary."""
        elapsed = time.perf_counter() - self.start_time

        # Score distribution
        scores = [s["score"] for s in self.strategies_evaluated]

        return {
            "total_evaluated": self.current,
            "elapsed_time": elapsed,
            "best_strategy": self.best_strategy,
            "best_score": self.best_score,
            "score_distribution": {
                "min": min(scores) if scores else 0,
                "max": max(scores) if scores else 0,
                "mean": sum(scores) / len(scores) if scores else 0,
                "std": np.std(scores) if scores else 0,
            },
            "strategies_by_score": sorted(
                self.strategies_evaluated, key=lambda x: x["score"], reverse=True
            )[:10],  # Top 10
        }


WEIGHTS = BlockingWeightProfile.get_weights("balanced")  # Default

# New constants
HARD_CAP_COMPS = 8_000_000  # Max estimated comparisons
MIN_ACCEPTABLE_QS = 0.3  # Minimum quality score for a strategy to be considered


class BlockingGuardrailError(RuntimeError):
    """Raised when projected blocking comparisons exceed guardrail limits."""

    pass


# Define a whitelist of safe symbols for the evaluator
SAFE_SYMS = {"abs": abs, "min": min, "max": max, "len": len}


def _projected_pairs_from_strategy(strategy: dict, brute_pairs: int) -> Optional[int]:
    """Derive projected candidate pair count from strategy metadata."""
    if not isinstance(strategy, dict):
        return None
    estimated = strategy.get("estimated_pairs_after")
    if estimated is not None:
        try:
            return int(estimated)
        except (TypeError, ValueError):
            pass
    guard_info = strategy.get("guard_estimates")
    if isinstance(guard_info, dict):
        projected = guard_info.get("projected_candidate_pairs")
        if projected is not None:
            try:
                return int(projected)
            except (TypeError, ValueError):
                pass
    raw_metrics = strategy.get("raw_metrics")
    if isinstance(raw_metrics, dict):
        rr = raw_metrics.get("RR")
        if rr is not None:
            try:
                return int(brute_pairs * (1.0 - float(rr)))
            except (TypeError, ValueError):
                return None
    return None


# Add to engine.py


class BlockingColumnValidator:
    """Validates whether columns are suitable for blocking."""

    def __init__(self, stats_cache: ColumnStatsCache | None = None) -> None:
        self.stats_cache = stats_cache or ColumnStatsCache()

    def validate_column(
        self, series: pd.Series, sample_size: int = 10000, context: dict = None
    ) -> tuple[bool, list[str], dict]:
        """
        Comprehensive validation of a potential blocking column.

        Returns:
            (is_valid, issues, metrics)
        """
        issues = []
        metrics = {}

        header = getattr(series, "name", "")
        if self.stats_cache:
            series = self.stats_cache.remember_series(
                series, series.attrs.get("_fmatch_parent_df_id", id(series))
            )

        # Sample if needed
        if len(series) > sample_size:
            sample = series.sample(n=sample_size, random_state=42)
        else:
            sample = series

        # 1. Null check
        null_ratio = series.isnull().sum() / len(series)
        metrics["null_ratio"] = null_ratio
        if null_ratio > 0.5:
            issues.append(f"High null ratio: {null_ratio:.1%}")

        # 2. Cardinality check
        n_unique = series.nunique()
        unique_ratio = n_unique / len(series)
        metrics["unique_ratio"] = unique_ratio
        metrics["n_unique"] = n_unique

        if n_unique < 3:
            issues.append(f"Too few unique values: {n_unique}")
        elif unique_ratio > 0.95:
            issues.append(f"Too unique: {unique_ratio:.1%} distinct")

        # 3. Distribution check
        value_counts = series.value_counts()
        if len(value_counts) > 0:
            top_value_ratio = value_counts.iloc[0] / len(series)
            metrics["top_value_ratio"] = top_value_ratio

            if top_value_ratio > 0.8:
                top_value = value_counts.index[0]
                issues.append(
                    f"Highly skewed: '{top_value}' is {top_value_ratio:.1%} of data"
                )

        # 4. Empty/meaningless values
        if series.dtype == "object" or pd.api.types.is_string_dtype(series):
            string_sample = sample.astype(str)
            empty_ratio = (string_sample.str.strip() == "").sum() / len(sample)
            metrics["empty_ratio"] = empty_ratio

            if empty_ratio > 0.3:
                issues.append(f"High empty value ratio: {empty_ratio:.1%}")

            # Check for placeholder values
            placeholders = [
                "n/a",
                "na",
                "null",
                "none",
                "unknown",
                "tbd",
                "not available",
                "not applicable",
            ]
            placeholder_mask = string_sample.str.lower().isin(placeholders)
            placeholder_ratio = placeholder_mask.sum() / len(sample)

            if placeholder_ratio > 0.1:
                issues.append(f"High placeholder ratio: {placeholder_ratio:.1%}")

        # 5. Type-specific checks
        if pd.api.types.is_datetime64_any_dtype(series):
            issues.append("Date column - needs preprocessing (year/month extraction)")
        elif pd.api.types.is_bool_dtype(series):
            issues.append("Boolean column - maximum 2 blocks")

        # 6. Context-aware checks
        if context:
            expected_blocks = context.get("expected_blocks")
            if expected_blocks and n_unique > expected_blocks * 10:
                issues.append(f"May create too many blocks: {n_unique} unique values")

        # Calculate overall validity
        is_valid = len(issues) == 0

        # Add quality score
        quality_score = 1.0
        quality_score -= null_ratio * 0.3
        quality_score -= abs(unique_ratio - 0.1) * 0.2  # Ideal around 10%
        if "top_value_ratio" in metrics:
            quality_score -= max(0, metrics["top_value_ratio"] - 0.5) * 0.3
        quality_score = max(0, quality_score)

        metrics["quality_score"] = quality_score

        if self.stats_cache:
            self.stats_cache.cache_series_metrics(series, header, metrics.copy())

        return is_valid, issues, metrics

    @staticmethod
    def suggest_preprocessing(series: pd.Series, issues: list[str]) -> list[str]:
        """Suggest preprocessing based on validation issues."""
        suggestions = []

        issue_text = " ".join(issues).lower()

        if "date column" in issue_text:
            suggestions.append("Extract year: df['year'] = df['date_col'].dt.year")
            suggestions.append("Extract month: df['month'] = df['date_col'].dt.month")
            suggestions.append(
                "Extract quarter: df['quarter'] = df['date_col'].dt.quarter"
            )

        if "too unique" in issue_text:
            suggestions.append("Use first N characters for blocking")
            suggestions.append("Apply phonetic encoding (soundex/metaphone)")
            suggestions.append("Extract domain/stem/category")

        if "high null ratio" in issue_text:
            suggestions.append("Combine with another column for composite key")
            suggestions.append("Use a fallback value for nulls")

        if "highly skewed" in issue_text:
            suggestions.append("Split the dominant value into sub-blocks")
            suggestions.append("Use composite blocking with another column")

        return suggestions


def validate_blocking_candidates(
    candidates: list[tuple[str, str]],
    df_src: pd.DataFrame,
    df_ref: Optional[pd.DataFrame],
    mode: BlockingMode,
    *,
    stats_cache: ColumnStatsCache | None = None,
) -> list[dict]:
    """
    Validate and score blocking candidates with exotic data awareness.
    """
    validator = BlockingColumnValidator(stats_cache)
    validated_candidates = []

    for src_col, ref_col in candidates:
        # Validate source column
        src_series = df_src[src_col]
        if stats_cache:
            src_series = stats_cache.remember_series(src_series, id(df_src))
        src_valid, src_issues, src_metrics = validator.validate_column(
            src_series, context={"mode": mode}
        )

        # NEW: Add exotic data profiling
        if ExoticDataDetector:
            try:
                exotic_profile = ExoticDataDetector.detect_characteristics(
                    df_src[src_col].head(1000)
                )
                src_metrics.update(exotic_profile)
            except Exception as e:
                log.debug(f"ExoticDataDetector failed for {src_col}: {e}")
                # Fail-open policy: continue without exotic detection

        candidate_info = {
            "src_col": src_col,
            "ref_col": ref_col,
            "src_valid": src_valid,
            "src_issues": src_issues,
            "src_metrics": src_metrics,
            "overall_valid": src_valid,
            "quality_score": src_metrics.get("quality_score", 0),
        }

        # Validate reference column for MATCH mode
        if mode == BlockingMode.MATCH and df_ref is not None and ref_col:
            ref_series = df_ref[ref_col]
            if stats_cache:
                ref_series = stats_cache.remember_series(ref_series, id(df_ref))
            ref_valid, ref_issues, ref_metrics = validator.validate_column(
                ref_series, context={"mode": mode}
            )

            # NEW: Add exotic data profiling for reference
            if ExoticDataDetector:
                try:
                    ref_exotic_profile = ExoticDataDetector.detect_characteristics(
                        df_ref[ref_col].head(1000)
                    )
                    ref_metrics.update(ref_exotic_profile)
                except Exception as e:
                    log.debug(f"ExoticDataDetector failed for ref {ref_col}: {e}")

            candidate_info.update(
                {
                    "ref_valid": ref_valid,
                    "ref_issues": ref_issues,
                    "ref_metrics": ref_metrics,
                    "overall_valid": src_valid and ref_valid,
                    "quality_score": (
                        src_metrics.get("quality_score", 0)
                        + ref_metrics.get("quality_score", 0)
                    )
                    / 2,
                }
            )

        validated_candidates.append(candidate_info)

    # Sort by quality score
    validated_candidates.sort(key=lambda x: x["quality_score"], reverse=True)

    return validated_candidates


def calculate_adaptive_thresholds(df_sample: pd.DataFrame) -> dict:
    """Calculate data-specific thresholds for blocking analysis."""
    if df_sample is None or df_sample.empty:
        # Return default conservative thresholds if no data
        return {"min_acceptable_qs": 0.3, "max_p1": 0.5, "min_entropy": 0.3}

    row_count = len(df_sample)
    # Calculate average cardinality only on string/object columns as they are typical for blocking
    string_cols = df_sample.select_dtypes(include=["object", "string"])
    avg_cardinality = string_cols.nunique().mean() if not string_cols.empty else 0

    # Adjust thresholds based on data characteristics
    # Lower QS for small datasets where stats are less reliable
    min_qs = 0.2 if row_count < 10000 else 0.3
    # Allow larger dominant blocks (higher P1) for low cardinality data
    max_p1 = 0.7 if avg_cardinality < 100 else 0.5
    # Require less entropy for low cardinality data
    min_entropy = 0.2 if avg_cardinality < 50 else 0.3

    thresholds = {
        "min_acceptable_qs": min_qs,
        "max_p1": max_p1,
        "min_entropy": min_entropy,
    }

    log.info(
        f"Adaptive thresholds calculated: {thresholds} (row_count={row_count}, avg_cardinality={avg_cardinality:.2f})"
    )
    return thresholds


def apply_custom_rules(df: pd.DataFrame, rules: list[dict]) -> pd.DataFrame:
    """
    Applies a list of user-defined match rules to a DataFrame of potential matches
    using a secure and vectorized approach.
    """
    if not rules:
        df["custom_rule_score"] = 0.0
        return df

    df["custom_rule_score"] = 0.0
    # Initialize a sandboxed interpreter with only safe symbols
    aeval = asteval.Interpreter(symtable=SAFE_SYMS, use_numpy=False, minimal=True)

    for i, rule in enumerate(rules):
        cond = rule.get("condition")
        adj = rule.get("score_adjustment", 0)
        if not cond:
            continue

        try:
            # Vectorized evaluation using pandas' fast engine
            mask = df.eval(cond, engine="python", local_dict=aeval.symtable)
            df.loc[mask, "custom_rule_score"] += adj
        except Exception as exc:
            # Log the specific rule that failed for easier debugging
            log.warning(f"Custom rule #{i} ('{cond}') failed to evaluate: {exc}")

    return df


# Ã¢â€â‚¬Ã¢â€â‚¬ Enums Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
class BlockingMode(Enum):
    DEDUPE = "duplicate"
    MATCH = "match"


class MatchAlgorithm(str, Enum):
    """Enum for supported matching algorithms."""

    wratio = "WRatio"
    qratio = "QRatio"
    ratio = "Ratio"
    partial_ratio = "PartialRatio"
    token_set_ratio = "TokenSetRatio"
    token_sort_ratio = "TokenSortRatio"
    jaro = "Jaro"
    jaro_winkler = "JaroWinkler"
    soundex = "Soundex"
    exact = "Exact"


# Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
#  Job configuration objects (passed from GUI / API into workers)
# Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
@dataclass
class MatchConfig:
    # Non-defaults first
    job_id: str  # type: ignore
    src_path: Union[str, Path, pd.DataFrame]  # MODIFIED: Allow DataFrame
    ref_path: Union[str, Path, pd.DataFrame]  # MODIFIED: Allow DataFrame
    # Defaults follow
    maps: List[Dict[str, Any]] = field(default_factory=list)
    algorithm: str = DEFAULT_ALGORITHM_KEY  # Ensure this name and default
    threshold: int = 80
    ruleset_name: str = DEFAULT_RULESET_NAME
    block_key_length: int = 3
    src_id_col: Optional[str] = None
    ref_id_col: Optional[str] = None
    cache_path: Optional[str] = None
    block_col: str = _ACTIVE_BLOCK_KEY_
    mode: str = "match"
    chunksize: Optional[int] = None  # Added chunksize
    # progress_q: Optional[Any] = None # <<< REMOVE THIS LINE
    processing_mode: Literal["speed", "scale"] = "scale"  # Added processing_mode
    output_dir: Union[str, Path] = "bulk_results"  # Ensure this matches your builder
    apply_blocking: bool = True  # <<< NEW FIELD, defaults to True
    result_mode: str = "all"  # SPRINT A: "all" or "best"
    preprocessing_rules: List[str] = field(default_factory=list)
    custom_match_rules: List[dict] = field(default_factory=list)
    preproc_variant: str = "alnum"  # Preprocessing variant for blocking key generation
    strip_b2c_domains: bool = True  # Enable B2C domain filtering
    source_block_col: Optional[str] = None
    ref_block_col: Optional[str] = None
    residual_handling: Optional[dict] = None
    max_workers: int = CDIST_WORKERS
    coverage_gate: float = 0.60  # Minimum field coverage required before penalty. Lower = more lenient matching

    def __post_init__(self):
        # Validate threshold
        if not 0 <= self.threshold <= 100:
            raise ValueError(f"Threshold must be 0-100, got {self.threshold}")

        # Validate algorithm exists
        if self.algorithm not in ALGORITHM_MAP:
            raise ValueError(f"Unknown algorithm: {self.algorithm}")

        # Validate maps structure
        if self.maps:
            for i, m in enumerate(self.maps):
                if "source" not in m:
                    raise ValueError(f"Mapping {i} must have 'source' field")
                if self.mode == "match" and "ref" not in m:
                    raise ValueError(f"Match mode mapping {i} must have 'ref' field")

        # Validate coverage_gate
        if not 0 <= self.coverage_gate <= 1:
            raise ValueError(
                f"coverage_gate must be between 0 and 1, got {self.coverage_gate}"
            )


@dataclass
class DedupeConfig:
    # Non-default fields (builder should ensure these are provided)
    job_id: str
    input_path: Union[str, Path, pd.DataFrame]
    rec_id_col: str
    algorithm: str
    threshold: int

    # Historically required positional args; provide safe defaults so legacy
    # callers that omit them do not crash.
    ruleset_name: str = "default"
    block_key_length: int = 3
    dup_block_limit: int = DEFAULT_DUP_BLOCK_LIMIT

    # Fields with defaults
    maps: List[Dict[str, Any]] = field(default_factory=list)
    cache_path: Optional[str] = None
    block_col: str = _ACTIVE_BLOCK_KEY_
    mode: str = "duplicate"
    chunksize: Optional[int] = None
    processing_mode: Literal["speed", "scale"] = "scale"
    output_dir: Path = Path("bulk_results")
    apply_blocking: bool = True
    result_mode: str = "all"
    preprocessing_rules: List[str] = field(default_factory=list)
    custom_match_rules: List[dict] = field(default_factory=list)
    preproc_variant: str = "alnum"
    strip_b2c_domains: bool = True
    b2c_config: Optional[Dict[str, Any]] = None  # You just added this
    norm_rules: Dict[str, Any] = field(default_factory=dict)  # ADD THIS LINE
    max_workers: int = CDIST_WORKERS
    coverage_gate: float = 0.60  # Minimum field coverage required before penalty. Lower = more lenient matching
    source_block_col: Optional[str] = (
        None  # Column to use for blocking (e.g., "Domain", "Email")
    )
    residual_handling: Optional[dict] = None

    def to_dict(self) -> dict:
        """Safely serializes the dataclass to a dictionary for RQ."""
        d = asdict(self)
        # Ensure Path objects are converted to strings for JSON serialization
        d["input_path"] = str(self.input_path) if self.input_path else None
        d["output_dir"] = str(self.output_dir) if self.output_dir else None
        return d

    def __post_init__(self):
        # Similar validations
        if not 0 <= self.threshold <= 100:
            raise ValueError(f"Threshold must be 0-100, got {self.threshold}")

        if self.algorithm not in ALGORITHM_MAP:
            raise ValueError(f"Unknown algorithm: {self.algorithm}")

        # Validate coverage_gate
        if not 0 <= self.coverage_gate <= 1:
            raise ValueError(
                f"coverage_gate must be between 0 and 1, got {self.coverage_gate}"
            )


# Back-compat shims / Alias definitions
MatchJobConfig = MatchConfig
DedupeJobConfig = DedupeConfig


@dataclass
class EnhancedMatchConfig(MatchConfig):
    """Enhanced match configuration with multi-algorithm support."""

    enable_multi_algo: bool = True
    column_algorithms: Optional[Dict[str, str]] = None


@dataclass
class EnhancedDedupeConfig(DedupeConfig):
    """Enhanced dedupe configuration with multi-algorithm support."""

    enable_multi_algo: bool = True
    column_algorithms: Optional[Dict[str, str]] = None


def get_algorithm_function(algorithm_name: str) -> Callable:
    """Get the algorithm function by name."""
    return ALGORITHM_MAP.get(algorithm_name, rf_fuzz.WRatio)


def get_algorithms_used(results: List[Dict]) -> Dict[str, int]:
    """Extract algorithm usage statistics from results."""
    algo_counts = {}
    for result in results:
        for algo in result.get("algorithms_used", []):
            algo_counts[algo] = algo_counts.get(algo, 0) + 1
    return algo_counts


def _require(d: dict, *alts: str):  # type: ignore
    """
    Utility to ensure that at least one of a set of alternative keys is present in a dictionary.
    Raises ValueError if none are present.
    Returns the value of the first key found.
    """
    for k in alts:
        if k in d:
            if d[k] is None:  # Check if the value for the key is None
                log.warning(f"Required key '{k}' is present but its value is None.")
                # Depending on desired behavior, you might want to continue searching
                # or raise an error/return a default. For now, it returns None.
            return d[k]
    raise ValueError(f"At least one of {alts} must be provided in {list(d.keys())}")


def _estimate_df_mem(*dfs: pd.DataFrame) -> int:
    """Return a lower-bound byte estimate for all provided DataFrames."""
    return sum(df.memory_usage(deep=True).sum() for df in dfs if df is not None)


# In src/fmatch/engine.py


def build_match_config(**kwargs: Any) -> MatchConfig:
    """
    Helper to build MatchConfig from kwargs, handling both paths and DataFrames.
    Accepts both modern `mappings` and legacy `maps` keywords.
    Now supports multi-algorithm configurations.
    """
    log.debug("build_match_config called.")

    # --- Robustly handle src_path (from new code) ---
    src_data = _require(kwargs, "src_path")
    if isinstance(src_data, (str, os.PathLike)):
        final_src_path = Path(src_data)
    elif isinstance(src_data, pd.DataFrame):
        final_src_path = src_data
    else:
        raise TypeError(
            f"src_path must be str, Path, or DataFrame, got {type(src_data).__name__}"
        )

    # --- Robustly handle ref_path (from new code) ---
    ref_data = _require(kwargs, "ref_path")
    if isinstance(ref_data, (str, os.PathLike)):
        final_ref_path = Path(ref_data)
    elif isinstance(ref_data, pd.DataFrame):
        final_ref_path = ref_data
    else:
        raise TypeError(
            f"ref_path must be str, Path, or DataFrame, got {type(ref_data).__name__}"
        )

    # --- Logic from existing code ---
    maps = kwargs.pop("mappings", kwargs.get("maps", None))

    output_dir_val = _require(kwargs, "output_dir")
    if not isinstance(output_dir_val, Path):
        output_dir_val = Path(output_dir_val)
    if not output_dir_val.is_absolute():
        log.warning(
            f"build_match_config: output_dir '{output_dir_val}' is not absolute."
        )

    # --- Enhanced ID Detection ---
    src_id_col = kwargs.get("src_id_col")
    ref_id_col = kwargs.get("ref_id_col")
    file_source = kwargs.get("file_source")  # Optional for telemetry

    # Auto-detect source ID column if not provided and we have a DataFrame
    if isinstance(final_src_path, pd.DataFrame) and not src_id_col:
        detected_src_id = auto_select_id_column(final_src_path, file_source)
        if detected_src_id:
            src_id_col = detected_src_id
            log.info(f"Auto-detected source ID column: '{src_id_col}'")
        else:
            log.warning("Failed to auto-detect source ID column")

    # Auto-detect reference ID column if not provided and we have a DataFrame
    if isinstance(final_ref_path, pd.DataFrame) and not ref_id_col:
        detected_ref_id = auto_select_id_column(final_ref_path, file_source)
        if detected_ref_id:
            ref_id_col = detected_ref_id
            log.info(f"Auto-detected reference ID column: '{ref_id_col}'")
        else:
            log.warning("Failed to auto-detect reference ID column")

    # +++ MULTI-ALGO: Check if multi-algorithm support is requested +++
    enable_multi_algo = kwargs.get("enable_multi_algo", False)

    if enable_multi_algo:
        # Create EnhancedMatchConfig for multi-algorithm support
        log.info("Creating EnhancedMatchConfig for multi-algorithm support")

        new_match_config = EnhancedMatchConfig(
            job_id=kwargs.get("job_id", str(uuid.uuid4())),
            src_path=final_src_path,
            ref_path=final_ref_path,
            maps=maps,
            algorithm=kwargs.get("algorithm", DEFAULT_ALGORITHM_KEY),
            threshold=kwargs.get("threshold", 80),
            ruleset_name=kwargs.get("ruleset_name", DEFAULT_RULESET_NAME),
            block_key_length=kwargs.get("block_key_length", 3),
            src_id_col=src_id_col,
            ref_id_col=ref_id_col,
            cache_path=kwargs.get("cache_path"),
            block_col=kwargs.get("block_col", _ACTIVE_BLOCK_KEY_),
            chunksize=kwargs.get("chunksize", 50_000),
            output_dir=output_dir_val,
            processing_mode=kwargs.get("processing_mode", "scale"),
            apply_blocking=kwargs.get("apply_blocking", True),
            result_mode=kwargs.get("result_mode", "all"),
            preproc_variant=kwargs.get("preproc_variant", "alnum"),
            strip_b2c_domains=kwargs.get("strip_b2c_domains", True),
            source_block_col=kwargs.get("source_block_col"),
            ref_block_col=kwargs.get("ref_block_col"),
            residual_handling=kwargs.get("residual_handling"),
            # Enhanced fields
            enable_multi_algo=True,
            column_algorithms=kwargs.get("column_algorithms", None),
        )

        log.info(
            f"EnhancedMatchConfig created: job_id={new_match_config.job_id}, "
            f"algo={new_match_config.algorithm}, multi_algo=True, "
            f"input_types='{type(new_match_config.src_path).__name__}', "
            f"'{type(new_match_config.ref_path).__name__}'"
        )
    else:
        # Create standard MatchConfig
        new_match_config = MatchConfig(
            job_id=kwargs.get("job_id", str(uuid.uuid4())),
            src_path=final_src_path,
            ref_path=final_ref_path,
            maps=maps,
            algorithm=kwargs.get("algorithm", DEFAULT_ALGORITHM_KEY),
            threshold=kwargs.get("threshold", 80),
            ruleset_name=kwargs.get("ruleset_name", DEFAULT_RULESET_NAME),
            block_key_length=kwargs.get("block_key_length", 3),
            src_id_col=src_id_col,
            ref_id_col=ref_id_col,
            cache_path=kwargs.get("cache_path"),
            block_col=kwargs.get("block_col", _ACTIVE_BLOCK_KEY_),
            chunksize=kwargs.get("chunksize", 50_000),
            output_dir=output_dir_val,
            processing_mode=kwargs.get("processing_mode", "scale"),
            apply_blocking=kwargs.get("apply_blocking", True),
            result_mode=kwargs.get("result_mode", "all"),
            preproc_variant=kwargs.get("preproc_variant", "alnum"),
            strip_b2c_domains=kwargs.get("strip_b2c_domains", True),
            source_block_col=kwargs.get("source_block_col"),
            ref_block_col=kwargs.get("ref_block_col"),
            residual_handling=kwargs.get("residual_handling"),
        )

        log.info(
            f"MatchConfig created: job_id={new_match_config.job_id}, "
            f"algo={new_match_config.algorithm}, "
            f"input_types='{type(new_match_config.src_path).__name__}', "
            f"'{type(new_match_config.ref_path).__name__}'"
        )
    # +++ END MULTI-ALGO +++

    return new_match_config


def build_dedupe_config(
    progress_q: Optional[mp.Queue] = None, **kwargs: Any
) -> DedupeConfig:
    """
    Helper to build DedupeConfig from kwargs, handling both paths and DataFrames.
    Now supports multi-algorithm configurations.
    """
    log.debug("build_dedupe_config called.")

    # --- Handle input_path that can be str, Path, or DataFrame ---
    input_data = _require(kwargs, "input_path")

    # Conditionally wrap with Path() only if it's a string/path
    if isinstance(input_data, (str, os.PathLike)):
        final_input_path = Path(input_data)
    elif isinstance(input_data, pd.DataFrame):
        # Pass DataFrame through directly
        final_input_path = input_data
    else:
        # Handle unexpected types gracefully
        raise TypeError(
            f"input_path must be str, Path, or DataFrame, got {type(input_data).__name__}"
        )

    # Handle output_dir (always a Path)
    output_dir_from_main = kwargs.get("output_dir")
    final_output_dir_param = (
        Path(output_dir_from_main) if output_dir_from_main else None
    )

    # --- Enhanced ID Detection for Dedupe ---
    rec_id_col = kwargs.get("rec_id_col")
    file_source = kwargs.get("file_source")  # Optional for telemetry

    # Auto-detect record ID column if not provided and we have a DataFrame
    if isinstance(final_input_path, pd.DataFrame) and not rec_id_col:
        detected_rec_id = auto_select_id_column(final_input_path, file_source)
        if detected_rec_id:
            rec_id_col = detected_rec_id
            log.info(f"Auto-detected record ID column: '{rec_id_col}'")
        else:
            # For dedupe, we really need an ID column
            log.error("Failed to auto-detect record ID column for deduplication")
            raise ValueError(
                "rec_id_col is required for deduplication but could not be auto-detected"
            )

    # +++ MULTI-ALGO: Check if multi-algorithm support is requested +++
    enable_multi_algo = kwargs.get("enable_multi_algo", False)

    if enable_multi_algo:
        # Create EnhancedDedupeConfig for multi-algorithm support
        log.info("Creating EnhancedDedupeConfig for multi-algorithm support")

        new_dedupe_config = EnhancedDedupeConfig(
            job_id=kwargs.get("job_id", str(uuid.uuid4())),
            input_path=final_input_path,
            output_dir=final_output_dir_param,
            rec_id_col=rec_id_col,
            algorithm=kwargs.get("algorithm", DEFAULT_ALGORITHM_KEY),
            threshold=kwargs.get("threshold", DEFAULT_THRESHOLD),
            maps=kwargs.get("maps", []),
            ruleset_name=kwargs.get("ruleset_name"),
            dup_block_limit=kwargs.get("dup_block_limit", DEFAULT_DUP_BLOCK_LIMIT),
            cache_path=kwargs.get("cache_path"),
            processing_mode=kwargs.get("processing_mode", "scale"),
            result_mode=kwargs.get("result_mode", "all"),
            apply_blocking=kwargs.get("apply_blocking", True),
            block_key_length=kwargs.get("block_key_length", 3),
            preproc_variant=kwargs.get("preproc_variant", "alnum"),
            strip_b2c_domains=kwargs.get("strip_b2c_domains", False),
            b2c_config=kwargs.get("b2c_config"),
            norm_rules=kwargs.get("norm_rules", {}),
            # Enhanced fields
            enable_multi_algo=True,
            column_algorithms=kwargs.get("column_algorithms", None),
        )

        log.info(
            f"EnhancedDedupeConfig created: job_id={new_dedupe_config.job_id}, "
            f"algo={new_dedupe_config.algorithm}, multi_algo=True, "
            f"input_type='{type(new_dedupe_config.input_path).__name__}', "
            f"processing_mode='{new_dedupe_config.processing_mode}'"
        )
    else:
        # Create standard DedupeConfig
        new_dedupe_config = DedupeConfig(
            job_id=kwargs.get("job_id", str(uuid.uuid4())),
            input_path=final_input_path,
            output_dir=final_output_dir_param,
            rec_id_col=rec_id_col,
            algorithm=kwargs.get("algorithm", DEFAULT_ALGORITHM_KEY),
            threshold=kwargs.get("threshold", DEFAULT_THRESHOLD),
            maps=kwargs.get("maps", []),
            ruleset_name=kwargs.get("ruleset_name"),
            dup_block_limit=kwargs.get("dup_block_limit", DEFAULT_DUP_BLOCK_LIMIT),
            cache_path=kwargs.get("cache_path"),
            processing_mode=kwargs.get("processing_mode", "scale"),
            result_mode=kwargs.get("result_mode", "all"),
            apply_blocking=kwargs.get("apply_blocking", True),
            block_key_length=kwargs.get("block_key_length", 3),
            preproc_variant=kwargs.get("preproc_variant", "alnum"),
            strip_b2c_domains=kwargs.get("strip_b2c_domains", False),
            b2c_config=kwargs.get("b2c_config"),
            norm_rules=kwargs.get("norm_rules", {}),
        )

        log.info(
            f"DedupeConfig created: job_id={new_dedupe_config.job_id}, "
            f"algo={new_dedupe_config.algorithm}, "
            f"input_type='{type(new_dedupe_config.input_path).__name__}', "
            f"processing_mode='{new_dedupe_config.processing_mode}'"
        )
    # +++ END MULTI-ALGO +++

    return new_dedupe_config


def _get_paths(cfg):
    """Return (src_df, ref_df) for MatchConfig OR (input_df, None) for DedupeConfig."""
    if isinstance(cfg, MatchConfig):
        return cfg.src_path, cfg.ref_path
    else:  # DedupeConfig
        return cfg.input_path, None


# In engine.py

# import cProfile, pstats, os, uuid, tempfile # Ensure tempfile is imported (already covered by existing imports)


def _profiled_worker(func):
    # @functools.wraps(func) # Optional: to preserve metadata if you use it elsewhere
    def inner(*args, **kwargs):
        profile_dir_base = (
            kwargs.get("config").output_dir
            if "config" in kwargs and hasattr(kwargs.get("config"), "output_dir")
            else Path(tempfile.gettempdir())
        )
        # Fallback if output_dir is not Path or is None (should ideally come from config)
        if not isinstance(profile_dir_base, Path):
            profile_dir_base = Path(tempfile.gettempdir())

        # Use the profiles subdirectory as intended by existing worker code
        profile_dir = profile_dir_base / "profiles"
        profile_dir.mkdir(parents=True, exist_ok=True)

        # Sanitize job_id from config if available, else use uuid
        job_id_for_filename = "unknown_job"
        if "config" in kwargs and hasattr(kwargs.get("config"), "job_id"):
            job_id_for_filename = sanitize_column_name(
                str(kwargs.get("config").job_id)
            )  # Use your existing sanitize_column_name

        # Construct filename using process ID and a unique part of job_id or uuid
        # Including the original function name is good for identifying profile types
        unique_part = (
            job_id_for_filename.split("_")[-1]
            if "_" in job_id_for_filename
            else job_id_for_filename
        )
        if (
            not unique_part or len(unique_part) < 5
        ):  # ensure unique_part is somewhat useful
            unique_part = uuid.uuid4().hex[:8]

        fname = profile_dir / f"{func.__name__}_{os.getpid()}_{unique_part}.prof"

        log.debug(
            f"PROFILING: Worker {func.__name__} (PID: {os.getpid()}) output to: {fname}"
        )

        with cProfile.Profile() as pr:
            result = func(*args, **kwargs)
        pr.dump_stats(str(fname))  # dump_stats takes a string path
        return result

    return inner


def create_single_algorithm_scorer(config, mappings: List[EnhancedMapping]):
    """Create a single algorithm scorer for backward compatibility."""

    class SingleAlgorithmScorer:
        def __init__(self, algorithm: str, mappings: List[EnhancedMapping]):
            self.algorithm = algorithm
            self.mappings = mappings
            self.algo_func = ALGORITHM_MAP.get(algorithm)
            if not self.algo_func:
                log.warning(f"Algorithm '{algorithm}' not found. Defaulting to WRatio.")
                self.algo_func = ALGORITHM_MAP.get("WRatio")

        def score_pair(self, src_record: Dict, ref_record: Dict) -> Dict:
            field_scores = []
            total_weight = 0.0
            weighted_sum = 0.0

            for mapping in self.mappings:
                transforms = mapping.transforms or []
                src_raw = src_record.get(mapping.source)
                ref_raw = ref_record.get(mapping.ref)

                src_val = apply_transforms(src_raw, transforms or ("strip",))
                ref_val = apply_transforms(ref_raw, transforms or ("strip",))

                if not src_val or not ref_val:
                    continue

                algo_name = mapping.preferred_algo or self.algorithm
                algo_func = ALGORITHM_MAP.get(algo_name, self.algo_func)
                score = algo_func(src_val, ref_val)

                field_scores.append(
                    {
                        "source_field": mapping.source,
                        "ref_field": mapping.ref,
                        "algorithm": algo_name,
                        "score": score,
                        "weight": mapping.weight,
                    }
                )

                weighted_sum += score * mapping.weight
                total_weight += mapping.weight

            final_score = weighted_sum / total_weight if total_weight > 0 else 0.0

            return {
                "confidence_score": final_score,
                "field_score_details": field_scores,
                "algorithms_used": [self.algorithm],
            }

    return SingleAlgorithmScorer(config.algorithm, mappings)


def worker_match_enhanced(
    config: EnhancedMatchConfig,
    src_chunk: pd.DataFrame,
    ref_df: pd.DataFrame,
    progress_callback=None,
) -> List[Dict]:
    """Enhanced match worker with multi-algorithm support."""

    # Convert config maps to EnhancedMapping objects
    enhanced_mappings = []
    for m in config.maps:
        enhanced_mappings.append(
            EnhancedMapping(
                source=m["source"],
                ref=m["ref"],
                weight=m.get("weight", 1.0),
                preferred_algo=m.get("preferred_algo"),
                algo_confidence=m.get("algo_confidence", 0.0),
                transforms=m.get("transforms") or [],
                preprocessing=m.get("preprocessing"),
            )
        )

    threshold = float(config.threshold) if config.threshold is not None else 0.0
    if threshold <= 1:
        threshold *= 100.0
        config.threshold = threshold

    domain_exact_count = sum(
        1
        for m in enhanced_mappings
        if _enhanced_mapping_is_domain(m)
        and (m.preferred_algo or "").lower() == "exact"
    )
    if domain_exact_count and threshold > 65:
        display_before = threshold / 100.0 if threshold > 1 else threshold
        log.info(
            "Domain identity present (enhanced match); lowering threshold from %.2f to 0.65",
            display_before,
        )
        threshold = 65.0
        config.threshold = threshold

    if enhanced_mappings:
        scorer = MultiAlgorithmScorer(
            enhanced_mappings, coverage_gate=getattr(config, "coverage_gate", 0.60)
        )
        config.enable_multi_algo = True
        display_threshold = threshold / 100.0 if threshold > 1 else threshold
        log.info(
            "[EnhancedMatch] per-mapping scorer active | maps=%d | domain_exact=%d | threshold=%.2f",
            len(enhanced_mappings),
            domain_exact_count,
            display_threshold,
        )
    else:
        scorer = None
        config.enable_multi_algo = False

    results = []
    total_rows = len(src_chunk)

    for idx, src_row in src_chunk.iterrows():
        best_match = None
        best_score = 0.0
        best_details = None

        for ref_idx, ref_row in ref_df.iterrows():
            if not scorer:
                continue
            score_result = scorer.score_pair(src_row.to_dict(), ref_row.to_dict())

            if score_result["confidence_score"] > best_score:
                best_score = score_result["confidence_score"]
                best_match = ref_idx
                best_details = score_result

        if best_details and best_score >= config.threshold:
            results.append(
                {
                    "source_idx": idx,
                    "ref_idx": best_match,
                    "confidence_score": best_score,
                    "field_scores": best_details.get("field_score_details", []),
                    "algorithms_used": best_details.get("algorithms_used", []),
                }
            )

        # Report progress - FIXED: removed await
        if progress_callback and idx % 100 == 0:
            progress_callback(idx / total_rows)  # <-- No await, just call it

    # Clear cache periodically to manage memory
    if hasattr(scorer, "clear_cache"):
        scorer.clear_cache()

    return results


async def worker_dedupe_enhanced(
    config: EnhancedDedupeConfig, src_chunk: pd.DataFrame, progress_callback=None
) -> List[Dict]:
    """Enhanced dedupe worker with multi-algorithm support."""

    # Convert config maps to EnhancedMapping objects
    enhanced_mappings = []
    for m in config.maps:
        enhanced_mappings.append(
            EnhancedMapping(
                source=m["source"],
                ref=m["source"],  # Same column for dedupe
                weight=m.get("weight", 1.0),
                preferred_algo=m.get("preferred_algo"),
                algo_confidence=m.get("algo_confidence", 0.0),
                transforms=m.get("transforms") or [],
                preprocessing=m.get("preprocessing"),
            )
        )

    threshold = float(config.threshold) if config.threshold is not None else 0.0
    if threshold <= 1:
        threshold *= 100.0
        config.threshold = threshold

    domain_exact_count = sum(
        1
        for m in enhanced_mappings
        if _enhanced_mapping_is_domain(m)
        and (m.preferred_algo or "").lower() == "exact"
    )
    if domain_exact_count and threshold > 65:
        display_before = threshold / 100.0 if threshold > 1 else threshold
        log.info(
            "Domain identity present (enhanced dedupe); lowering threshold from %.2f to 0.65",
            display_before,
        )
        threshold = 65.0
        config.threshold = threshold

    if enhanced_mappings:
        scorer = MultiAlgorithmScorer(
            enhanced_mappings, coverage_gate=getattr(config, "coverage_gate", 0.60)
        )
        config.enable_multi_algo = True
        display_threshold = threshold / 100.0 if threshold > 1 else threshold
        log.info(
            "[EnhancedDedupe] per-mapping scorer active | maps=%d | domain_exact=%d | threshold=%.2f",
            len(enhanced_mappings),
            domain_exact_count,
            display_threshold,
        )
    else:
        scorer = None
        config.enable_multi_algo = False

    duplicates = []
    total_rows = len(src_chunk)

    rows_with_indices = [(idx, row.to_dict()) for idx, row in src_chunk.iterrows()]

    for i in range(len(rows_with_indices)):
        idx1, row1 = rows_with_indices[i]
        for j in range(i + 1, len(rows_with_indices)):
            idx2, row2 = rows_with_indices[j]

            if not scorer:
                continue
            score_result = scorer.score_pair(row1, row2)

            if score_result["confidence_score"] >= config.threshold:
                duplicates.append(
                    {
                        "idx1": idx1,
                        "idx2": idx2,
                        "confidence_score": score_result["confidence_score"],
                        "field_scores": score_result.get("field_score_details", []),
                        "algorithms_used": score_result.get("algorithms_used", []),
                    }
                )

        # Report progress based on outer loop
        if progress_callback and i % 100 == 0:
            await progress_callback(i / total_rows)

    # Clear cache
    if hasattr(scorer, "clear_cache"):
        scorer.clear_cache()

    return duplicates


# ---------------------------------------------------------------------
# Public worker Ã¢â‚¬â€œ used by GUI / API
# ---------------------------------------------------------------------
# ---------------------------------------------------------------------
# match-worker Ã¢â‚¬â€œ runs in the background thread (or a process later on)
# ---------------------------------------------------------------------
# In engine.py

# In engine.py
# (Ensure all necessary imports like pd, Optional, Dict, Any, MatchConfig, etc. are present)
# (Ensure log, RunStats, _init_score_cache_db, process_dataframe, time are defined/imported)

# In engine.py


# In engine.py, can be placed after the configuration dataclasses


def stream_candidate_pairs(
    db_connection: duckdb.DuckDBPyConnection,
    sql_query: str,
    chunk_size: int = 250_000,
) -> Generator[pd.DataFrame, None, None]:
    """
    Executes a SQL query on a DuckDB connection and streams the results
    as pandas DataFrames.
    """
    log.info(f"Streaming candidate pairs as pandas chunks of size: {chunk_size}...")
    try:
        # Use fetch_df_chunk for direct-to-pandas streaming
        for chunk in db_connection.execute(sql_query).fetch_df_chunk(
            vectors_per_chunk=chunk_size
        ):
            log.debug(f"Streaming chunk with {len(chunk)} rows.")
            # --- REVERT: No copy needed if the upstream fix is correct ---
            yield chunk
    except Exception as e:
        log.error(f"Error during streaming of candidate pairs: {e}", exc_info=True)
        raise


def process_dataframe(
    input_df: Optional[pd.DataFrame],
    ref_df: Optional[pd.DataFrame],
    config: Union["MatchConfig", "DedupeConfig"],
    rec_id_col: Optional[str] = None,
    src_id_col: Optional[str] = None,
    ref_id_col: Optional[str] = None,
    progress_q: Optional[mp.Queue] = None,
    run_stats: Optional["RunStats"] = None,
):
    # --- Extract parameters from the config object ---
    mode = config.mode
    ruleset_name = config.ruleset_name
    algorithm = config.algorithm
    threshold = config.threshold
    maps = config.maps
    block_col = config.block_col
    block_key_length = config.block_key_length
    processing_mode = config.processing_mode
    dup_block_limit = getattr(config, "dup_block_limit", 2000)
    score_cache_conn_str = config.cache_path
    apply_blocking = config.apply_blocking

    # --- SPRINT A: New parameter ---
    result_mode = getattr(config, "result_mode", "all")

    # Resolve ID columns
    effective_rec_id_col = (
        rec_id_col if rec_id_col is not None else getattr(config, "rec_id_col", None)
    )
    effective_src_id_col = (
        src_id_col if src_id_col is not None else getattr(config, "src_id_col", None)
    )
    effective_ref_id_col = (
        ref_id_col if ref_id_col is not None else getattr(config, "ref_id_col", None)
    )

    log.info(
        f"==> ENTERING process_dataframe (mode: {mode}, apply_blocking: {config.apply_blocking}, processing_mode: {config.processing_mode})"
    )
    # ... (rest of initial logging and validation)
    if not maps:
        log.warning(
            f"process_dataframe [{mode} mode]: 'maps' parameter is None or empty."
        )
        if mode == "match":
            log.error(
                "process_dataframe [match mode]: 'maps' are required for matching. Please provide column mappings."
            )
            return pd.DataFrame(
                columns=[
                    (effective_src_id_col or "s_id"),
                    (effective_ref_id_col or "r_id"),
                    "confidence_score",
                ]
            )
        elif mode == "duplicate":
            if (
                input_df is None
                or not effective_rec_id_col
                or effective_rec_id_col not in input_df.columns
            ):
                log.error(
                    f"process_dataframe [duplicate mode]: Cannot auto-generate maps. rec_id_col '{effective_rec_id_col}' missing or input_df is None."
                )
                return pd.DataFrame(
                    columns=[
                        f"{effective_rec_id_col}_1" if effective_rec_id_col else "id1",
                        f"{effective_rec_id_col}_2" if effective_rec_id_col else "id2",
                        "confidence_score",
                    ]
                )

            log.info(
                f"process_dataframe [duplicate mode]: Auto-generating identity maps for all columns except rec_id_col '{effective_rec_id_col}'."
            )
            maps = [
                {"source": c, "weight": 1.0}
                for c in input_df.columns
                if c != effective_rec_id_col
            ]
            if not maps:
                log.warning(
                    f"process_dataframe [{mode} mode]: Auto-map for rec_id_col '{effective_rec_id_col}' resulted in empty maps. Check input_df columns."
                )
            log.debug(
                f"process_dataframe: Auto-generated maps (count: {len(maps) if maps else 0}) for duplicate mode."
            )

    # ==============================================================================
    # MATCH MODE LOGIC
    # ==============================================================================
    if mode == "match":
        if input_df is None or ref_df is None:
            log.error("process_dataframe [match mode]: input_df or ref_df is None.")
            return pd.DataFrame()

        log.info("Match mode processing is starting.")

        # This call now correctly passes the `run_stats` parameter from the function signature
        match_results_list = worker_match(
            chunk=input_df,
            ref=ref_df,
            config=config,
            progress_q=progress_q,
            run_stats=run_stats,  # <-- FIX: Was 'run_stats_instance'
        )

        t2 = time.perf_counter()
        if match_results_list:
            result_df = pd.DataFrame(match_results_list)
        else:
            s_id_col_name = effective_src_id_col or "s_id"
            r_id_col_name = effective_ref_id_col or "r_id"
            result_df = pd.DataFrame(
                columns=[s_id_col_name, r_id_col_name, "confidence_score"]
            )
        t3 = time.perf_counter()
        log.info(f"DataFrame creation from results took {t3-t2:.1f}s")

        # --- SPRINT A POST-FILTERING LOGIC FOR MATCH MODE ---
        if result_mode == "best" and not result_df.empty:
            s_id_filter_col = effective_src_id_col or "s_id"
            if s_id_filter_col in result_df.columns:
                result_df = result_df.sort_values(
                    "confidence_score", ascending=False
                ).drop_duplicates(subset=[s_id_filter_col], keep="first")
                log.info(
                    f"Filtered match results to 'Best 1:1'. Kept {len(result_df)} pairs."
                )
            else:
                log.warning(
                    f"Cannot apply 'Best 1:1' filter for match mode: "
                    f"Source ID column '{s_id_filter_col}' not found in result_df columns: {list(result_df.columns)}"
                )
        # --- END NEW LOGIC ---
        log.debug(
            f"process_dataframe [match]: Returning result_df with shape: {result_df.shape if result_df is not None else 'None'}"
        )
        return result_df

    # ==============================================================================
    # DUPLICATE MODE LOGIC
    # ==============================================================================
    elif mode == "duplicate":
        if input_df is None or effective_rec_id_col is None:
            log.error(
                "process_dataframe [duplicate mode]: input_df or rec_id_col is None."
            )
            return pd.DataFrame(
                columns=[
                    f"{effective_rec_id_col}_1" if effective_rec_id_col else "id1",
                    f"{effective_rec_id_col}_2" if effective_rec_id_col else "id2",
                    "confidence_score",
                ]
            )

        log.info("Duplicate mode processing is starting.")

        results_list: List[Dict[str, Any]] = []

        if not input_df.empty:
            if apply_blocking:
                log.debug(f"Applying blocking. Grouping by '{block_col}'.")
                # Set the phase ONCE before processing any blocks
                if run_stats:
                    run_stats.set_phase(ProgressPhase.RUN.value)
                try:
                    grouped_data = input_df.groupby(block_col, observed=True)
                    total_groups = grouped_data.ngroups
                    log.info(f"Found {total_groups} blocks to process.")

                    for i, (group_key, group_df) in enumerate(grouped_data):
                        if len(group_df) < 2:
                            # No possible duplicates in a block of one, just update progress
                            if run_stats:
                                run_stats.update(0, len(group_df), progress_q)
                            continue

                        if len(group_df) > dup_block_limit:
                            log.warning(
                                f"Block '{group_key}' ({len(group_df)} records) exceeds limit {dup_block_limit}, skipping."
                            )
                            # Update progress for the skipped records
                            if run_stats:
                                run_stats.update(0, len(group_df), progress_q)
                            continue

                        # Call the worker for the current block
                        dup_pairs = worker_duplicate(
                            source_df_processed=group_df,
                            config=config,
                            run_stats=run_stats,
                            progress_q=progress_q,
                        )
                        if dup_pairs:
                            results_list.extend(dup_pairs)

                except KeyError:
                    log.error(
                        f"Blocking column '{block_col}' not found in DataFrame for duplicate mode."
                    )

            elif not apply_blocking and processing_mode == "speed":
                log.info("Processing duplicates without blocking (speed mode).")

                # Call the worker on the entire dataframe
                dup_pairs = worker_duplicate(
                    source_df_processed=input_df,
                    config=config,
                    run_stats=run_stats,
                    progress_q=progress_q,
                )
                if dup_pairs:
                    results_list.extend(dup_pairs)

        # Create final DataFrame from collected results
        final_df = (
            pd.DataFrame(results_list)
            if results_list
            else pd.DataFrame(
                columns=[
                    f"{effective_rec_id_col}_1" if effective_rec_id_col else "id1",
                    f"{effective_rec_id_col}_2" if effective_rec_id_col else "id2",
                    "confidence_score",
                ]
            )
        )

        # --- SPRINT A POST-FILTERING LOGIC (already in your code) ---
        if result_mode == "best" and not final_df.empty:
            id1_col_name = (
                f"{effective_rec_id_col}_1" if effective_rec_id_col else "id1"
            )
            if id1_col_name in final_df.columns:
                final_df = final_df.sort_values(
                    "confidence_score", ascending=False
                ).drop_duplicates(subset=[id1_col_name], keep="first")
                log.info(
                    f"Filtered duplicate results to 'Best 1:1'. Kept {len(final_df)} pairs."
                )
            else:
                log.warning(
                    f"Cannot apply 'Best 1:1' filter for duplicate mode: "
                    f"ID1 column '{id1_col_name}' not found in final_df columns: {list(final_df.columns)}"
                )
        # --- END SPRINT A LOGIC ---
        log.debug(
            f"process_dataframe [duplicate]: Returning final_df with shape: {final_df.shape if final_df is not None else 'None'}"
        )
        return final_df


# In src/fmatch/core/engine.py, replace the entire function


def make_blocking_column(
    df: pd.DataFrame,
    cfg: Union[MatchConfig, DedupeConfig],
    df_type: Literal["source", "reference"] = "source",
) -> pd.DataFrame:
    """
    High-level wrapper to apply preprocessing and generate the active blocking key column.
    Extracts necessary parameters from the configuration object.

    Args:
        df: The DataFrame to process.
        cfg: The configuration object (MatchConfig or DedupeConfig).
        df_type: Specifies if the DataFrame is the 'source' or 'reference' file,
                 used only in 'match' mode to select the correct ID column for blocking.
    """
    log.info(f"make_blocking_column called for mode: {cfg.mode}, df_type: {df_type}")

    # Avoid expensive preprocessing/copies if blocking was already disabled upstream.
    # (Also prevents KeyErrors later when _active_block_key_ is absent.)
    if hasattr(cfg, "apply_blocking") and not getattr(cfg, "apply_blocking", True):
        return df

    block_from_col = None
    # NEW LOGIC: Prioritize explicit blocking columns from the strategy
    if hasattr(cfg, "source_block_col") and cfg.source_block_col:
        if df_type == "source":
            block_from_col = cfg.source_block_col
        else:  # reference
            block_from_col = getattr(cfg, "ref_block_col", cfg.source_block_col)
    else:
        # IMPORTANT: Do NOT fall back to ID columns. IDs usually create singleton blocks
        # and will silently yield zero matches/dupes. Instead, infer a reasonable blocking
        # column from mappings (preferred), or disable blocking.

        def _col_rank(col: str, transforms: list[str] | None = None) -> int:
            name = (col or "").lower()
            transforms = transforms or []
            if any(t == "to_domain" for t in transforms):
                return 1000
            if any(k in name for k in ("domain", "website", "url", "hostname", "host")):
                return 950
            if "email" in name:
                return 900
            if any(k in name for k in ("phone", "mobile", "tel")):
                return 850
            if any(k in name for k in ("company", "account", "org", "organization", "organisation", "name")):
                return 800
            if "address" in name:
                return 700
            return 100

        maps = getattr(cfg, "maps", None) or []
        best_idx = None
        best_score = None
        best_source = None
        best_ref = None

        # Exclude ID columns from inference
        src_id = getattr(cfg, "src_id_col", None) if isinstance(cfg, MatchConfig) else None
        ref_id = getattr(cfg, "ref_id_col", None) if isinstance(cfg, MatchConfig) else None
        rec_id = getattr(cfg, "rec_id_col", None) if isinstance(cfg, DedupeConfig) else None

        for i, m in enumerate(maps):
            s_col = m.get("source")
            if not s_col or not isinstance(s_col, str):
                continue
            r_col = m.get("ref") if isinstance(cfg, MatchConfig) else s_col
            if isinstance(cfg, MatchConfig) and (not r_col or not isinstance(r_col, str)):
                continue

            # Skip ID columns entirely
            if rec_id and s_col == rec_id:
                continue
            if src_id and s_col == src_id:
                continue
            if ref_id and r_col == ref_id:
                continue

            transforms = m.get("transforms") or []
            try:
                weight = float(m.get("weight", 1.0))
            except (TypeError, ValueError):
                weight = 1.0
            score = _col_rank(s_col, transforms) + _col_rank(r_col or s_col, transforms)
            score += int(weight * 10)

            if best_score is None or score > best_score:
                best_score = score
                best_idx = i
                best_source = s_col
                best_ref = r_col

        if best_source:
            if isinstance(cfg, MatchConfig):
                cfg.source_block_col = best_source
                cfg.ref_block_col = best_ref or best_source
                log.info(
                    "Inferred blocking columns from maps[%s]: source='%s' reference='%s'",
                    best_idx,
                    cfg.source_block_col,
                    cfg.ref_block_col,
                )
            else:
                cfg.source_block_col = best_source
                log.info(
                    "Inferred blocking column from maps[%s]: source='%s'",
                    best_idx,
                    cfg.source_block_col,
                )

            if df_type == "source":
                block_from_col = best_source
            else:
                block_from_col = best_ref or best_source

    if not block_from_col:
        log.warning(
            "make_blocking_column: No usable blocking column found for mode '%s' and df_type '%s'. "
            "Disabling blocking (engine will fall back to non-blocking matching/dedupe).",
            cfg.mode,
            df_type,
        )
        # Ensure downstream engine does not expect _active_block_key_
        try:
            cfg.apply_blocking = False
        except Exception:
            pass
        return df

    # If the inferred/explicit column does not exist, do not create an all-empty key column.
    # Disabling blocking is safer than silently producing zero matches.
    if block_from_col not in df.columns:
        log.warning(
            "make_blocking_column: Blocking column '%s' not found in df columns. Disabling blocking.",
            block_from_col,
        )
        try:
            cfg.apply_blocking = False
        except Exception:
            pass
        return df

    # Call the main preprocess function. It handles adding the _active_block_key_ column.
    return preprocess(
        df=df,
        block_key_length=cfg.block_key_length,
        ruleset_name=cfg.ruleset_name,
        block_from_col=block_from_col,
        processing_mode=cfg.processing_mode,
        preproc_variant=cfg.preproc_variant,
        strip_b2c_domains=cfg.strip_b2c_domains,
        config=cfg,
    )


def decide_processing_mode(
    src_df: pd.DataFrame,
    ref_df: pd.DataFrame | None = None,
) -> Literal["speed", "scale"]:
    """
    Heuristically pick *speed* vs *scale* using MemoryManager and row thresholds.
    Runtime-aware when available.
    """
    log.debug("decide_processing_mode called.")

    # Use runtime manager if available
    if RUNTIME_AWARE:
        src_rows = len(src_df)
        ref_rows = len(ref_df) if ref_df is not None else 0
        mode = runtime_get_processing_mode(src_rows, ref_rows)
        log.info(
            f"[runtime] Processing mode decision: {mode} (rows={src_rows + ref_rows:,})"
        )
        return mode

    try:
        # Fallback to original logic
        # Initialize MemoryManager
        memory_manager = MemoryManager()

        # Calculate metrics once
        total_rows = len(src_df) + (len(ref_df) if ref_df is not None else 0)
        est_need_bytes = _estimate_df_mem(src_df, ref_df)

        log.debug(f"Total rows: {total_rows}, Est memory: {est_need_bytes/1e6:.1f}MB")

        # Check row count first (fast check)
        if total_rows > SPEED_MAX_ROWS:
            log.info(
                f"Mode decision: 'scale' (row count {total_rows} > {SPEED_MAX_ROWS})"
            )
            return "scale"

        # Check memory constraints
        if memory_manager.should_use_chunking(est_need_bytes):
            log.info(
                f"Mode decision: 'scale' (memory constraint: {est_need_bytes/1e6:.1f}MB)"
            )
            return "scale"

        # Both checks passed - use speed mode
        log.info("Mode decision: 'speed' (low rows + sufficient memory)")
        return "speed"

    except Exception as e:
        log.warning(
            f"Error in mode decision, defaulting to 'scale': {e}", exc_info=True
        )
        return "scale"


# ---------------------------------------------------------------------------+
# Back-compat             determine_optimal_blocking_strategy                |
# ---------------------------------------------------------------------------+
# * Accepts all historical calling conventions from the GUI and API tests.
# * Maps everything to the canonical `_determine_optimal_blocking_strategy`.
# ---------------------------------------------------------------------------+
def determine_optimal_blocking_strategy(
    src_df: pd.DataFrame,
    ref_df: Optional[pd.DataFrame] = None,
    mappings: Optional[list[dict]] = None,
    mode: "BlockingMode | str" = BlockingMode.DEDUPE,
    ruleset: str = DEFAULT_RULESET_NAME,
    **kw: Any,
) -> dict:
    """
    Universal facade that understands all historical call signatures from the GUI and API.
    This version uses explicit keyword arguments to avoid ambiguity and prevent errors.
    """
    # --- 1. Argument Validation ---
    if not isinstance(src_df, pd.DataFrame):
        raise TypeError(
            f"determine_optimal_blocking_strategy: `src_df` must be a pandas DataFrame, "
            f"but got type {type(src_df).__name__}."
        )

    if ref_df is not None and not isinstance(ref_df, pd.DataFrame):
        raise TypeError(
            f"determine_optimal_blocking_strategy: `ref_df` must be a pandas DataFrame or None, "
            f"but got type {type(ref_df).__name__}."
        )

    if mappings is not None and not isinstance(mappings, list):
        raise TypeError(
            f"determine_optimal_blocking_strategy: `mappings` must be a list of dicts or None, "
            f"but got type {type(mappings).__name__}."
        )

    # --- 2. Handle legacy keywords from kwargs for backward compatibility ---
    # This allows old callers using 'df_src' or 'restrict_to_pairs' to still work.
    if "df_src" in kw and isinstance(kw["df_src"], pd.DataFrame):
        src_df = kw["df_src"]

    if "restrict_to_pairs" in kw and isinstance(kw["restrict_to_pairs"], list):
        mappings = [{"source": s, "ref": r} for s, r in kw["restrict_to_pairs"]]

    # --- 3. Delegate to the core implementation ---
    # The core function's parameter names are now aligned for clarity.
    kw.setdefault("stats_cache", ColumnStatsCache())

    return _determine_optimal_blocking_strategy(
        df_src=src_df,
        ref_df=ref_df,
        mappings=mappings,
        mode=mode,
        ruleset=ruleset,
        **kw,
    )


# --- REVISED: Now handles both DuckDB files and in-memory SQLite ---
def _init_score_cache_db(path: str):
    """
    Initializes a score cache. Uses DuckDB for file-based caches
    and a simple SQLite database for in-memory testing.
    """
    if not path:
        log.warning("Cache path not provided, skipping score_cache DB initialization.")
        return

    # If the test suite requests an in-memory DB, use the fast and safe SQLite version.
    if path == ":memory:":
        log.info("Initializing IN-MEMORY score cache for testing using SQLite.")
        _get_sqlite_inmem_conn()  # This ensures the table is created.
        return

    # Otherwise, proceed with the production DuckDB implementation for file-based caches.
    log.info(f"Initializing file-based score cache at: {path}")
    db_dir = os.path.dirname(path)
    if db_dir:
        os.makedirs(db_dir, exist_ok=True)

    try:
        with closing(duckdb.connect(database=str(path), read_only=False)) as conn:
            conn.execute(
                f"""
                CREATE TABLE IF NOT EXISTS {_SCORE_CACHE_TABLE} (
                    a TEXT, b TEXT, algorithm TEXT, ruleset TEXT, score SMALLINT,
                    PRIMARY KEY (a, b, algorithm, ruleset)
                )
                """
            )
            conn.execute(
                f"""
                CREATE UNIQUE INDEX IF NOT EXISTS score_cache_idx
                ON {_SCORE_CACHE_TABLE} (a, b, algorithm, ruleset);
                """
            )
        log.info(
            f"Score cache table '{_SCORE_CACHE_TABLE}' and index are ready in db: {path}"
        )
    except Exception as e:
        log.error(
            f"Failed to initialize score_cache table in {path}: {e}", exc_info=True
        )


# Ã¢â€â‚¬Ã¢â€â‚¬ DuckDB-based RapidFuzz score cache Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
_CACHE_ENABLED = True  # turn caching on/off globally
_SCORE_CACHE_TABLE = "__rf_score_cache__"  # table name inside DuckDB


# --- NEW: SQLite in-memory helper for tests ---
@lru_cache(maxsize=1)
def _get_sqlite_inmem_conn():
    """
    Returns a singleton, in-memory, thread-safe SQLite connection for tests.
    This avoids the Windows crash by not using DuckDB in the test hot-path.
    """
    log.debug("Creating singleton in-memory SQLite connection for testing.")
    # The connection is cached, so this block only runs once per process.
    conn = sqlite3.connect(":memory:", check_same_thread=False, isolation_level=None)
    conn.execute(f"""
        CREATE TABLE IF NOT EXISTS {_SCORE_CACHE_TABLE} (
            a TEXT, b TEXT, algorithm TEXT, ruleset TEXT,
            score SMALLINT,
            PRIMARY KEY (a, b, algorithm, ruleset)
        );
    """)
    conn.execute(f"""
        CREATE UNIQUE INDEX IF NOT EXISTS score_cache_idx
        ON {_SCORE_CACHE_TABLE} (a, b, algorithm, ruleset);
    """)
    return conn


# --- Core Scoring Logic (for LRU cache) ---
@lru_cache(maxsize=250_000)
def _compute_score(s1: str, s2: str, algo: str) -> int:
    """
    This is the pure, non-database scoring logic. It remains unchanged.
    It takes two strings and an algorithm, and returns a score from 0-100.
    """
    s1_str, s2_str = str(s1 or ""), str(s2 or "")
    if not s1_str or not s2_str:
        return 0

    scorer_func = ALGORITHM_MAP.get(algo)
    if not scorer_func:
        log.warning(f"Unknown algorithm '{algo}' in _compute_score. Returning 0.")
        return 0

    try:
        score_val = scorer_func(s1_str, s2_str)
        return max(0, min(100, int(round(score_val))))
    except Exception as e:
        log.warning(f"Score computation failed ({algo}) for '{s1}' vs '{s2}': {e}")
        return 0


def _rf_pair_scores(left: Sequence[str], right: Sequence[str], scorer) -> np.ndarray:
    """Compute pairwise scores for aligned string sequences."""
    return np.fromiter(
        (scorer(a, b) for a, b in zip(left, right)), dtype=np.float32, count=len(left)
    )


def vector_score(
    candidate_chunk: pd.DataFrame, config: Dict[str, Any]
) -> Optional[pd.DataFrame]:
    """Score a chunk of candidate pairs using element-wise vectorization."""
    pid = os.getpid()
    if candidate_chunk.empty:
        log.debug(
            f"[Worker {pid}] Empty candidate chunk received; skipping vector_score."
        )
        return None

    maps_for_algo = (
        config.get("maps") or config.get("mappings") or config.get("field_mappings")
    )
    if not maps_for_algo:
        log.warning(f"[Worker {pid}] No mappings supplied; skipping vector_score.")
        return None

    algo_key = config.get("algorithm", DEFAULT_ALGORITHM_KEY)
    scorer_callable = ALGORITHM_MAP.get(algo_key, rf_fuzz.WRatio)

    threshold_raw = config.get("threshold", 80)
    try:
        threshold = float(threshold_raw)
    except (TypeError, ValueError):
        threshold = 80.0
    if threshold <= 1.0:
        threshold *= 100.0
    threshold = int(min(100, max(0, threshold)))

    num_rows = len(candidate_chunk)
    total_weighted_scores = np.zeros(num_rows, dtype=np.float32)
    total_weights_used = np.zeros(num_rows, dtype=np.float32)
    total_weight_sum = 0.0

    for mapping in maps_for_algo:
        source_label = (
            mapping.get("source")
            or mapping.get("source_column")
            or mapping.get("source_field")
        )
        reference_label = (
            mapping.get("reference")
            or mapping.get("ref")
            or mapping.get("reference_column")
            or mapping.get("reference_field")
            or mapping.get("target")
        )
        if not source_label or not reference_label:
            continue

        s_col = f"s_{sanitize_column_name(source_label)}"
        r_col = f"r_{sanitize_column_name(reference_label)}"
        if s_col not in candidate_chunk.columns or r_col not in candidate_chunk.columns:
            continue

        try:
            weight = float(mapping.get("weight", 1.0))
        except (TypeError, ValueError):
            weight = 1.0
        if weight <= 0:
            continue
        total_weight_sum += weight

        s_series = candidate_chunk[s_col].fillna("").astype(str).str.strip()
        r_series = candidate_chunk[r_col].fillna("").astype(str).str.strip()
        s_values = s_series.to_numpy()
        r_values = r_series.to_numpy()

        valid_mask = (s_values != "") & (r_values != "")
        if not valid_mask.any():
            continue

        pair_scores = np.zeros(num_rows, dtype=np.float32)
        scores_valid = _rf_pair_scores(
            s_values[valid_mask].tolist(),
            r_values[valid_mask].tolist(),
            scorer_callable,
        )
        pair_scores[valid_mask] = scores_valid

        total_weighted_scores += pair_scores * weight
        total_weights_used += valid_mask.astype(np.float32) * weight

    if total_weight_sum <= 0:
        log.debug(f"[Worker {pid}] No positive mapping weights; skipping chunk.")
        return None

    if not np.any(total_weights_used):
        log.debug(f"[Worker {pid}] No valid scores computed in chunk.")
        return None

    final_scores = np.divide(
        total_weighted_scores,
        total_weights_used,
        out=np.zeros(num_rows, dtype=np.float32),
        where=(total_weights_used != 0),
    )

    passing_mask = final_scores >= threshold
    if not passing_mask.any():
        return None

    passing_rows = candidate_chunk.loc[passing_mask, ["source_id", "ref_id"]]
    return pd.DataFrame(
        {
            "source_id": passing_rows["source_id"].to_numpy(),
            "ref_id": passing_rows["ref_id"].to_numpy(),
            "confidence_score": final_scores[passing_mask],
        }
    )


def _get_worker_db_write_conn(db_path: str) -> Optional[duckdb.DuckDBPyConnection]:
    """
    Gets or creates ONE READ-WRITE DuckDB connection stored in threading.local
    for the current worker process for cache writes.
    """
    if mp.current_process().name != "MainProcess":
        return None

    pid = os.getpid()
    conn_attr = (
        f"conn_rw_{pid}"  # Use a different attribute name for the write connection
    )

    conn = getattr(_WORKER_DB_CONN_CACHE, conn_attr, None)

    if conn:
        try:
            conn.execute("SELECT 42;")
            return conn
        except (DuckDBError, ConnectionException, RuntimeError) as e:
            log.warning(
                f"[Worker {pid}] Write connection lost ({type(e).__name__}). Reconnecting."
            )
            try:
                conn.close()
            except Exception:
                pass
            setattr(_WORKER_DB_CONN_CACHE, conn_attr, None)

    if db_path:
        try:
            log.info(f"[Worker {pid}] Establishing new RW DB connection to {db_path}")
            conn = duckdb.connect(database=db_path, read_only=False)
            setattr(_WORKER_DB_CONN_CACHE, conn_attr, conn)
            return conn
        except Exception as e:
            log.error(
                f"[Worker {pid}] Failed to init/connect DuckDB RW cache at {db_path}: {e}",
                exc_info=True,
            )

    return None


def _get_worker_db_conn(db_path: str) -> Optional[duckdb.DuckDBPyConnection]:
    """
    Gets or creates ONE read-only DuckDB connection stored in threading.local
    for the current worker process.
    """
    pid = os.getpid()
    conn_attr = f"conn_ro_{pid}"

    conn = getattr(_WORKER_DB_CONN_CACHE, conn_attr, None)

    if conn:
        try:
            conn.execute("SELECT 1;")
            return conn
        except (DuckDBError, ConnectionException, RuntimeError):
            try:
                conn.close()
            except Exception:
                pass
            setattr(_WORKER_DB_CONN_CACHE, conn_attr, None)

    if db_path:
        try:
            conn = duckdb.connect(database=db_path, read_only=True)
            setattr(_WORKER_DB_CONN_CACHE, conn_attr, conn)
            return conn
        except Exception as e:
            log.debug(f"[Worker {pid}] Failed to open RO cache at {db_path}: {e}")

    return None


# --- DuckDB Worker Connection Management ---


# --------------------------------------------------------------------------- #
#  Custom phonetic helper Ã¢â‚¬â€œ American Soundex (4-char code)
# --------------------------------------------------------------------------- #
_RE_SOUNDEX_NON_ALPHA = re.compile(r"[^A-Za-z]")


@lru_cache(maxsize=1024)  # Adjust maxsize as appropriate
def soundex(name: str) -> str | None:
    """
    Rough phonetic encoding. Returns a 4-character Soundex code
    or ``None`` if the input contains no alphabetic characters.
    """
    # Normalize text while preserving non-Latin scripts (replaces unidecode)
    name_str = normalize_text(str(name))
    name_str = _RE_SOUNDEX_NON_ALPHA.sub("", name_str)
    if not name_str:
        return None

    s_code = name_str[0].upper()
    name_str = name_str[1:].lower()

    # Code mappings for consonants after the first letter
    codes = {
        "1": "bfpv",
        "2": "cgjkqsxz",
        "3": "dt",
        "4": "l",
        "5": "mn",
        "6": "r",
    }

    # Invert the map for quick lookup
    char_to_code = {char: code for code, chars in codes.items() for char in chars}

    last_code = char_to_code.get(s_code.lower())

    for char in name_str:
        code = char_to_code.get(char)
        # Add code if it's different from the last one and not a vowel-like separator
        if code and code != last_code:
            s_code += code
            if len(s_code) == 4:
                break
        # Update last_code only if it's not from a vowel separator (h, w)
        if char not in "hw":
            last_code = code

    # Pad with zeros if necessary
    s_code += "000"
    return s_code[:4]


# --------------------------------------------------------------------------- #
#  Algorithm registry Ã¢â‚¬â€œ maps external names -> callables
# --------------------------------------------------------------------------- #
def _normalize_domain_value(val: str) -> str:
    """Lightweight domain normalization for algorithm registry."""
    dom = str(val or "").strip().lower()
    if "://" in dom:
        dom = dom.split("://", 1)[1]
    if dom.startswith("www."):
        dom = dom[4:]
    dom = dom.split("/", 1)[0]
    dom = dom.split(":", 1)[0]
    return dom


def _base_domain(val: str) -> str:
    parts = val.split(".")
    return ".".join(parts[-2:]) if len(parts) >= 2 else val


def _domain_score(s1, s2, **kwargs) -> int:
    dom_l = _normalize_domain_value(s1)
    dom_r = _normalize_domain_value(s2)
    if not dom_l or not dom_r:
        return 0
    if dom_l == dom_r:
        return 100
    if _base_domain(dom_l) == _base_domain(dom_r):
        return 95
    return int(round(rf_fuzz.ratio(dom_l, dom_r)))


ALGORITHM_MAP: Dict[str, Any] = {
    "WRatio": rf_fuzz.WRatio,
    "QRatio": rf_fuzz.QRatio,
    "Ratio": rf_fuzz.ratio,
    "PartialRatio": rf_fuzz.partial_ratio,
    "TokenSetRatio": rf_fuzz.token_set_ratio,
    "TokenSortRatio": rf_fuzz.token_sort_ratio,
    "Jaro": Jaro.similarity,
    "JaroWinkler": JaroWinkler.similarity,
    "Soundex": lambda a, b, **kwargs: 100 if soundex(a) == soundex(b) else 0,
    "Exact": lambda s1, s2, **kwargs: 100 if s1 == s2 else 0,
    "Domain": _domain_score,
    "ExactDomain": _domain_score,
}

# --- Back-compat for rapidfuzz v2.x (Jaro/JaroWinkler now return 0-1) ---


def _percent(scorer):
    @functools.wraps(scorer)
    def wrapper(s1, s2, *args, **kwargs):
        return int(round(scorer(s1, s2, *args, **kwargs) * 100))

    return wrapper


ALGORITHM_MAP["Jaro"] = _percent(Jaro.similarity)
ALGORITHM_MAP["JaroWinkler"] = _percent(JaroWinkler.similarity)

# --- End back-compat ---

ALGORITHM_DISPLAY_NAMES = list(ALGORITHM_MAP)  # GUI needs this

# In engine.py

# Complete fix for smooth progress and CPS display
# Replace the entire RunStats class with this version


def _safe_str(v: object, preserve_none_strings: bool = False) -> str:
    """
    Enhanced string sanitization for fuzzy matching.
    Handles None, pandas NA, bytes, and datetime objects.
    """
    if v is None:
        return ""

    # Handle pandas NA, which is not None but evaluates to True for pd.isna()
    if pd.isna(v):
        return ""

    # Handle bytes by attempting to decode
    if isinstance(v, bytes):
        try:
            return v.decode("utf-8", errors="ignore").strip()
        except Exception:
            return ""

    # Handle datetime objects by converting to a standard string format
    if isinstance(v, (datetime, date)):
        return v.isoformat()

    s = str(v).strip()

    # Conditionally treat "none" strings as empty
    if not preserve_none_strings and s.lower() in {"nan", "none", "null"}:
        return ""

    # Use script-aware normalization instead of naive lowercasing
    return normalize_text(s)


# Phase 3 Performance Optimizations
def vectorized_safe_str(series: pd.Series) -> pd.Series:
    """Vectorized version of _safe_str with script-aware normalization."""
    result = series.fillna("").astype(str).str.strip()
    na_mask = result.isin(["nan", "none", "null", ""])
    result.loc[na_mask] = ""

    # Fast path: detect CJK/Cyrillic and separate processing
    mask_special = result.str.contains(
        r"[\u3400-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF\u0400-\u04FF]",
        regex=True,
        na=False,
    )

    # Latin-only fast path: strip accents and casefold
    if mask_special.any():
        result.loc[~mask_special] = (
            result.loc[~mask_special].map(_strip_accents_latin).str.casefold()
        )
        result.loc[mask_special] = result.loc[mask_special].map(
            lambda x: _ud.normalize("NFKC", x).casefold() if x else ""
        )
    else:
        # All Latin - use fast vectorized path
        result = result.map(_strip_accents_latin).str.casefold()

    return result


# Add this entire new function
def _is_domain_column(column_name: str) -> bool:
    """
    Simple helper to check if a column likely contains domain or email data.
    """
    if not isinstance(column_name, str):
        return False

    col_lower = column_name.lower()
    domain_keywords = ["domain", "website", "url", "site", "email"]

    return any(keyword in col_lower for keyword in domain_keywords)


# --- DuckDB Caching Function ---


# Helper for logging potentially long strings
def _log_str_repr(s: str, max_len: int = 50) -> str:
    """Returns a possibly truncated string representation for logging."""
    s_str = str(s)  # Ensure it's a string
    if len(s_str) > max_len:
        return s_str[: max_len - 3] + "..."
    return s_str


def robust_score_calculation(
    s1: str, s2: str, algo: str, fallback_algo: str = "Ratio", **kwargs
) -> int:
    """Score calculation with fallback and retry logic"""
    try:
        score = cached_score(s1=s1, s2=s2, algo=algo, **kwargs)
        if score is not None:
            return score
        # If score is None, it's a failure case that should trigger the fallback.
        raise ValueError(f"Primary algo {algo} returned None, indicating a problem.")
    except Exception as e:
        log.warning(f"Primary algo {algo} failed for '{str(s1)[:20]}...': {e}")
        try:
            # Try fallback algorithm
            fallback_score = cached_score(s1=s1, s2=s2, algo=fallback_algo, **kwargs)
            if fallback_score is not None:
                return fallback_score
            raise ValueError(f"Fallback algo {fallback_algo} returned None.")
        except Exception as e2:
            log.error(f"Fallback algo {fallback_algo} also failed: {e2}")
            # Last resort - simple exact match
            s1_str = str(s1 or "")
            s2_str = str(s2 or "")
            return 100 if s1_str == s2_str else 0


# Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
#  Score helper Ã¢â‚¬â€œ now truly Ã¢â‚¬Å“speed-awareÃ¢â‚¬Â
# Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
# Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
#  Score helper Ã¢â‚¬â€œ now truly Ã¢â‚¬Å“speed-awareÃ¢â‚¬Â
# Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
def cached_score(
    *,
    s1: str,
    s2: str,
    algo: str = DEFAULT_ALGORITHM_KEY,
    processing_mode: Literal["speed", "scale"] = "scale",
    ruleset_name: str = DEFAULT_RULESET_NAME,
    db_path: str | None = None,
) -> int | None:
    """
    Returns an integer similarity score for (s1, s2).
    - In "speed" mode, it relies on an in-memory LRU cache.
    - In "scale" mode, it uses a database cache (SQLite for tests, DuckDB for prod).
    """
    # In "speed" mode, we bypass all database logic and use the fast LRU cache.
    if processing_mode == "speed":
        return _compute_score(s1, s2, algo)

    # --- Try using global ScoreCache if available (from worker pool) ---
    global _GLOBAL_SCORE_CACHE
    if _GLOBAL_SCORE_CACHE is not None:
        try:
            # Try to get from cache
            cached_val = _GLOBAL_SCORE_CACHE.get(algo, s1, s2)
            if cached_val is not None:
                return int(cached_val)

            # Cache miss - compute and store
            result = _compute_score(s1, s2, algo)
            if result is not None:
                _GLOBAL_SCORE_CACHE.set(algo, s1, s2, float(result))
            return result
        except Exception as e:
            log.debug(f"ScoreCache error, falling back to direct computation: {e}")
            return _compute_score(s1, s2, algo)

    # --- Database Caching Logic for "scale" mode ---
    if not db_path:
        # If no db_path is provided, compute live without caching.
        return _compute_score(s1, s2, algo)

    cache_s1, cache_s2 = sorted((str(s1 or ""), str(s2 or "")))

    # Use the safe, in-memory SQLite path for tests
    if db_path == ":memory:":
        conn = _get_sqlite_inmem_conn()
        cur = conn.cursor()
        cur.execute(
            f"SELECT score FROM {_SCORE_CACHE_TABLE} WHERE a=? AND b=? AND algorithm=? AND ruleset=?",
            (cache_s1, cache_s2, algo, ruleset_name),
        )
        row = cur.fetchone()
        if row:
            return row[0]

        # Cache miss: compute and store
        result = _compute_score(s1, s2, algo)
        cur.execute(
            f"INSERT OR REPLACE INTO {_SCORE_CACHE_TABLE} VALUES (?,?,?,?,?)",
            (cache_s1, cache_s2, algo, ruleset_name, result),
        )
        return result

    # Use the high-performance DuckDB path for production file-based caches
    else:
        try:
            ro_conn = _get_worker_db_conn(db_path)
            if ro_conn:
                res = ro_conn.execute(
                    f"SELECT score FROM {_SCORE_CACHE_TABLE} WHERE a=? AND b=? AND algorithm=? AND ruleset=?",
                    [cache_s1, cache_s2, algo, ruleset_name],
                ).fetchone()

                if res and res[0] is not None:
                    return int(res[0])
        except Exception as read_exc:
            log.debug(
                f"DuckDB cache read failed: {read_exc} Ã¢â‚¬â€œ will compute live."
            )

        # Cache miss: compute and write to DuckDB cache
        result = _compute_score(s1, s2, algo)
        try:
            write_conn = _get_worker_db_write_conn(db_path)
            if write_conn:
                write_conn.execute(
                    f"INSERT OR IGNORE INTO {_SCORE_CACHE_TABLE} VALUES (?, ?, ?, ?, ?)",
                    [cache_s1, cache_s2, algo, ruleset_name, result],
                )
        except Exception as write_exc:
            log.warning(
                f"Failed to write to DuckDB score cache '{db_path}': {write_exc}"
            )

        return result


# --- Rule Loading & Preprocessing ---


def load_ruleset(filepath: Path) -> Optional[Dict]:  # Return full dict now
    """Loads and validates a single ruleset YAML file."""
    try:
        with open(filepath, "r", encoding="utf-8") as f:
            ruleset_data = yaml.safe_load(f)

        # --- Validation ---
        if not isinstance(ruleset_data, dict):
            log.warning(
                f"Ruleset file '{filepath.name}' should contain a dictionary, got {type(ruleset_data)}. Skipping."
            )
            return None
        if "name" not in ruleset_data:
            log.warning(
                f"Ruleset file '{filepath.name}' is missing the required 'name' key. Skipping."
            )
            return None
        # 'rules' key is optional, 'blocking' or 'block_field' is now more important for engine # type: ignore
        # Add more validation as needed (e.g., check types of nested elements)

        log.debug(
            f"Successfully parsed and validated: {filepath.name}"
        )  # Changed from log.info
        return ruleset_data  # Return the whole dict
    except FileNotFoundError:
        log.error(f"Ruleset file not found: {filepath}")
        return None
    except yaml.YAMLError as e:
        log.error(f"Error parsing YAML in {filepath.name}: {e}")
        return None
    except Exception as e:
        log.error(f"Unexpected error loading ruleset {filepath.name}: {e}")
        return None


def load_ruleset_dir(ruleset_dir: Union[str, Path]):
    """Loads all .yaml rulesets from a directory into the global cache."""
    global AVAILABLE_RULESETS
    rule_dir_path = Path(ruleset_dir)  # type: ignore
    log.debug(
        f"Attempting to load rulesets from directory: {rule_dir_path.resolve()}"
    )  # Changed from log.info
    if not rule_dir_path.is_dir():
        log.warning(f"Ruleset directory not found: {rule_dir_path.resolve()}")
        return

    loaded_files = []
    new_rulesets = {}
    try:
        yaml_files = list(rule_dir_path.glob("*.yaml")) + list(
            rule_dir_path.glob("*.yml")
        )  # type: ignore
        log.debug(
            f"Glob found {len(yaml_files)} potential files in {rule_dir_path.resolve()}: {[f.name for f in yaml_files]}"
        )  # Changed from log.info
        for filepath in yaml_files:
            log.debug(f"Processing file: {filepath.name}")  # Changed from log.info
            ruleset_content = load_ruleset(filepath)  # type: ignore # Returns the full dict or None
            if ruleset_content is not None and "name" in ruleset_content:
                ruleset_name = ruleset_content["name"]  # Use name from inside the file
                new_rulesets[ruleset_name] = ruleset_content  # Store the whole dict
                loaded_files.append(filepath.name)
            else:
                log.warning(
                    f"Skipping file {filepath.name} due to load error or missing 'name' key."
                )
    except Exception as e:
        log.error(
            f"Error scanning ruleset directory {rule_dir_path.resolve()}: {e}",
            exc_info=True,
        )

    with _RULESET_LOCK:
        # Add 'none' ruleset placeholder
        AVAILABLE_RULESETS = {
            "none": {
                "name": "none",
                "rules": [],
                "description": "No preprocessing rules applied.",
            }
        }
        AVAILABLE_RULESETS.update(new_rulesets)  # type: ignore
        log.debug(
            f"Files processed in {rule_dir_path.resolve()}: {loaded_files}"
        )  # Changed from log.info
        log.debug(
            f"Rulesets loaded by load_ruleset_dir. Count: {len(AVAILABLE_RULESETS)-1}. Available: {list(AVAILABLE_RULESETS.keys())}"
        )  # Changed from log.info


# NEW HELPER FUNCTIONS START
def _rm_punct(series: pd.Series) -> pd.Series:
    if series.empty:
        return series
    return (
        series.astype(str)
        .str.replace(_RE_PUNCT, "", regex=True)
        .replace("", pd.NA)
        .astype("string")
    )


def _to_ascii(series: pd.Series) -> pd.Series:
    if series.empty:
        return series

    # Option 1: Using unidecode (recommended for better results)
    # Make sure to add 'unidecode' to your project's dependencies.
    # try:
    #     import unidecode
    #     # Apply unidecode, ensure it handles NaNs gracefully if astype(str) converts them to "nan"
    #     return series.astype(str).apply(lambda x: unidecode.unidecode(x) if pd.notna(x) and x != '' else x).replace('', pd.NA).astype('string')
    # except ImportError:
    #     log.warning("unidecode library not found for 'to_ascii' rule. Using basic replacement. "
    #                 "Install with: pip install unidecode")
    #     # Fall through to basic replacement if unidecode is not available

    # Option 2: Basic replacement (fallback or if unidecode is not used)
    replacements = {
        "ÃƒÂ¡": "a",
        "ÃƒÂ©": "e",
        "ÃƒÂ­": "i",
        "ÃƒÂ³": "o",
        "ÃƒÂº": "u",
        "ÃƒÂ¼": "u",
        "ÃƒÂ§": "c",
        "ÃƒÂ": "A",
        "Ãƒâ€°": "E",
        "ÃƒÂ": "I",
        "Ãƒâ€œ": "O",
        "ÃƒÅ¡": "U",
        "ÃƒÅ“": "U",
        "Ãƒâ€¡": "C",
        "ÃƒÂ ": "a",
        "ÃƒÂ¨": "e",
        "ÃƒÂ¬": "i",
        "ÃƒÂ²": "o",
        "ÃƒÂ¹": "u",
        "Ãƒâ‚¬": "A",
        "ÃƒË†": "E",
        "ÃƒÅ’": "I",
        "Ãƒâ€™": "O",
        "Ãƒâ„¢": "U",
        "ÃƒÂ¤": "a",
        "ÃƒÂ«": "e",
        "ÃƒÂ¯": "i",
        "ÃƒÂ¶": "o",
        "Ãƒâ€ž": "A",
        "Ãƒâ€¹": "E",
        "ÃƒÂ": "I",
        "Ãƒâ€“": "O",
        "ÃƒÂ¢": "a",
        "ÃƒÂª": "e",
        "ÃƒÂ®": "i",
        "ÃƒÂ´": "o",
        "ÃƒÂ»": "u",
        "Ãƒâ€š": "A",
        "ÃƒÅ ": "E",
        "ÃƒÅ½": "I",
        "Ãƒâ€": "O",
        "Ãƒâ€º": "U",
        "ÃƒÂ±": "n",
        "Ãƒâ€˜": "N",
        # Add more common accented characters as needed
    }

    def replace_accents(text: str) -> str:
        if not isinstance(text, str) or text == "":  # Handle NaNs or empty strings
            return (
                pd.NA
            )  # Return pandas NA for empty or non-string inputs after astype(str)
        for accented_char, unaccented_char in replacements.items():
            text = text.replace(accented_char, unaccented_char)
        return text

    # Apply replacement, ensure NaNs are handled consistently
    return series.astype(str).apply(replace_accents).replace("", pd.NA).astype("string")


# NEW HELPER FUNCTIONS END


def apply_rules(df: pd.DataFrame, rules: List[Dict[str, Any]]) -> pd.DataFrame:
    """Applies a list of preprocessing rules to a DataFrame."""
    if not rules:
        return df

    df_processed = df.copy()
    # Cache for series transformations. This helps if a rule's output is used
    # as input for a subsequent rule acting on the same (potentially renamed) column.
    column_cache: Dict[str, pd.Series] = {}

    for rule_idx, rule in enumerate(rules):
        action = rule.get("action", "").lower()

        # 'columns_spec' is what comes from the rule definition.
        # It's expected to be a list of actual column names by the time it's used in this loop.
        # Resolution of keywords like "__ALL_STRINGS__" should happen in the calling function (e.g., engine.preprocess).
        columns_to_process_for_this_rule = rule.get("columns", [])

        params = rule.get("params", {})
        output_col_template = rule.get(
            "output", "{col}"
        )  # Default is to modify in-place or use sanitized input col name

        if not isinstance(columns_to_process_for_this_rule, list):
            if isinstance(
                columns_to_process_for_this_rule, str
            ):  # Allow single column string
                columns_to_process_for_this_rule = [columns_to_process_for_this_rule]
            else:
                log.warning(
                    f"Rule #{rule_idx} (action: '{action}') has invalid 'columns' specification: "
                    f"{columns_to_process_for_this_rule}. Skipping rule."
                )
                continue

        if not columns_to_process_for_this_rule:
            log.warning(
                f"Rule #{rule_idx} (action: '{action}') has no columns specified. Skipping rule."
            )
            continue

        # Placeholder for 'transforms' logic if it becomes more complex
        # transforms = rule.get("transforms")
        # if transforms:
        #     log.warning("Rule 'transforms' are not fully processed by this version of apply_rules.")

        for col_name_from_spec in columns_to_process_for_this_rule:
            # Ensure the column currently exists in df_processed
            if col_name_from_spec not in df_processed.columns:
                log.warning(
                    f"Rule action '{action}' skipped for column '{col_name_from_spec}': Column not found in DataFrame."
                )
                continue

            # Determine the actual series to work on (from cache or df_processed)
            # Always convert to pandas 'string' dtype for text operations, handling NaNs.
            if col_name_from_spec in column_cache:
                current_series = column_cache[col_name_from_spec]
                # Ensure it's string type if fetched from cache
                if (
                    not pd.api.types.is_string_dtype(current_series)
                    and not current_series.empty
                ):
                    current_series = current_series.fillna(pd.NA).astype(
                        "string", copy=False
                    )
            else:
                current_series = (
                    df_processed[col_name_from_spec]
                    .fillna(pd.NA)
                    .astype("string", copy=False)
                )

            # Determine the output column name
            # If output_col_template is "{col}", it implies modifying the original column or its sanitized version.
            # If a specific output name is given in YAML (e.g. for 'block'), that should be used.
            final_output_col_name = output_col_template.format(
                col=sanitize_column_name(col_name_from_spec)
            )
            if (
                action == "block" and "output_name" in params
            ):  # Specific handling for block's output
                final_output_col_name = params["output_name"]
            elif (
                output_col_template == "{col}"
            ):  # Modifying the original column (or its sanitized name if that's how template works)
                final_output_col_name = col_name_from_spec  # Prefer original name if template implies in-place

            processed_series_for_action: Optional[pd.Series] = None

            try:
                if action == "lower":
                    processed_series_for_action = current_series.str.lower()
                elif action == "upper":
                    processed_series_for_action = current_series.str.upper()
                elif action == "strip":
                    processed_series_for_action = current_series.str.strip()
                elif action == "collapse_ws":
                    # First strip leading/trailing whitespace, then replace internal multiple spaces
                    processed_series_for_action = (
                        current_series.str.strip().str.replace(r"\s+", " ", regex=True)
                    )
                    log.debug(
                        f"Applied 'collapse_ws' to column '{col_name_from_spec}' -> '{final_output_col_name}'"
                    )
                elif action == "remove_chars":
                    chars_to_remove = params.get("chars", "")
                    processed_series_for_action = current_series.str.replace(
                        f"[{re.escape(chars_to_remove)}]", "", regex=True
                    )
                elif action == "remove_regex":
                    regex_pattern_to_remove = params.get("pattern", "")
                    processed_series_for_action = current_series.str.replace(
                        regex_pattern_to_remove, "", regex=True
                    )
                elif action == "replace_regex":
                    regex_pattern_to_find = params.get("pattern", "")
                    replacement_string = params.get("repl", "")
                    processed_series_for_action = current_series.str.replace(
                        regex_pattern_to_find, replacement_string, regex=True
                    )
                elif action == "fillna":
                    value_to_fill_with = params.get("value", "")
                    # This action modifies df_processed directly for the specified input column.
                    # The output column is the same as the input column for fillna.
                    final_output_col_name = col_name_from_spec
                    df_processed[final_output_col_name] = df_processed[
                        col_name_from_spec
                    ].fillna(value_to_fill_with)
                    # Update current_series to reflect this change for potential caching
                    current_series = (
                        df_processed[final_output_col_name]
                        .fillna(pd.NA)
                        .astype("string", copy=False)
                    )
                    processed_series_for_action = current_series
                elif action == "slice":
                    start_index_param = params.get("start")
                    stop_index_param = params.get("stop")
                    start_idx = (
                        int(start_index_param)
                        if start_index_param is not None
                        else None
                    )
                    stop_idx = (
                        int(stop_index_param) if stop_index_param is not None else None
                    )
                    processed_series_for_action = current_series.str.slice(
                        start_idx, stop_idx
                    )
                elif action == "block":
                    key_size = int(params.get("size", 3))
                    # Output name for block action is handled by final_output_col_name logic above
                    temp_series_for_block_key = current_series.str.lower().str.replace(
                        r"[^\w]", "", regex=True
                    )  # Keep only alphanumeric
                    processed_series_for_action = temp_series_for_block_key.str.slice(
                        0, key_size
                    )
                    log.debug(
                        f"Created block column '{final_output_col_name}' from '{col_name_from_spec}' size {key_size}"
                    )
                elif action == "remove_punctuation":
                    processed_series_for_action = _rm_punct(current_series)
                    log.debug(
                        f"Applied 'remove_punctuation' to column '{col_name_from_spec}' -> '{final_output_col_name}'"
                    )
                elif action == "to_ascii":
                    processed_series_for_action = _to_ascii(current_series)
                    log.debug(
                        f"Applied 'to_ascii' to column '{col_name_from_spec}' -> '{final_output_col_name}'"
                    )
                else:
                    log.warning(
                        f"Unknown rule action '{action}' for column '{col_name_from_spec}'. Series remains unchanged."
                    )
                    processed_series_for_action = current_series  # Assign original series (already stringified and NA handled)

                # Update DataFrame and cache
                if processed_series_for_action is not None:
                    # Ensure result is pandas 'string' dtype and empty strings become pd.NA
                    final_series = processed_series_for_action.replace(
                        "", pd.NA
                    ).astype("string")
                    df_processed[final_output_col_name] = final_series
                    column_cache[final_output_col_name] = (
                        final_series  # Cache the processed series by its output name
                    )
                else:
                    # This case should ideally not be hit if fallbacks assign current_series
                    log.warning(
                        f"processed_series_for_action was None after action '{action}' on column '{col_name_from_spec}'. This is unexpected."
                    )
                    df_processed[final_output_col_name] = (
                        current_series  # Fallback to current_series (already string/NA handled)
                    )
                    column_cache[final_output_col_name] = current_series

            except Exception as e:
                log.error(
                    f"Error applying rule action '{action}' to column '{col_name_from_spec}' (outputting to '{final_output_col_name}'): {e}",
                    exc_info=True,
                )
                # On error, ensure the output column exists and contains the original data (stringified, NA handled)
                # to prevent missing columns and maintain DataFrame integrity.
                df_processed[final_output_col_name] = (
                    df[col_name_from_spec].fillna(pd.NA).astype("string", copy=False)
                )
                # Do not update column_cache with this original data if an error occurred in processing,
                # unless the intent is to cache the fallback state. For now, only cache successful transformations.

    return df_processed


# --------------------------------------------------------------------------- #
#  Helpers
# --------------------------------------------------------------------------- #
def _resolve_column(df: pd.DataFrame, requested: str) -> str | None:
    """
    Return the *actual* column name in *df* that matches *requested*.

    Strategy (in order):
    1. Exact match
    2. Case-insensitive match
    3. Match on the sanitised token produced by `sanitize_column`
    """
    # 1. exact
    if requested in df.columns:
        return requested

    # 2. case-insensitive
    lower_map = {c.lower(): c for c in df.columns}
    if col := lower_map.get(requested.lower()):
        return col

    # 3. sanitised token match  ("Company / Acct" -> company_acct)
    want_tok = sanitize_column_name(requested)  # Use internal sanitize
    tok_map = {sanitize_column_name(c): c for c in df.columns}
    return tok_map.get(want_tok)  # -> may be None


# Regexes for ID column heuristics
_RE_ID_KEYWORDS = re.compile(
    r"\b(id|key|uuid|code|number|num|no|#|identi)\b", re.IGNORECASE
)
_RE_UUID_PATTERN = re.compile(r"^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$")
_RE_ALL_DIGITS_6_18_PATTERN = re.compile(r"^\d{6,18}$")
# Salesforce and general SaaS ID like patterns (example, can be expanded)
# Looks for common prefixes then a certain length of alphanumeric chars.
# e.g. 001D000000IBasN Ã˜Â£Ã™Ë† a0CD000000IFoobar
_RE_SAAS_ID_LIKE_PATTERN = re.compile(
    r"^(001|003|005|006|00D|00N|00Q|00O|00U|00T|00V|015|500|701|a[0-9a-zA-Z]{2})[a-zA-Z0-9]{10,17}$",
    re.IGNORECASE,
)

# --- Add this entire block ---
RE_PATTERNS = {
    # column-name checks
    "is_id_like": re.compile(
        r"\b(id|key|uuid|code|number|num|no|identi)\b", re.IGNORECASE
    ),
    "is_name_like": re.compile(
        r"\b(name|contact|person|owner|user|first.?name|last.?name)\b", re.IGNORECASE
    ),
    "is_company_like": re.compile(
        r"\b(company|organization|account|customer|business|client|supplier|vendor)\b",
        re.IGNORECASE,
    ),
    "is_geo_like": re.compile(
        r"\b(city|state|province|zip|postal|post.?code|country|region|address|street)\b",
        re.IGNORECASE,
    ),
    # cell-value checks
    "has_url_pattern": re.compile(
        r"^(?:https?://)?(?:[\w\-]+\.)+[a-z]{2,}(?:/.*)?$", re.IGNORECASE
    ),
    "has_email_pattern": re.compile(r"^[\w\.-]+@[\w\.-]+\.[a-z]{2,}$", re.IGNORECASE),
}
# --- End of new block ---

# Semantic and other patterns required for analysis
_SEMANTIC_KEYWORDS_PATTERNS = {
    "id": re.compile(r"\b(id|key|uuid|guid|identifier|_id|_key)\b", re.IGNORECASE),
    "email": re.compile(r"\b(email|mail)\b", re.IGNORECASE),
    "phone": re.compile(r"\b(phone|mobile|fax|contact.?num)\b", re.IGNORECASE),
    "website_domain": re.compile(r"\b(website|url|domain|site|uri)\b", re.IGNORECASE),
    "name_person": re.compile(
        r"\b(name|contact|person|owner|user|first.?name|last.?name|full.?name|attn|attention)\b",
        re.IGNORECASE,
    ),
    "name_company": re.compile(
        r"\b(company|organization|account|customer|business|client|supplier|vendor|firm|acct)\b",
        re.IGNORECASE,
    ),
    "address_street": re.compile(
        r"\b(address|street|line[123]|suite|ste)\b", re.IGNORECASE
    ),
    "city": re.compile(r"\b(city|town|suburb)\b", re.IGNORECASE),
    "state_province": re.compile(
        r"\b(state|province|county|region|territory)\b", re.IGNORECASE
    ),
    "zip_postal": re.compile(r"\b(zip|postal|post.?code)\b", re.IGNORECASE),
    "date": re.compile(
        r"\b(date|dt|time|created|modified|updated|joined|founded|birth|dob)\b",
        re.IGNORECASE,
    ),
    "country": re.compile(r"\b(country)\b", re.IGNORECASE),
}

_SEMANTIC_DATA_REGEX_PATTERNS = {
    "email": re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"),
    "website_domain": re.compile(
        r"^(?:https?:\/\/)?(?:www\.)?([a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,})"
    ),
    "phone": re.compile(
        r"^\(?([0-9]{3})\)?[-.Ã¢â€”Â\s]?([0-9]{3})[-.Ã¢â€”Â\s]?([0-9]{4})$"
    ),
}


def _check_series_regex_match(
    series_sample_str: pd.Series,
    pattern: re.Pattern,
    required_match_ratio: float = 0.75,
) -> bool:
    """Checks if a high ratio of non-null strings in a series sample match a regex pattern."""
    if series_sample_str.empty:
        return False
    # Ensure only non-null strings are matched
    valid_strings = series_sample_str.dropna()
    if valid_strings.empty:
        return False
    match_results = valid_strings.astype(str).str.match(pattern)
    if match_results.empty:
        return False
    return match_results.mean() >= required_match_ratio


# The main function for semantic type detection
def _get_semantic_type(column_name: str, column_sample: pd.Series) -> Optional[str]:
    """Infers the semantic type of a column based on its name and data."""
    processed_col_name = str(column_name).lower()

    # Check ID pattern first (highest priority)
    if _SEMANTIC_KEYWORDS_PATTERNS["id"].search(processed_col_name):
        return "id"

    for type_name, pattern in _SEMANTIC_KEYWORDS_PATTERNS.items():
        if type_name == "id":
            continue  # Already checked
        if pattern.search(processed_col_name):
            if type_name == "name_person" and _SEMANTIC_KEYWORDS_PATTERNS[
                "name_company"
            ].search(processed_col_name):
                continue
            return type_name
    if not column_sample.empty:
        str_sub_sample = column_sample.dropna().astype(str).head(100)
        if not str_sub_sample.empty:
            for type_name, pattern in _SEMANTIC_DATA_REGEX_PATTERNS.items():
                if _check_series_regex_match(
                    str_sub_sample, pattern, required_match_ratio=0.30
                ):
                    return type_name
    return None


def compute_numeric_ratio(series: pd.Series) -> float:
    """Calculates the ratio of numeric characters to total characters in a series sample."""
    sample_str = series.dropna().astype(str)
    if sample_str.empty:
        return 0.0

    total_chars = sample_str.str.len().sum()
    if total_chars == 0:
        return 0.0

    numeric_chars = sample_str.str.count(r"[0-9]").sum()
    return round(numeric_chars / total_chars, 4)


def compute_value_entropy(series: pd.Series) -> float:
    """Calculates the Shannon entropy of the value distribution in a series sample."""
    if series.dropna().empty:
        return 0.0

    counts = series.value_counts()
    props = counts / counts.sum()

    # NumPy-only Shannon entropy (base-2)
    props = props[props > 0]  # avoid log2(0)
    return round(float(-(props * np.log2(props)).sum()), 4)


def normalize_domain(domain: str) -> str:
    """Normalize domain for exact matching."""
    if pd.isna(domain) or not domain:
        return ""

    domain = str(domain).lower().strip()
    # Remove protocol
    domain = re.sub(r"^https?://", "", domain)
    # Remove www
    domain = re.sub(r"^www\.", "", domain)
    # Remove trailing slash
    domain = domain.rstrip("/")

    return domain


# --------------------------------------------------------------------------- #
#  Pre-processing
# --------------------------------------------------------------------------- #
# Inside engine.py


def preprocess(
    df: pd.DataFrame,
    block_key_length: int,
    ruleset_name: str = "default",
    gui_checkbox_rules: Optional[List[Dict[str, Any]]] = None,
    block_from_col: Optional[str] = None,
    processing_mode: Literal["speed", "scale"] = "scale",
    preproc_variant: Optional[str] = "alnum",
    strip_b2c_domains: bool = True,
    config: Optional[Union[MatchConfig, DedupeConfig]] = None,
) -> pd.DataFrame:
    """
    Applies ruleset transformations and creates the active blocking key column.
    Uses the modern preprocessing engine for enhanced performance and reliability.
    """
    # Always use modern preprocessing system
    from fmatch.preprocessing.core.pre_orchestrator import PreprocessOrchestrator
    from fmatch.preprocessing.config.preprocess_config import PreprocessConfig

    preprocess_config = PreprocessConfig(
        gui_rules=gui_checkbox_rules or [],
        yaml_ruleset=ruleset_name if ruleset_name != "none" else None,
        strip_b2c_domains=strip_b2c_domains,
        b2c_config=getattr(config, "b2c_config", None),
        blocking_enabled=bool(block_from_col),
        block_key_length=block_key_length,
        block_from_col=block_from_col,
    )

    orchestrator = PreprocessOrchestrator()
    result = orchestrator.process_dataframe(df, preprocess_config)
    return result.data


# --- REPLACE analyze_blocking_candidates function ---
def analyze_blocking_candidates(
    df_sample_str: str,  # Changed to accept serialized string
    candidate_cols: tuple[str, ...],  # Changed to tuple
    key_len_range: tuple[int, int],  # (min_len, max_len)
    ruleset_name: str = DEFAULT_RULESET_NAME,
) -> dict[str, dict[int, dict[str, Union[float, int]]]]:  # Return richer metrics
    """
    For each candidate column, try the given key lengths and record
    the full metrics dict from _estimate_reduction.
    Returns a nested dict:
        scores[col][key_len] = {'reduction_ratio': ..., 'unique_blocks': ..., ...}
    """
    global log  # Ensure log is accessible
    candidate_cols_repr = candidate_cols
    if (
        len(candidate_cols) > 10
    ):  # If more than 10 candidates, log length instead of full list
        candidate_cols_repr = f"<{len(candidate_cols)} candidates>"
    log.debug(
        f"analyze_blocking_candidates called. Candidates: {candidate_cols_repr}, KeyLenRange: {key_len_range}, Ruleset: {ruleset_name}"
    )

    df_sample = pickle.loads(base64.b64decode(df_sample_str))

    # Type hint for the scores dictionary
    scores: Dict[str, Dict[int, Dict[str, Union[float, int]]]] = {}

    # ---------------- main loop -----------------
    for col in candidate_cols:
        if col not in df_sample.columns:
            log.warning(
                f"analyze_blocking_candidates: Column '{col}' not found in the provided DataFrame sample. Skipping."
            )
            continue

        col_values = df_sample[col].astype(str).fillna("")
        scores.setdefault(col, {})  # Ensure the entry for the column exists

        for key_len in range(key_len_range[1], key_len_range[0] - 1, -1):
            # _estimate_reduction now returns a dictionary of metrics
            metrics_dict = _estimate_reduction(col_values, key_len, ruleset_name)
            scores[col][key_len] = metrics_dict  # Store the whole dict
            log.debug(
                f"Analyzed col='{col}', key_len={key_len}, metrics={metrics_dict}"
            )
    log.debug(
        f"analyze_blocking_candidates returning scores for columns: {list(scores.keys())}"
    )
    return scores


# WEIGHTS dictionary should be defined above this function.

# --- END analyze_blocking_candidates function ---

# --- END Blocking Analysis Helpers ---

# ... (Keep worker_match, worker_duplicate, etc.) ...

# engine.py

# --- ADD Blocking Analysis Helpers --- (This line is a comment in your existing file, the functions below will replace existing ones)


# Helper function to calculate raw Shannon entropy and normalize it
def _normalize_entropy(meaningful_block_counts: pd.Series) -> float:
    """
    Calculates raw Shannon entropy and normalizes it to the range [0, 1].
    Assumes meaningful_block_counts does not contain empty strings and is not empty.
    """
    global log  # Assuming log is defined globally in engine.py
    if meaningful_block_counts.empty:
        return 0.0

    # Pass the series of counts directly to _calculate_entropy
    # _calculate_entropy is called here to get the raw entropy for normalization.
    raw_H = _calculate_entropy(
        meaningful_block_counts.index, counts=meaningful_block_counts
    )

    num_distinct_meaningful_blocks = len(meaningful_block_counts)
    if num_distinct_meaningful_blocks > 1:
        # Normalize against the max possible entropy for this number of distinct blocks
        normalized_H = raw_H / math.log2(num_distinct_meaningful_blocks)
    elif num_distinct_meaningful_blocks == 1:  # Only one distinct block
        normalized_H = 0.0  # Entropy is 0
    else:  # num_distinct_meaningful_blocks == 0 (already handled by .empty() check)
        normalized_H = 0.0
    return max(0.0, min(normalized_H, 1.0))  # Clamp to [0,1]


# Placeholder for _calculate_entropy if it needs to be updated or referenced by new code.
# The existing _calculate_entropy seems fine based on its current usage.
# If calculate_raw_signals or other new functions require a different implementation,
# that would be a separate change. For now, we assume the existing one is sufficient.

# Assuming _calculate_gini and log are defined in engine.py
# Assuming BlockingMode and _normalize_entropy are defined as above


# In engine.py
def _discover_semantic_blocking_columns(
    df_src: pd.DataFrame, df_ref: pd.DataFrame | None, mode: BlockingMode
) -> list[tuple[str, str | None]]:
    """
    Discover blocking column candidates based on semantic patterns.

    Finds columns that look like:
    - Domains (domain, email_domain, AUTO_DOMAIN_FOR_BLOCKING_COL)
    - Emails (email, contact_email, work_email)
    - Phones (phone, mobile, phone_number)
    - Geographic (state, country, region, city)

    Returns pairs even if columns aren't in mappings - this allows enriched
    columns to be used for blocking.
    """
    global log
    discovered = []

    # Semantic patterns to look for
    domain_patterns = ["domain", "email", "website", "url"]
    phone_patterns = ["phone", "mobile", "tel", "contact"]
    geo_patterns = ["state", "country", "region", "city", "province"]

    # Search source columns
    for col in df_src.columns:
        col_lower = str(col).lower().replace("_", "").replace(" ", "")

        # Check if column matches any semantic pattern
        matched_pattern = None
        if any(p in col_lower for p in domain_patterns):
            matched_pattern = "domain"
        elif any(p in col_lower for p in phone_patterns):
            matched_pattern = "phone"
        elif any(p in col_lower for p in geo_patterns):
            matched_pattern = "geo"

        if not matched_pattern:
            continue

        # For DEDUPE mode, pair with itself
        if mode == BlockingMode.DEDUPE:
            discovered.append((col, None))
            log.debug(f"Discovered {matched_pattern} blocking column: {col}")
            continue

        # For MATCH mode, try to find matching column in reference
        if df_ref is None:
            continue

        # First, try exact match
        if col in df_ref.columns:
            discovered.append((col, col))
            log.debug(f"Discovered {matched_pattern} blocking pair: {col} <-> {col}")
            continue

        # Then, try to find similar column name in reference
        for ref_col in df_ref.columns:
            ref_col_lower = str(ref_col).lower().replace("_", "").replace(" ", "")
            # Check if reference column matches same pattern
            if matched_pattern == "domain" and any(
                p in ref_col_lower for p in domain_patterns
            ):
                discovered.append((col, ref_col))
                log.debug(
                    f"Discovered {matched_pattern} blocking pair: {col} <-> {ref_col}"
                )
                break
            elif matched_pattern == "phone" and any(
                p in ref_col_lower for p in phone_patterns
            ):
                discovered.append((col, ref_col))
                log.debug(
                    f"Discovered {matched_pattern} blocking pair: {col} <-> {ref_col}"
                )
                break
            elif matched_pattern == "geo" and any(
                p in ref_col_lower for p in geo_patterns
            ):
                discovered.append((col, ref_col))
                log.debug(
                    f"Discovered {matched_pattern} blocking pair: {col} <-> {ref_col}"
                )
                break

    return discovered


# REPLACE your existing _candidate_pairs method with this:

# In engine.py


def _candidate_pairs(
    mappings: list[dict],
    df_src: pd.DataFrame,
    mode: BlockingMode,
    df_ref: pd.DataFrame | None = None,
    allow_all_mapped_candidates: bool = False,  # <<< NEW PARAMETER
) -> list[tuple[str, str | None]]:
    """
    Generates a list of (source_column, reference_column_or_None) candidate pairs
    for blocking based on mappings and column characteristics from sampled data.
    Reference column is None or same as source in DEDUPE mode.
    If allow_all_mapped_candidates is True, uniqueness checks are bypassed for mapped columns.
    """
    global log
    pairs = []
    if not mappings:
        log.warning(
            "_candidate_pairs: No mappings provided. Cannot generate candidates based on mappings."
        )
        return []

    log.debug(
        f"_candidate_pairs: Received {len(mappings)} mappings. df_src_s cols: {list(df_src.columns) if df_src is not None else 'None'}. df_ref_s cols: {list(df_ref.columns) if df_ref is not None else 'None'}. allow_all_mapped_candidates: {allow_all_mapped_candidates}"
    )

    # Slightly relaxed uniqueness threshold for when filter is active
    HIGH_NUNIQUE_THRESHOLD = 0.995
    LOW_NUNIQUE_THRESHOLD = 0.02

    for m_idx, m in enumerate(mappings):
        s_col = m.get("source")
        r_col_for_candidate = m.get("ref") if mode == BlockingMode.MATCH else s_col
        log.debug(
            f"_candidate_pairs: Processing mapping #{m_idx + 1}: {m}. s_col='{s_col}', r_col_for_candidate='{r_col_for_candidate}'"
        )

        if not s_col:
            log.debug(
                f"_candidate_pairs: Skipping mapping {m} due to missing 'source' key."
            )
            continue

        if s_col not in df_src.columns:
            log.debug(
                f"_candidate_pairs: Source column '{s_col}' from mapping not in df_src_sample. Available: {list(df_src.columns)}. Skipping."
            )
            continue

        src_nunique_ratio = 0.0
        if not df_src[s_col].empty:
            src_distinct_count = df_src[s_col].nunique()
            src_len = len(df_src[s_col])
            src_nunique_ratio = src_distinct_count / src_len if src_len > 0 else 0.0
            log.debug(
                f"_candidate_pairs: Evaluating SOURCE column '{s_col}'. Sample Distinct: {src_distinct_count}, Sample Total Values: {src_len}, Nunique_Ratio: {src_nunique_ratio:.4f}"
            )
        else:
            log.debug(
                f"_candidate_pairs: SOURCE column '{s_col}' is empty in the provided df_src sample. Skipping."
            )
            continue

        # Apply uniqueness filters only if allow_all_mapped_candidates is False
        if not allow_all_mapped_candidates:
            if src_nunique_ratio < LOW_NUNIQUE_THRESHOLD:
                log.debug(
                    f"_candidate_pairs: Source column '{s_col}' too flat (nunique_ratio={src_nunique_ratio:.3f} < {LOW_NUNIQUE_THRESHOLD}). Skipping."
                )
                continue
            if src_nunique_ratio > HIGH_NUNIQUE_THRESHOLD:  # Using the new threshold
                log.debug(
                    f"_candidate_pairs: Source column '{s_col}' almost unique (nunique_ratio={src_nunique_ratio:.3f} > {HIGH_NUNIQUE_THRESHOLD}). Skipping."
                )
                continue
        elif allow_all_mapped_candidates:
            log.debug(
                f"_candidate_pairs: Bypassing uniqueness filters for mapped source column '{s_col}' as allow_all_mapped_candidates is True."
            )

        if mode == BlockingMode.MATCH:
            if not r_col_for_candidate:
                log.debug(
                    f"_candidate_pairs: Skipping mapping {m} for MATCH mode due to missing 'ref' key for r_col_for_candidate."
                )
                continue
            if df_ref is None or r_col_for_candidate not in df_ref.columns:
                log.debug(
                    f"_candidate_pairs: Reference column '{r_col_for_candidate}' from mapping not in df_ref_sample or df_ref_sample is None for MATCH mode. Available: {list(df_ref.columns) if df_ref is not None else 'N/A'}. Skipping."
                )
                continue

            ref_nunique_ratio = 0.0
            if not df_ref[r_col_for_candidate].empty:
                ref_distinct_count = df_ref[r_col_for_candidate].nunique()
                ref_len = len(df_ref[r_col_for_candidate])
                ref_nunique_ratio = ref_distinct_count / ref_len if ref_len > 0 else 0.0
                log.debug(
                    f"_candidate_pairs: Evaluating REFERENCE column '{r_col_for_candidate}'. Sample Distinct: {ref_distinct_count}, Sample Total Values: {ref_len}, Nunique_Ratio: {ref_nunique_ratio:.4f}"
                )
            else:
                log.debug(
                    f"_candidate_pairs: REFERENCE column '{r_col_for_candidate}' is empty in the provided df_ref sample. Skipping."
                )
                continue

            # Apply uniqueness filters only if allow_all_mapped_candidates is False
            if not allow_all_mapped_candidates:
                if ref_nunique_ratio < LOW_NUNIQUE_THRESHOLD:
                    log.debug(
                        f"_candidate_pairs: Ref column '{r_col_for_candidate}' too flat (nunique_ratio={ref_nunique_ratio:.3f} < {LOW_NUNIQUE_THRESHOLD}). Skipping."
                    )
                    continue
                if (
                    ref_nunique_ratio > HIGH_NUNIQUE_THRESHOLD
                ):  # Using the new threshold
                    log.debug(
                        f"_candidate_pairs: Ref column '{r_col_for_candidate}' almost unique (nunique_ratio={ref_nunique_ratio:.3f} > {HIGH_NUNIQUE_THRESHOLD}). Skipping."
                    )
                    continue
            elif allow_all_mapped_candidates:
                log.debug(
                    f"_candidate_pairs: Bypassing uniqueness filters for mapped reference column '{r_col_for_candidate}' as allow_all_mapped_candidates is True."
                )

            pairs.append((s_col, r_col_for_candidate))
        else:  # DEDUPE mode
            pairs.append((s_col, s_col))

    unique_pairs = sorted(list(set(pairs)))
    log.info(
        f"_candidate_pairs generated: {unique_pairs} from {len(mappings)} mappings (allow_all_mapped_candidates={allow_all_mapped_candidates})."
    )
    return unique_pairs


def _generate_composite_candidates(
    mappings: list[dict],
    df_src: pd.DataFrame,
    mode: BlockingMode,
    df_ref: Optional[pd.DataFrame] = None,
    top_n: int = 3,
) -> list[tuple[tuple[str, str], tuple[str, str]]]:
    """Generate composite blocking key candidates from top single-column candidates."""
    # Get single column candidates first
    single_candidates = _candidate_pairs(mappings, df_src, mode, df_ref)

    # Score them individually
    candidate_scores = []
    for src_col, ref_col in single_candidates[: top_n * 2]:
        # Quick quality assessment
        score = _quick_column_quality_score(df_src[src_col])
        candidate_scores.append((src_col, ref_col, score))

    # Generate pairs from top candidates
    composite_candidates = []
    for i, (src1, ref1, score1) in enumerate(candidate_scores[:top_n]):
        for src2, ref2, score2 in candidate_scores[i + 1 : top_n + 1]:
            composite_candidates.append(
                (
                    (src1, src2),  # Source composite
                    (ref1, ref2),  # Reference composite
                )
            )

    return composite_candidates


# Update build_key to handle composite keys
def build_composite_key(
    series1: pd.Series,
    series2: pd.Series,
    variant: str,
    k_len: int,
    separator: str = "|",
) -> pd.Series:
    """Build composite blocking key from two columns."""
    key1, _ = build_key(series1, variant, k_len // 2, "default")
    key2, _ = build_key(series2, variant, k_len // 2, "default")
    return key1.astype(str) + separator + key2.astype(str)


# Assuming 'soundex' function is already defined in engine.py
# Assuming 'log' is a globally defined logger instance


def _alnum(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    keys = s.str.lower().str.replace(r"\W+", "", regex=True).str.slice(0, k)
    return keys, k


def _soundex(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    # This variant now ignores k and always uses the standard 4-char length.
    effective_k = 4
    processed_keys = s.apply(lambda x: soundex(x) or "" if pd.notna(x) else "").astype(
        str
    )
    return processed_keys, effective_k


# In engine.py
# Ensure this is imported

# ... (other helper functions like _alnum, _soundex) ...


def _first_n_words(s: pd.Series, k: int, num_words: int) -> tuple[pd.Series, int]:
    """
    Takes the first 'num_words' from each string, concatenates them,
    applies alnum cleaning, and slices to k.
    """
    global log  # Assuming log is accessible

    def extract_n_words(text: str) -> str:
        if not text:
            return ""
        words = text.split()
        return "".join(words[:num_words])

    processed_series = s.apply(extract_n_words)
    alnum_cleaned_series = processed_series.str.lower().str.replace(
        r"[\W_]+", "", regex=True
    )
    final_keys = alnum_cleaned_series.str.slice(0, k)
    return final_keys, k  # Effective length is k


def _email_domain(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Extract domain from email addresses for blocking.

    Also handles columns that already contain plain domains (no '@').
    """
    raw = s.fillna("").astype(str)
    # Primary path: extract portion after '@'
    domains = raw.str.extract(_RE_EMAIL_DOMAIN, expand=False)

    # Fallback: if value already looks like a domain (contains a dot, no spaces), keep it
    fallback_mask = domains.isna()
    if fallback_mask.any():
        fallback_candidates = raw.str.strip().str.lower()
        fallback_candidates = fallback_candidates.where(
            fallback_candidates.str.contains(r"\.")
            & ~fallback_candidates.str.contains(r"\s"),
            "",
        )
        domains = domains.fillna(fallback_candidates)

    # Remove common free email providers for B2B matching when data truly came from an email
    if ENGINE_CONFIG.b2b_mode:
        common_providers = {
            "gmail.com",
            "yahoo.com",
            "hotmail.com",
            "outlook.com",
            "aol.com",
        }
        mask_consumer = domains.isin(common_providers)
        domains = domains.mask(mask_consumer, "")

    cleaned = domains.str.lower().str.replace(r"[^\w\.]", "", regex=True)
    return cleaned.str.slice(0, k).fillna(""), k


def _phone_digits(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Extract last N digits from phone numbers."""
    # Remove all non-digits
    phones = s.str.replace(_RE_PHONE_DIGITS, "", regex=True)
    # Take last k digits (handles international prefixes)
    effective_k = min(k, 10)  # Max 10 digits
    return phones.str.slice(-effective_k, None).fillna(""), effective_k


def _phone_area(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Extract area code from phone numbers."""
    phones = s.str.replace(_RE_PHONE_DIGITS, "", regex=True)
    # Try to extract area code (digits 1-3 after country code)
    # For US: skip first digit if it's 1
    phones_clean = phones.str.replace(_RE_PHONE_US_CLEAN, r"\1", regex=True)
    return phones_clean.str.slice(0, 3).fillna(""), 3


def _company_stem(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Remove company suffixes and extract stem."""
    # Clean in steps
    cleaned = s.str.lower()
    cleaned = cleaned.str.replace(_RE_COMPANY_SUFFIXES, "", regex=True)
    cleaned = cleaned.str.strip()
    cleaned = cleaned.str.replace(r"[^\w]", "", regex=True)

    return cleaned.str.slice(0, k).fillna(""), k


def _address_number(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Extract street number from addresses."""
    # Extract leading numbers
    numbers = s.str.extract(r"^(\d+)", expand=False)
    return numbers.fillna("").str.slice(0, k), k


def _domain_from_url(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Extract domain from URLs."""
    # Remove protocol and www
    domains = s.str.replace(_RE_URL_PROTOCOL, "", regex=True)
    domains = domains.str.replace(_RE_URL_WWW, "", regex=True)
    # Extract domain before first /
    domains = domains.str.extract(_RE_URL_DOMAIN, expand=False)
    # Remove TLD for better matching
    domains = domains.str.replace(_RE_URL_TLD, "", regex=True)

    return domains.str.lower().str.slice(0, k).fillna(""), k


def _name_initials(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Extract initials from names (for person matching)."""
    # Split on spaces and take first letter of each word
    initials = (
        s.str.split()
        .str[0:k]
        .apply(
            lambda x: "".join([w[0] for w in x if len(w) > 0])
            if isinstance(x, list)
            else ""
        )
    )
    return initials.str.lower(), k


def _metaphone(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Double Metaphone for better name matching than Soundex."""
    try:
        from metaphone import doublemetaphone

        def get_metaphone(text):
            if pd.isna(text) or text == "":
                return ""
            primary, _ = doublemetaphone(str(text))
            return primary[:k] if primary else ""

        return s.apply(get_metaphone), k
    except ImportError:
        log.warning("metaphone package not installed, falling back to soundex")
        return _soundex(s, k)


def _phone_normalize(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Normalize phone numbers to digits only"""
    phones = s.str.replace(r"[^\d]", "", regex=True).fillna("")
    return phones.str.slice(-10, None), 10  # Last 10 digits


def _company_clean(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Remove common company suffixes"""
    pattern = r"\b(inc|llc|ltd|corp|corporation|company|co|group|holdings|international|intl|plc|gmbh|partners|lp|llp|pvt|private|limited|associates|consulting|services|solutions|tech|technologies|labs|systems|software|global|worldwide)\b\.?"
    cleaned = (
        s.str.lower()
        .str.replace(pattern, "", regex=True, flags=re.IGNORECASE)
        .str.strip()
    )
    return cleaned.str.replace(r"\W+", "", regex=True).str.slice(0, k), k


def _unicode_normalize(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Normalize Unicode to handle encoding variations."""
    if EncodingFixer is None:
        normalized = s.fillna("").astype(str).map(lambda x: _ud.normalize("NFKC", x))
    else:
        normalized = s.apply(
            lambda x: EncodingFixer.safe_unicode_normalize(str(x)) if pd.notna(x) else ""
        )
    cleaned = (
        normalized.str.lower()
        .str.replace(r"[^\w\s]", "", regex=True)
        .str.replace(r"\s+", "", regex=True)
    )
    return cleaned.str.slice(0, k), k


def _exact_substring(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Exact substring - useful for codes and non-Latin scripts."""
    return s.astype(str).str.slice(0, k), k


def _strip_emoji(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Remove emoji before processing."""
    emoji_pattern = ExoticDataDetector.EMOJI_PATTERN
    cleaned = s.astype(str).apply(lambda x: emoji_pattern.sub("", x))
    return cleaned.str.lower().str.replace(r"\W+", "", regex=True).str.slice(0, k), k


def _first_n_chars(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Simple first N characters extraction without additional cleaning."""
    return s.astype(str).str.slice(0, k), k


def _name_prefix_variant(s: pd.Series, k: int) -> tuple[pd.Series, int]:
    """Company name prefix with optional second-token tails."""
    keys = _build_name_prefix_keys(s, max(k, 1))
    lengths = keys.str.len() if not keys.empty else pd.Series(dtype="int64")
    max_len = lengths.max() if not lengths.empty else None
    if max_len is None or pd.isna(max_len):
        effective = max(k, 1) + (
            _NAME_PREFIX_SECOND_TOKEN_LEN if _NAME_PREFIX_SECOND_TOKEN_LEN else 0
        )
    else:
        effective = int(max(max_len, k))
    return keys, effective


# Update PREPROC_REGISTRY
PREPROC_REGISTRY = {
    # Original variants
    "alnum": _alnum,
    "soundex": _soundex,
    "first_1_word": functools.partial(_first_n_words, num_words=1),
    "first_2_words": functools.partial(_first_n_words, num_words=2),
    # New variants for RevOps/MarketingOps
    "email_domain": _email_domain,
    "phone_digits": _phone_digits,
    "phone_area": _phone_area,
    "company_stem": _company_stem,
    "address_number": _address_number,
    "domain_from_url": _domain_from_url,
    "name_prefix": _name_prefix_variant,
    "name_initials": _name_initials,
    "metaphone": _metaphone,
}
PREPROC_REGISTRY.update(
    {
        "unicode_normalize": _unicode_normalize,
        "exact_substring": _exact_substring,
        "strip_emoji": _strip_emoji,
        "first_n_chars": _first_n_chars,
    }
)

# Add variant categories for GUI
VARIANT_CATEGORIES = {
    "basic": ["alnum", "first_1_word", "first_2_words"],
    "phonetic": ["soundex", "metaphone"],
    "contact": ["email_domain", "phone_digits", "phone_area"],
    "company": ["company_stem", "domain_from_url", "name_prefix"],
    "address": ["address_number"],
    "person": ["name_initials", "metaphone"],
}
VARIANT_CATEGORIES.update(
    {
        "exotic": [
            "unicode_normalize",
            "exact_substring",
            "strip_emoji",
            "first_n_chars",
        ],
    }
)

# Add variant descriptions for GUI tooltips
VARIANT_DESCRIPTIONS = {
    "alnum": "Alphanumeric characters only (general purpose)",
    "soundex": "Phonetic encoding (4 characters)",
    "metaphone": "Advanced phonetic encoding",
    "first_1_word": "First word only",
    "first_2_words": "First two words",
    "email_domain": "Domain from email addresses",
    "phone_digits": "Last N digits from phone numbers",
    "phone_area": "Area code from phone numbers",
    "company_stem": "Company name without suffixes",
    "address_number": "Street number from addresses",
    "domain_from_url": "Domain name from URLs",
    "name_prefix": "Company name prefix (3-5 chars + optional tail)",
    "name_initials": "Initials from person names",
}
VARIANT_DESCRIPTIONS.update(
    {
        "unicode_normalize": "Unicode normalization for accented characters",
        "exact_substring": "Exact substring matching (no cleaning)",
        "strip_emoji": "Remove emoji before matching",
        "first_n_chars": "First N characters (good for CJK)",
    }
)


def build_key(
    series: pd.Series,
    preproc_variant: str,  # Renamed from 'variant' for clarity
    k_len: int,  # This is the target/input k_len
    ruleset_name: str,  # Renamed from 'ruleset' for clarity
    stats_cache: ColumnStatsCache | None = None,
) -> tuple[pd.Series, int]:  # Now returns keys and their effective_length
    """
    Generates blocking keys for a given series by applying a ruleset,
    a specific preprocessing variant, and truncating to k_len.
    Returns (keys_series, effective_key_length). Keys_series is empty if keys are overwhelmingly blank.
    """
    global log, _RULESET_LOCK, AVAILABLE_RULESETS, apply_rules, sanitize_column_name
    global PREPROC_REGISTRY, _RULESET_CACHE  # Ensure globals are accessible

    # Validate and sanitize k_len
    try:
        k_len = int(k_len)
        if k_len <= 0:  # k_len here is the input/target k_len
            log.warning(f"build_key: k_len ({k_len}) is not positive. Defaulting to 1.")
            k_len = 1
    except ValueError:
        log.error(
            f"build_key: Input k_len '{k_len}' is not a valid integer. Defaulting to 1."
        )
        k_len = 1

    original_series_name = (
        series.name if series.name is not None else "__unnamed_series__"
    )

    # FIX 1: Store the original index at the beginning for guaranteed access.
    cache_enabled = stats_cache is not None
    if cache_enabled:
        series = stats_cache.remember_series(
            series, series.attrs.get("_fmatch_parent_df_id", id(series))
        )

    original_index = series.index

    s = series.fillna("").astype("string", copy=False).str.strip()

    if ruleset_name and ruleset_name.lower() != "none":
        # Include size and sample hash in cache key to avoid index mismatch
        sample_hash = getattr(series, "_fmatch_sample_hash", None)
        cache_key = (original_series_name, ruleset_name, len(series), sample_hash)
        if cache_key in _RULESET_CACHE:
            cached_s = _RULESET_CACHE[cache_key]
            # Belt-and-suspenders: verify size still matches
            if len(cached_s) == len(series):
                s = cached_s
                log.debug(
                    f"build_key: Cache HIT for ruleset '{ruleset_name}' on series '{original_series_name}'."
                )
            else:
                log.debug(
                    f"build_key: Cache size mismatch ({len(cached_s)} vs {len(series)}); rebuilding"
                )
                cache_key = None  # Force rebuild below
        else:
            cache_key = None  # Force rebuild below

        if cache_key is None or cache_key not in _RULESET_CACHE:
            log.debug(
                f"build_key: Cache MISS for ruleset '{ruleset_name}' on series '{original_series_name}'. Applying rules."
            )
            with _RULESET_LOCK:
                rules = AVAILABLE_RULESETS.get(ruleset_name.lower(), {}).get(
                    "rules", []
                )

            if rules:
                temp_df_col_name = original_series_name
                df_for_rules = pd.DataFrame({temp_df_col_name: s})
                processed_df = apply_rules(df_for_rules, rules)

                if len(processed_df.columns) == 1:
                    out_col_name = processed_df.columns[0]
                elif temp_df_col_name in processed_df.columns:
                    out_col_name = temp_df_col_name
                elif sanitize_column_name(temp_df_col_name) in processed_df.columns:
                    out_col_name = sanitize_column_name(temp_df_col_name)
                else:
                    log.warning(
                        f"build_key: Could not determine output column after applying ruleset '{ruleset_name}' to '{original_series_name}'. Column names: {list(processed_df.columns)}. Using series as is (post initial coercion)."
                    )
                    out_col_name = None
                if out_col_name:
                    s = (
                        processed_df[out_col_name]
                        .fillna("")
                        .astype("string", copy=False)
                        .str.strip()
                    )
            # Rebuild cache key with correct parameters
            sample_hash = getattr(series, "_fmatch_sample_hash", None)
            cache_key = (original_series_name, ruleset_name, len(series), sample_hash)
            _RULESET_CACHE[cache_key] = s
    try:
        variant_func = PREPROC_REGISTRY[preproc_variant]
    except KeyError:
        log.warning(
            f"build_key: Unknown preproc_variant '{preproc_variant}'. Falling back to 'alnum'."
        )
        variant_func = PREPROC_REGISTRY["alnum"]

    if cache_enabled:
        cached_keys, effective_k = stats_cache.get_variant_output(
            series,
            preproc_variant,
            k_len,
            ruleset_name or "",
            lambda: variant_func(s, k_len),
        )
        keys = cached_keys.copy()
    else:
        keys, effective_k = variant_func(s, k_len)
    keys = keys.fillna("").astype("string", copy=False)

    # ENHANCED: More robust empty key detection and fallback
    if keys.empty or (keys == "").all() or keys.isna().all():
        log.warning(
            f"build_key for '{original_series_name}' (variant: {preproc_variant}) "
            f"resulted in all blank/empty keys. Generating fallback keys."
        )
        # Generate deterministic fallback based on hash of original values
        fallback_keys = []
        for i, val in enumerate(series):
            if pd.isna(val) or str(val).strip() == "":
                fallback_keys.append(f"_empty_{i % 100}")
            else:
                # Use hash of content for deterministic blocking
                hash_val = (
                    int(hashlib.md5(str(val).encode("utf-8")).hexdigest(), 16) % 1000
                )
                fallback_keys.append(f"_fallback_{hash_val}")

        # FIX 2: Use the stored original_index to ensure the fallback Series is correctly aligned.
        keys = pd.Series(
            fallback_keys,
            index=original_index,
            dtype="string",
            name=original_series_name,
        )
        # The effective_k is less meaningful here, but we'll keep the input k_len
        # as the "intended" length for consistency in logs.
        log.info(
            f"Generated {keys.nunique()} unique fallback blocks for '{original_series_name}'"
        )
    else:
        # FIX 3: Add a safety check to force the index to be correct if any prior step modified it.
        if not keys.index.equals(original_index):
            # Additional check: ensure lengths match before set_axis
            if len(keys) == len(original_index):
                keys = keys.set_axis(original_index)
            else:
                # Rebuild rather than forcing a broken index
                log.warning(
                    f"build_key: Index length mismatch ({len(keys)} vs {len(original_index)}); rebuilding keys"
                )
                variant_func = PREPROC_REGISTRY.get(
                    preproc_variant, PREPROC_REGISTRY["alnum"]
                )
                keys, effective_k = variant_func(s, k_len)
                keys = keys.fillna("").astype("string", copy=False)

    log.debug(
        f"build_key: For '{original_series_name}', Variant='{preproc_variant}', InputKLen={k_len}, EffectiveKLen={effective_k}. Preview: {keys.head().to_list() if not keys.empty else 'Empty'}"
    )
    return keys, effective_k


def early_reject(raw_signals: dict, thresholds: dict) -> bool:
    """
    Determines if a blocking strategy should be rejected early based on raw signals
    and adaptive thresholds.
    """
    global log

    # Entropy Floor Check
    h_value = raw_signals.get("H", 1.0)
    min_entropy_threshold = thresholds.get("min_entropy", 0.3)
    if h_value < min_entropy_threshold:
        log.info(
            f"early_reject: Rejecting strategy due to low entropy (H = {h_value:.3f} < {min_entropy_threshold})."
        )
        return True

    # Dominant Block Check
    p1_value = raw_signals.get("P1", 0.0)
    max_p1_threshold = thresholds.get("max_p1", 0.5)
    if p1_value > max_p1_threshold:
        log.info(
            f"early_reject: Rejecting strategy due to dominant block (P1 = {p1_value:.3f} > {max_p1_threshold})."
        )
        return True

    # Excessive fragmentation check (keep this hard-coded as it's a fundamental issue)
    rr_value = raw_signals.get("RR", 0.0)
    if rr_value > 0.999 and h_value < 0.1:
        log.info(
            f"early_reject: Rejecting strategy due to high RR and low Entropy (RR={rr_value:.3f} > 0.999, H={h_value:.3f} < 0.1). Indicates excessive fragmentation."
        )
        return True

    log.debug(
        f"early_reject: Strategy not rejected. Signals: P1={p1_value:.3f}, RR={rr_value:.3f}, H={h_value:.3f}"
    )
    return False


# In engine.py
# REPLACE your existing simple_fallback method with this:
# Replace simple_fallback in engine.py


def smart_fallback(
    df_src_sample: pd.DataFrame,
    mode: BlockingMode,
    ruleset_name: str,
    df_ref_sample: Optional[pd.DataFrame] = None,
    weight_profile: str = "balanced",
    stats_cache: ColumnStatsCache | None = None,
) -> dict:
    """
    Intelligently selects fallback blocking strategy when automatic selection fails.
    Enhanced with exotic data awareness.
    """
    log.info(f"smart_fallback: Evaluating fallback options for {mode.name} mode")

    # NEW: Analyze failure pattern
    failure_analysis = _analyze_failure_pattern(df_src_sample)
    log.info(f"Failure analysis: {failure_analysis}")

    # NEW: Choose specific fallback based on failure type
    if failure_analysis["all_too_unique"]:
        return _truncation_fallback(df_src_sample, mode, k=4)
    elif failure_analysis["all_too_flat"]:
        return _composite_fallback(df_src_sample, mode, top_n_cols=2)
    elif failure_analysis["encoding_chaos"]:
        return _unicode_safe_fallback(df_src_sample, mode)
    elif failure_analysis["extreme_nulls"]:
        # Find column with least nulls
        string_cols = df_src_sample.select_dtypes(include=["object", "string"]).columns
        best_col = None
        min_nulls = 1.0

        for col in string_cols:
            null_ratio = df_src_sample[col].isnull().sum() / len(df_src_sample)
            if null_ratio < min_nulls:
                min_nulls = null_ratio
                best_col = col

        if best_col:
            return {
                "apply_blocking": True,
                "source_block_col": best_col,
                "ref_block_col": best_col,
                "block_key_len": 4,
                "preproc": "alnum",
                "quality_score": 0.25,
                "is_fallback": True,
                "fallback_reason": f"High null ratio - using least null column ({best_col})",
            }

    # If no specific failure pattern detected, use existing smart logic
    validator = BlockingColumnValidator(stats_cache)
    candidates = []

    # Get string/object columns
    string_cols = df_src_sample.select_dtypes(
        include=["object", "string"]
    ).columns.tolist()

    if not string_cols:
        log.error("smart_fallback: No string columns found")
        return {
            "apply_blocking": False,
            "reason": "No suitable string columns for blocking",
            "is_fallback": True,
        }

    # Evaluate each column
    for col in string_cols:
        # Skip obviously bad columns
        col_lower = col.lower()
        skip_patterns = [
            "description",
            "notes",
            "comment",
            "memo",
            "text",
            "details",
            "summary",
            "content",
            "body",
            "message",
        ]

        if any(pattern in col_lower for pattern in skip_patterns):
            log.debug(f"smart_fallback: Skipping '{col}' - likely free text")
            continue

        # Validate column
        is_valid, issues, metrics = validator.validate_column(df_src_sample[col])

        # NEW: Add exotic data profiling
        if ExoticDataDetector:
            try:
                exotic_profile = ExoticDataDetector.detect_characteristics(
                    df_src_sample[col].head(1000)
                )
                metrics.update(exotic_profile)
            except Exception as e:
                log.debug(f"ExoticDataDetector failed for {col} in fallback: {e}")

        # Calculate suitability score
        score = 0

        # Base score from validation
        score += metrics.get("quality_score", 0) * 100

        # Bonus for semantic hints in name
        if any(term in col_lower for term in ["id", "key", "code", "number"]):
            score += 20
        elif any(term in col_lower for term in ["name", "company", "account"]):
            score += 15
        elif any(term in col_lower for term in ["email", "phone", "address"]):
            score += 10
        elif any(term in col_lower for term in ["city", "state", "country"]):
            score += 10

        # NEW: Adjust score based on exotic characteristics
        if metrics.get("script") not in ["latin", "unknown"]:
            # Non-Latin script - adjust scoring
            if metrics.get("script") == "mixed":
                score -= 10  # Mixed scripts can be problematic
            else:
                score += 5  # Pure non-Latin is fine if handled correctly

        if metrics.get("encoding_issues", False):
            score -= 30  # Significant penalty for encoding issues

        if metrics.get("has_emoji", False):
            score -= 5  # Small penalty for emoji

        # Penalty for issues
        score -= len(issues) * 10

        # Cardinality preference
        unique_ratio = metrics.get("unique_ratio", 0)
        if 0.01 < unique_ratio < 0.5:  # Sweet spot
            score += 20
        elif unique_ratio > 0.9:  # Too unique
            score -= 20

        # For MATCH mode, check if column exists in reference
        ref_column = None  # Initialize ref_column
        ref_score_factor = 1.0
        if mode == BlockingMode.MATCH and df_ref_sample is not None:
            if col in df_ref_sample.columns:
                # Exact match found
                ref_column = col
                # Validate reference column too
                ref_valid, ref_issues, ref_metrics = validator.validate_column(
                    df_ref_sample[col]
                )
                ref_score_factor = ref_metrics.get("quality_score", 0.5)
            else:
                # Try fuzzy match
                ref_cols = df_ref_sample.columns.tolist()
                matches = rf_process.extract(
                    col, ref_cols, scorer=rf_fuzz.WRatio, limit=1
                )
                if matches and matches[0][1] > 80:
                    matched_col = matches[0][0]
                    ref_column = matched_col  # Set the fuzzy-matched column
                    ref_valid, ref_issues, ref_metrics = validator.validate_column(
                        df_ref_sample[matched_col]
                    )
                    ref_score_factor = ref_metrics.get("quality_score", 0.5) * 0.8
                else:
                    ref_score_factor = 0.1  # Heavy penalty

        final_score = score * ref_score_factor

        candidates.append(
            {
                "column": col,
                "score": final_score,
                "metrics": metrics,
                "issues": issues,
                "unique_ratio": unique_ratio,
                "ref_column": ref_column,  # Now properly set
            }
        )

    if not candidates:
        log.error("smart_fallback: No viable candidates after evaluation")
        return {
            "apply_blocking": False,
            "reason": "All columns failed validation",
            "is_fallback": True,
        }

    # Sort by score
    candidates.sort(key=lambda x: x["score"], reverse=True)
    best = candidates[0]

    log.info(
        f"smart_fallback: Selected '{best['column']}' "
        f"(score: {best['score']:.1f}, unique_ratio: {best['unique_ratio']:.3f})"
    )

    # Determine preprocessing variant - now with exotic awareness
    variant = detect_optimal_variants(
        df_src_sample[best["column"]],
        best["column"],
        best["metrics"],  # Pass metrics for exotic-aware variant selection
    )[0]  # Take first detected variant

    # Calculate metrics for the fallback
    src_keys, eff_k = build_key(
        df_src_sample[best["column"]],
        variant,
        3,  # Default key length
        ruleset_name,
    )

    ref_keys = None
    if mode == BlockingMode.MATCH and df_ref_sample is not None:
        ref_col = best.get("ref_column")  # Use the properly determined ref column
        if ref_col and ref_col in df_ref_sample.columns:
            ref_keys, _ = build_key(df_ref_sample[ref_col], variant, 3, ruleset_name)
            log.info(
                f"smart_fallback: Using reference column '{ref_col}' (matched from '{best['column']}')"
            )

    # Calculate raw signals
    raw_metrics = calculate_raw_signals(src_keys, eff_k, mode, ref_keys)

    if raw_metrics is None:
        log.warning("smart_fallback: Failed to calculate metrics")
        raw_metrics = {"reason": "Metric calculation failed"}

    return {
        "apply_blocking": True,
        "source_block_col": best["column"],
        "ref_block_col": best.get(
            "ref_column", best["column"]
        ),  # Use the determined ref column
        "block_key_len": 3,
        "preproc": variant,
        "quality_score": best["score"] / 150,  # Normalize to 0-1
        "raw_metrics": raw_metrics,
        "is_fallback": True,
        "fallback_reason": f"Selected best available column: {best['column']}",
        "validation_metrics": best["metrics"],
    }


# Helper functions for smart_fallback


def _analyze_failure_pattern(df_sample: pd.DataFrame) -> dict:
    """Analyze why blocking strategy selection failed."""
    analysis = {
        "all_too_unique": False,
        "all_too_flat": False,
        "encoding_chaos": False,
        "no_string_columns": False,
        "extreme_nulls": False,
    }

    string_cols = df_sample.select_dtypes(include=["object", "string"]).columns

    if len(string_cols) == 0:
        analysis["no_string_columns"] = True
        return analysis

    # Check uniqueness across all columns
    unique_ratios = []
    null_ratios = []
    encoding_issues_count = 0

    for col in string_cols[:10]:  # Check first 10 columns
        series = df_sample[col]

        # Null ratio
        null_ratio = series.isnull().sum() / len(series)
        null_ratios.append(null_ratio)

        # Uniqueness
        if null_ratio < 0.99:
            unique_ratio = series.nunique() / (len(series) - series.isnull().sum())
            unique_ratios.append(unique_ratio)

        # Encoding issues
        if ExoticDataDetector:
            try:
                profile = ExoticDataDetector.detect_characteristics(series.head(100))
                if profile.get("encoding_issues", False):
                    encoding_issues_count += 1
            except:
                pass

    # Determine failure pattern
    if unique_ratios and np.mean(unique_ratios) > 0.95:
        analysis["all_too_unique"] = True
    elif unique_ratios and np.mean(unique_ratios) < 0.05:
        analysis["all_too_flat"] = True

    if null_ratios and np.mean(null_ratios) > 0.9:
        analysis["extreme_nulls"] = True

    if encoding_issues_count > len(string_cols) * 0.5:
        analysis["encoding_chaos"] = True

    return analysis


def _truncation_fallback(df_src: pd.DataFrame, mode: BlockingMode, k: int = 4) -> dict:
    """Fallback strategy using aggressive truncation."""
    # Find the least unique string column
    string_cols = df_src.select_dtypes(include=["object", "string"]).columns

    best_col = None
    best_unique_ratio = 1.0

    for col in string_cols:
        unique_ratio = df_src[col].nunique() / len(df_src)
        if 0.1 < unique_ratio < best_unique_ratio:
            best_col = col
            best_unique_ratio = unique_ratio

    if best_col is None and string_cols.size > 0:
        best_col = string_cols[0]

    return {
        "apply_blocking": True,
        "source_block_col": best_col,
        "ref_block_col": best_col,
        "block_key_len": k,
        "preproc": "exact_substring",
        "quality_score": 0.3,
        "is_fallback": True,
        "fallback_reason": f"High uniqueness detected - using truncation (k={k})",
    }


def _composite_fallback(
    df_src: pd.DataFrame, mode: BlockingMode, top_n_cols: int = 2
) -> dict:
    """Fallback strategy using composite keys."""
    # Find columns with moderate uniqueness
    string_cols = df_src.select_dtypes(include=["object", "string"]).columns

    candidates = []
    for col in string_cols[:10]:
        unique_ratio = df_src[col].nunique() / len(df_src)
        if 0.05 < unique_ratio < 0.8:
            candidates.append((col, unique_ratio))

    # Sort by uniqueness (prefer moderate uniqueness)
    candidates.sort(key=lambda x: abs(x[1] - 0.3))

    if len(candidates) >= 2:
        col1, col2 = candidates[0][0], candidates[1][0]
        composite_col = f"{col1}+{col2}"

        return {
            "apply_blocking": True,
            "source_block_col": composite_col,
            "ref_block_col": composite_col,
            "block_key_len": 6,
            "preproc": "alnum",
            "quality_score": 0.4,
            "is_fallback": True,
            "fallback_reason": f"Low uniqueness detected - using composite key ({col1}+{col2})",
        }

    # If can't create composite, fall back to single column
    return _truncation_fallback(df_src, mode, k=3)


def _unicode_safe_fallback(df_src: pd.DataFrame, mode: BlockingMode) -> dict:
    """Fallback strategy for encoding issues."""
    # Find string columns
    string_cols = df_src.select_dtypes(include=["object", "string"]).columns

    # Look for columns with less encoding chaos
    best_col = None
    least_chaos = float("inf")

    for col in string_cols[:10]:
        if ExoticDataDetector:
            try:
                profile = ExoticDataDetector.detect_characteristics(
                    df_src[col].head(100)
                )
                if not profile.get("encoding_issues", False):
                    best_col = col
                    break
                # Count special chars as proxy for encoding issues
                special_ratio = profile.get("special_chars_ratio", 1.0)
                if special_ratio < least_chaos:
                    least_chaos = special_ratio
                    best_col = col
            except:
                continue
        else:
            # Fallback if ExoticDataDetector not available
            if col == string_cols[0]:
                best_col = col

    if best_col is None and string_cols.size > 0:
        best_col = string_cols[0]

    return {
        "apply_blocking": True,
        "source_block_col": best_col,
        "ref_block_col": best_col,
        "block_key_len": 5,
        "preproc": "unicode_normalize",
        "quality_score": 0.35,
        "is_fallback": True,
        "fallback_reason": "Encoding issues detected - using Unicode normalization",
    }


def _resolve_name_prefix_ref_column(
    src_col: str,
    df_ref_sample: Optional[pd.DataFrame],
    mappings: Optional[list[dict]],
) -> Optional[str]:
    if df_ref_sample is None or df_ref_sample.empty:
        return None
    if src_col in df_ref_sample.columns:
        return src_col
    if mappings:
        for mapping in mappings:
            if mapping.get("source") == src_col:
                candidate = mapping.get("ref")
                if candidate and candidate in df_ref_sample.columns:
                    return candidate
    lower = src_col.lower()
    for ref_col in df_ref_sample.columns:
        if ref_col.lower() == lower:
            return ref_col
    for ref_col in df_ref_sample.columns:
        if any(keyword in ref_col.lower() for keyword in _NAME_PREFIX_KEYWORDS):
            return ref_col
    return None


def _generate_name_prefix_strategy(
    df_src_sample: pd.DataFrame,
    df_ref_sample: Optional[pd.DataFrame],
    mode: BlockingMode,
    mappings: Optional[list[dict]],
    *,
    target_bucket: int = _NAME_PREFIX_BUCKET_TARGET,
    bucket_range: tuple[int, int] = _NAME_PREFIX_BUCKET_RANGE,
    prefix_range: tuple[int, int] = (3, 5),
    total_src_rows: Optional[int] = None,
    total_ref_rows: Optional[int] = None,
) -> Optional[dict]:
    if df_src_sample is None or df_src_sample.empty:
        return None

    string_cols = df_src_sample.select_dtypes(include=["object", "string"]).columns
    if len(string_cols) == 0:
        return None

    total_src_rows = total_src_rows or len(df_src_sample)
    total_ref_rows = total_ref_rows or (
        len(df_ref_sample) if df_ref_sample is not None else 0
    )

    candidates: list[dict] = []

    for col in string_cols:
        col_lower = col.lower()
        if not any(keyword in col_lower for keyword in _NAME_PREFIX_KEYWORDS):
            continue
        series = df_src_sample[col]
        if series.dropna().empty:
            continue
        sample_size_src = len(series)
        unique_ratio = series.dropna().nunique() / max(sample_size_src, 1)
        if unique_ratio <= 0.9:
            continue
        tokens_series = series.fillna("").map(_normalize_company_tokens)
        coverage = float(tokens_series.map(bool).mean()) if sample_size_src else 0.0
        if coverage < 0.2:
            continue
        candidates.append(
            {
                "col": col,
                "coverage": coverage,
                "unique_ratio": unique_ratio,
                "tokens": tokens_series,
            }
        )

    if not candidates:
        return None

    candidates.sort(
        key=lambda item: (item["coverage"], item["unique_ratio"]), reverse=True
    )
    best = candidates[0]
    tokens_series = best["tokens"]

    evaluations: list[dict] = []
    prefix_start, prefix_stop = prefix_range
    for prefix_len in range(max(1, prefix_start), max(prefix_start, prefix_stop) + 1):
        keys = tokens_series.map(lambda toks: _render_name_prefix(toks, prefix_len))
        if keys.empty:
            continue
        mask = keys != ""
        if mask.sum() == 0:
            continue
        coverage = float(mask.mean())
        counts = keys[mask].value_counts()
        avg_bucket = float(counts.mean()) if len(counts) else 0.0
        p90_bucket = float(np.percentile(counts.to_numpy(), 90)) if len(counts) else 0.0
        evaluations.append(
            {
                "prefix_len": prefix_len,
                "coverage": coverage,
                "avg_bucket": avg_bucket,
                "p90_bucket": p90_bucket,
                "keys": keys,
                "counts": counts,
            }
        )

    if not evaluations:
        return None

    def _score_eval(item: dict) -> float:
        bucket_penalty = abs(item["avg_bucket"] - target_bucket)
        if not (bucket_range[0] <= item["avg_bucket"] <= bucket_range[1]):
            bucket_penalty *= 3
        coverage_penalty = max(0.0, 0.85 - item["coverage"]) * 200
        return bucket_penalty + coverage_penalty

    best_eval = min(evaluations, key=_score_eval)
    src_keys = best_eval["keys"]
    src_counts = best_eval["counts"]
    src_mask = src_keys != ""
    blocked_fraction_src = float(src_mask.mean()) if len(src_keys) else 0.0
    avg_bucket = float(best_eval["avg_bucket"])
    p90_bucket = float(best_eval["p90_bucket"])

    ref_col = None
    blocked_fraction_ref = None
    avg_bucket_ref = None
    p90_bucket_ref = None
    ref_keys = None
    ref_counts = None

    if (
        mode == BlockingMode.MATCH
        and df_ref_sample is not None
        and not df_ref_sample.empty
    ):
        ref_col = _resolve_name_prefix_ref_column(best["col"], df_ref_sample, mappings)
        if ref_col is None:
            return None
        ref_tokens = df_ref_sample[ref_col].fillna("").map(_normalize_company_tokens)
        ref_keys = ref_tokens.map(
            lambda toks: _render_name_prefix(toks, best_eval["prefix_len"])
        )
        if not ref_keys.empty:
            ref_mask = ref_keys != ""
            blocked_fraction_ref = float(ref_mask.mean())
            ref_counts = ref_keys[ref_mask].value_counts()
            avg_bucket_ref = float(ref_counts.mean()) if len(ref_counts) else 0.0
            p90_bucket_ref = (
                float(np.percentile(ref_counts.to_numpy(), 90))
                if len(ref_counts)
                else 0.0
            )
        else:
            blocked_fraction_ref = 0.0
            avg_bucket_ref = 0.0
            p90_bucket_ref = 0.0
            ref_counts = pd.Series(dtype=float)
    else:
        ref_counts = pd.Series(dtype=float)

    blocked_percentage = blocked_fraction_src * 100.0
    message = (
        f"Auto-blocking used {best['col']} (prefix). ~{blocked_percentage:.0f}% of rows blocked; "
        f"avg candidates ~= {avg_bucket:.0f}."
    )

    sample_size_src = len(df_src_sample)
    counts_array = (
        src_counts.to_numpy(dtype=float)
        if not src_counts.empty
        else np.array([], dtype=float)
    )
    ordered_sum_sample = (
        float((counts_array * (counts_array - 1)).sum()) if counts_array.size else 0.0
    )
    pair_sum_sample = ordered_sum_sample / 2.0
    scale_src = total_src_rows / max(sample_size_src, 1)
    scaled_counts = (
        counts_array * scale_src if counts_array.size else np.array([], dtype=float)
    )
    ordered_sum_projected = (
        float((scaled_counts * (scaled_counts - 1)).sum())
        if scaled_counts.size
        else 0.0
    )
    projected_pairs = ordered_sum_projected / 2.0
    avg_candidates_per_src_row_sample = ordered_sum_sample / max(sample_size_src, 1)
    avg_candidates_per_src_row_projected = ordered_sum_projected / max(
        total_src_rows, 1
    )

    if (
        mode == BlockingMode.MATCH
        and ref_col is not None
        and ref_counts is not None
        and not ref_counts.empty
    ):
        sample_size_ref = len(df_ref_sample) if df_ref_sample is not None else 0
        ref_counts_array = ref_counts.reindex(
            src_counts.index, fill_value=0.0
        ).to_numpy(dtype=float)
        scale_ref = total_ref_rows / max(sample_size_ref, 1) if sample_size_ref else 0.0
        match_candidates_sample = (
            float((counts_array * ref_counts_array).sum()) if counts_array.size else 0.0
        )
        match_candidates_projected = (
            float(((counts_array * scale_src) * (ref_counts_array * scale_ref)).sum())
            if counts_array.size
            else 0.0
        )
        avg_candidates_per_src_row_sample = match_candidates_sample / max(
            sample_size_src, 1
        )
        avg_candidates_per_src_row_projected = match_candidates_projected / max(
            total_src_rows, 1
        )
        projected_pairs = match_candidates_projected
        ordered_sum_projected = (
            match_candidates_projected * 1.0
        )  # For consistency in guard output
    else:
        scale_ref = 0.0
        match_candidates_sample = None

    strategy = {
        "apply_blocking": True,
        "source_block_col": best["col"],
        "ref_block_col": ref_col,
        "block_key_len": best_eval["prefix_len"],
        "preproc": "name_prefix",
        "quality_score": 0.32,
        "is_fallback": True,
        "fallback_reason": "Name-based prefix fallback",
        "engine_block_strategy": "name_prefix",
        "blocked_fraction": round(blocked_fraction_src, 4),
        "avg_bucket_size": round(avg_bucket, 2),
        "p90_bucket_size": round(p90_bucket, 2),
        "user_message": message,
        "partial_blocking": blocked_fraction_src < 0.98
        or (blocked_fraction_ref is not None and blocked_fraction_ref < 0.98),
        "residual_handling": {
            "mode": "cap_candidates_per_row",
            "limit": 1000,
        },
        "guardrail_triggered": "name_prefix",
        "name_prefix_metrics": {
            "blocked_fraction_src": blocked_fraction_src,
            "blocked_fraction_ref": blocked_fraction_ref,
            "avg_bucket_src": avg_bucket,
            "avg_bucket_ref": avg_bucket_ref,
            "p90_bucket_src": p90_bucket,
            "p90_bucket_ref": p90_bucket_ref,
        },
        "estimated_pairs_after": int(projected_pairs),
        "estimated_pairs_before": int(
            (total_src_rows * (total_src_rows - 1)) // 2
            if mode == BlockingMode.DEDUPE
            else total_src_rows * max(total_ref_rows, 0)
        ),
        "projected_candidate_pairs": int(projected_pairs),
        "guard_estimates": {
            "avg_candidates_per_src_row_sample": round(
                avg_candidates_per_src_row_sample, 6
            ),
            "avg_candidates_per_src_row_projected": round(
                avg_candidates_per_src_row_projected, 6
            ),
            "projected_candidate_pairs": int(projected_pairs),
            "sample_ordered_candidates": round(ordered_sum_sample, 3),
            "projected_ordered_candidates": round(ordered_sum_projected, 3),
            "sample_size_src": sample_size_src,
            "total_src_rows": total_src_rows,
            "scale_src": scale_src,
            "residual_fraction_src": round(max(0.0, 1.0 - blocked_fraction_src), 4),
        },
    }

    if blocked_fraction_ref is not None:
        strategy.update(
            {
                "blocked_fraction_ref": round(blocked_fraction_ref, 4),
            }
        )
    if avg_bucket_ref is not None:
        strategy.update(
            {
                "avg_bucket_size_ref": round(avg_bucket_ref or 0.0, 2),
                "p90_bucket_size_ref": round(p90_bucket_ref or 0.0, 2),
            }
        )
    if mode == BlockingMode.MATCH and ref_col is not None:
        strategy["guard_estimates"].update(
            {
                "total_ref_rows": total_ref_rows,
                "scale_ref": scale_ref,
            }
        )
    if ref_keys is not None:
        strategy["ref_sample_keys"] = ref_keys.head(5).tolist()
    strategy["src_sample_keys"] = src_keys.head(5).tolist()

    return strategy


def detect_optimal_variants(
    series: pd.Series, col_name: str, metrics: Optional[dict] = None
) -> list[str]:
    """Detect which preprocessing variants would work best for this column."""
    variants = ["alnum"]  # Always include basic

    # Get exotic characteristics from metrics if available
    if metrics and "script" in metrics:
        script = metrics.get("script", "unknown")
        semantic_type = metrics.get("semantic_type")
        has_emoji = metrics.get("has_emoji", False)
        encoding_issues = metrics.get("encoding_issues", False)
    else:
        # Fallback to detection if metrics not passed
        if ExoticDataDetector:
            try:
                profile = ExoticDataDetector.detect_characteristics(series.head(1000))
                script = profile.get("script", "unknown")
                semantic_type = profile.get("semantic_type")
                has_emoji = profile.get("has_emoji", False)
                encoding_issues = profile.get("encoding_issues", False)
            except:
                script = "unknown"
                semantic_type = None
                has_emoji = False
                encoding_issues = False
        else:
            script = "unknown"
            semantic_type = None
            has_emoji = False
            encoding_issues = False

    # Sample for analysis
    sample = series.dropna().head(1000).astype(str)
    if sample.empty:
        return variants

    col_lower = col_name.lower()

    # Script-aware variant selection
    if script in ["arabic", "hebrew"]:
        # RTL scripts - avoid soundex
        variants.append("unicode_normalize")
        if "soundex" in variants:
            variants.remove("soundex")
    elif script == "cyrillic":
        # Cyrillic - transliteration might help
        variants.append("unicode_normalize")
        variants.append("first_2_words")  # Often useful for Slavic names
    elif script == "cjk":
        # CJK - character-based approaches
        variants = ["exact_substring", "first_n_chars"]
        # Don't use phonetic encodings for CJK
    elif script == "thai":
        # Thai - no word boundaries
        variants = ["exact_substring", "unicode_normalize"]

    # Encoding issues - add safe variants
    if encoding_issues:
        variants.append("unicode_normalize")
        # Prefer exact matching when encoding is messy
        if "exact" not in variants:
            variants.insert(0, "exact_substring")

    # Emoji handling
    if has_emoji:
        # Strip emoji variant
        variants.append("strip_emoji")

    # Semantic-based selection (existing logic enhanced)
    if semantic_type == "email":
        variants.append("email_domain")
    elif semantic_type == "phone":
        variants.extend(["phone_digits", "phone_area"])
    elif semantic_type == "url":
        variants.append("domain_from_url")
    elif semantic_type == "postal_code":
        variants.append("exact_substring")
    elif semantic_type == "product_code":
        # Product codes need exact or very similar matches
        variants = ["exact", "alnum"]

    # Column name based detection (fallback)
    if semantic_type is None:
        # Email detection
        if any(term in col_lower for term in ["email", "mail"]):
            variants.append("email_domain")
        # Phone detection
        elif any(term in col_lower for term in ["phone", "mobile", "tel"]):
            variants.extend(["phone_digits", "phone_area"])
        # Company detection
        elif any(term in col_lower for term in ["company", "account", "customer"]):
            variants.append("company_stem")
        # Address detection
        elif any(term in col_lower for term in ["address", "street"]):
            variants.append("address_number")
        # Name detection
        elif any(term in col_lower for term in ["name", "contact", "person"]):
            if script == "latin":
                variants.extend(["name_initials", "metaphone"])
            else:
                # Non-Latin names - avoid phonetic
                variants.append("first_1_word")

    # Add phonetic only if appropriate
    if script == "latin" and semantic_type not in [
        "email",
        "product_code",
        "postal_code",
    ]:
        unique_ratio = series.nunique() / len(series)
        if (
            unique_ratio > 0.8
            and "metaphone" not in variants
            and "soundex" not in variants
        ):
            variants.append("soundex")

    return list(dict.fromkeys(variants))  # Remove duplicates, preserve order


# In engine.py


def _determine_optimal_blocking_strategy(
    df_src: pd.DataFrame,
    ref_df: Optional[pd.DataFrame],
    mappings: list[dict],
    mode: BlockingMode,
    ruleset: str,
    weight_profile: str = "balanced",  # NEW: Configurable weights
    custom_weights: Optional[dict] = None,  # NEW: Custom weight support
    progress_callback: Optional[callable] = None,  # NEW: Progress monitoring
    preferred_src: Optional[list[str]] = None,
    preferred_ref: Optional[list[str]] = None,
    restrict_to_pairs: Optional[list[tuple[str, str]]] = None,
    sample_rows: int = 50_000,
    key_len_range: tuple[int, int] = (2, 6),
    preprocessing_variants: Optional[
        tuple[str, ...]
    ] = None,  # Now optional for auto-detection
    enable_validation: bool = True,  # NEW: Column validation
    algorithm: str = DEFAULT_ALGORITHM_KEY,  # NEW: Algorithm to evaluate
    stats_cache: ColumnStatsCache | None = None,
) -> dict:
    """
    Determines an optimal blocking strategy by evaluating candidate columns,
    key lengths, and preprocessing variants using a Quality Score.

    Enhanced with:
    - Intelligent preprocessing variant selection
    - Configurable weight profiles
    - Column validation
    - Smart fallback strategies
    - Progress monitoring
    - Adaptive data-specific thresholds
    - Algorithm fitness evaluation
    """
    global log, HARD_CAP_COMPS, MIN_ACCEPTABLE_QS, ENGINE_CONFIG

    # The 'mode' argument can come in as a string from a shim, convert to Enum
    if isinstance(mode, str):
        mode = BlockingMode(mode)

    log.info(
        f"Determining optimal blocking strategy. Mode: {mode.name}, Ruleset: {ruleset}, "
        f"WeightProfile: {weight_profile}, SampleRows: {sample_rows}, Algorithm: {algorithm}"
    )
    log.debug(
        f"Mappings: {mappings}, KeyLenRange: {key_len_range}, PreprocessingVariants: {preprocessing_variants}"
    )

    stats_cache = stats_cache or ColumnStatsCache()

    algorithm_to_test = algorithm

    # ---------- 0. Setup Weights ----------
    if custom_weights:
        is_valid, msg = BlockingWeightProfile.validate_weights(custom_weights)
        if not is_valid:
            log.warning(
                f"Invalid custom weights: {msg}. Using {weight_profile} profile."
            )
            active_weights = BlockingWeightProfile.get_weights(weight_profile)
        else:
            active_weights = custom_weights
            log.info("Using custom weights")
    else:
        active_weights = BlockingWeightProfile.get_weights(weight_profile)

    log.debug(f"Active weights: {active_weights}")

    # ---------- 1. Brute-force skip (Layer A) ----------
    N_src = len(df_src)
    N_ref = (
        len(ref_df)
        if ref_df is not None and mode == BlockingMode.MATCH
        else (N_src if mode == BlockingMode.DEDUPE else 0)
    )

    if N_src == 0:
        log.warning(
            "determine_optimal_blocking_strategy: Source DataFrame is empty. Cannot apply blocking."
        )
        return_val = {"apply_blocking": False, "reason": "Source DataFrame empty"}
        log.critical(
            f"DEBUG RETURN PATH: Source DataFrame empty. Returning: {return_val}"
        )
        return return_val

    brute_pairs = (
        (N_src * (N_src - 1) // 2)
        if mode == BlockingMode.DEDUPE
        else (N_src * N_ref if N_ref > 0 else 0)
    )

    log.debug(
        f"Estimated brute-force pairs: {brute_pairs}. HARD_CAP_COMPS: {HARD_CAP_COMPS}"
    )
    if brute_pairs < HARD_CAP_COMPS:
        log.info("Brute-force pair count is below threshold. Recommending NO BLOCKING.")
        return_val = {
            "apply_blocking": False,
            "reason": f"Brute-force pairs ({brute_pairs}) < threshold ({HARD_CAP_COMPS})",
        }
        log.critical(
            f"DEBUG RETURN PATH: HARD_CAP_COMPS triggered (brute_pairs < threshold). Returning: {return_val}"
        )
        return return_val

    # ---------- 2. Sample DataFrames (Layer B part 1) -----------
    # Sample source DataFrame
    actual_src_sample_size = min(sample_rows, N_src)
    df_src_s = (
        df_src.sample(n=actual_src_sample_size, random_state=42)
        if N_src > 0
        else df_src.copy()
    )

    df_ref_s = None
    if mode == BlockingMode.MATCH and ref_df is not None and not ref_df.empty:
        # For reference DataFrame, use the same sample size as source OR the full ref if it's smaller
        N_ref = len(ref_df)
        if N_ref <= actual_src_sample_size:
            # Reference is smaller than or equal to source sample - use it all
            df_ref_s = ref_df.copy()
        else:
            # Reference is larger - sample the same size as source sample
            df_ref_s = ref_df.sample(n=actual_src_sample_size, random_state=42)

    # ---------- 2b. Calculate Adaptive Thresholds (NEW) ----------
    adaptive_thresholds = calculate_adaptive_thresholds(df_src_s)
    min_qs_adaptive = adaptive_thresholds["min_acceptable_qs"]
    log.info(f"Using adaptive thresholds: {adaptive_thresholds}")

    log.debug(
        f"Sampled df_src_s: {len(df_src_s)} rows. Sampled df_ref_s: {len(df_ref_s) if df_ref_s is not None else 'N/A'} rows."
    )

    # ---------- 3. Generate Candidate Blocking Pairs (Layer B part 2) --------
    # First, discover semantic blocking columns (domain, email, phone, etc.) even if unmapped
    discovered_pairs = _discover_semantic_blocking_columns(df_src_s, df_ref_s, mode)

    # Then, get candidates from mappings
    mapped_pairs = _candidate_pairs(
        mappings=mappings,
        df_src=df_src_s,
        mode=mode,
        df_ref=df_ref_s,
        allow_all_mapped_candidates=True,
    )

    # Merge discovered + mapped, removing duplicates
    candidate_blocking_pairs = list(set(discovered_pairs + mapped_pairs))
    log.info(
        f"Candidate blocking columns: {len(discovered_pairs)} discovered + {len(mapped_pairs)} mapped = {len(candidate_blocking_pairs)} total"
    )

    # ---------- 3a. Validate Candidates (NEW) ----------
    validated_candidates = []
    if enable_validation and candidate_blocking_pairs:
        log.info("Validating candidate blocking columns...")
        validated_candidates = validate_blocking_candidates(
            candidate_blocking_pairs, df_src_s, df_ref_s, mode
        )

        # Filter out invalid candidates
        valid_pairs = [
            (vc["src_col"], vc["ref_col"])
            for vc in validated_candidates
            if vc["overall_valid"]
        ]

        # Log validation results
        invalid_count = len(candidate_blocking_pairs) - len(valid_pairs)
        if invalid_count > 0:
            log.warning(
                f"Validation removed {invalid_count} unsuitable candidate columns"
            )
            for vc in validated_candidates:
                if not vc["overall_valid"]:
                    log.debug(f"  Column '{vc['src_col']}' issues: {vc['src_issues']}")

        # Update candidates
        if valid_pairs:
            candidate_blocking_pairs = valid_pairs
        else:
            log.warning(
                "All candidates failed validation. Proceeding with original candidates."
            )

    if not candidate_blocking_pairs:
        log.warning(
            "No suitable candidate blocking columns found. Attempting name-prefix fallback."
        )
        name_prefix_strategy = _generate_name_prefix_strategy(
            df_src_sample=df_src_s,
            df_ref_sample=df_ref_s,
            mode=mode,
            mappings=mappings,
            total_src_rows=N_src,
            total_ref_rows=N_ref if mode == BlockingMode.MATCH else None,
        )
        if name_prefix_strategy:
            log.info(
                "Name-prefix fallback strategy selected (source column: %s).",
                name_prefix_strategy.get("source_block_col"),
            )
            log.critical(
                f"DEBUG RETURN PATH: Name-prefix fallback applied. Returning: {name_prefix_strategy}"
            )
            return name_prefix_strategy
        log.warning("Name-prefix fallback unavailable. Applying smart fallback.")
        fallback_result = smart_fallback(
            df_src_s, mode, ruleset, df_ref_s, weight_profile, stats_cache=stats_cache
        )
        log.critical(
            f"DEBUG RETURN PATH: No candidate blocking pairs. Calling smart_fallback. Fallback Result: {fallback_result}"
        )
        return fallback_result

    # ---------- 3b. Determine Preprocessing Variants (NEW) ----------
    detected_variants_per_col = {}

    if preprocessing_variants is None:
        log.info("Auto-detecting optimal preprocessing variants...")

        for idx, (src_col, ref_col) in enumerate(candidate_blocking_pairs):
            # Get validation metrics if available
            col_metrics = None
            if idx < len(validated_candidates):
                col_metrics = validated_candidates[idx].get("src_metrics", {})

            # Detect variants for source column with metrics
            src_variants = detect_optimal_variants(
                df_src_s[src_col], src_col, col_metrics
            )
            detected_variants_per_col[src_col] = src_variants
            log.debug(f"Column '{src_col}' detected variants: {src_variants}")

        # Aggregate all detected variants
        all_variants = set()
        for variants in detected_variants_per_col.values():
            all_variants.update(variants)

        # Ensure we have some variants
        if not all_variants:
            all_variants = {"alnum", "soundex", "first_1_word"}

        preprocessing_variants = tuple(all_variants)
        log.info(f"Using auto-detected variants: {preprocessing_variants}")

    # ---------- 4. Setup Progress Monitoring (NEW) ----------
    total_strategies = (
        len(candidate_blocking_pairs)
        * len(preprocessing_variants)
        * (key_len_range[1] - key_len_range[0] + 1)
    )
    monitor = (
        BlockingAnalysisMonitor(total_strategies, progress_callback)
        if progress_callback
        else None
    )

    evaluated_strategies = []
    best_strategy_so_far = None

    # ---------- 5. Main Evaluation Loop --------
    log.info(f"Starting evaluation of {total_strategies} blocking strategies.")

    for idx, (src_col, ref_col_actual) in enumerate(candidate_blocking_pairs):
        # Use default key length range unless it's a domain column
        current_key_len_range = key_len_range
        if "domain" in src_col.lower() or "website" in src_col.lower():
            # Domains need longer keys to be useful, but not too long
            current_key_len_range = (3, 5)  # Instead of default (2, 6)
            log.info(
                f"Domain column detected ('{src_col}'), using key_len_range {current_key_len_range}"
            )

        # Get column validation metrics if available
        src_col_metrics = {}
        if idx < len(validated_candidates):
            src_col_metrics = validated_candidates[idx].get("src_metrics", {})

        # Check if we should use column-specific variants
        if src_col in detected_variants_per_col:
            col_specific_variants = detected_variants_per_col[src_col]
        else:
            col_specific_variants = preprocessing_variants

        for preproc_variant in col_specific_variants:
            for k_len in range(
                current_key_len_range[1], current_key_len_range[0] - 1, -1
            ):
                strategy_info = {
                    "src_col": src_col,
                    "ref_col": ref_col_actual,
                    "k_len": k_len,
                    "preproc": preproc_variant,
                }

                if log.isEnabledFor(logging.DEBUG):
                    log.debug(
                        f"Evaluating: SrcCol='{src_col}', RefCol='{ref_col_actual}', "
                        f"Preproc='{preproc_variant}', InputKLen={k_len}"
                    )

                # ---- Build keys for current strategy ----
                try:
                    # *** THE FIX - PART 1 ***
                    # Do NOT reset the index. Let all derived series share the same
                    # index as the sample DataFrame (df_src_s) to prevent KeyErrors.
                    src_col_series_for_eval = df_src_s[src_col]
                    if stats_cache:
                        src_col_series_for_eval = stats_cache.remember_series(
                            src_col_series_for_eval, id(df_src_s)
                        )
                    current_src_keys, eff_src_k = build_key(
                        src_col_series_for_eval,
                        preproc_variant,
                        k_len,
                        ruleset,
                        stats_cache=stats_cache,
                    )
                except Exception as e:
                    log.warning(
                        f"Failed to build keys for {src_col} with {preproc_variant}: {e}"
                    )
                    if monitor:
                        monitor.update(strategy_info, 0.0)
                    continue

                current_ref_keys, eff_ref_k = (None, k_len)
                ref_col_series_for_eval = None  # Initialize
                if (
                    mode == BlockingMode.MATCH
                    and df_ref_s is not None
                    and ref_col_actual in df_ref_s.columns
                ):
                    try:
                        # *** THE FIX - PART 2 ***
                        # Same fix for the reference dataframe series.
                        ref_col_series_for_eval = df_ref_s[ref_col_actual]
                        if stats_cache:
                            ref_col_series_for_eval = stats_cache.remember_series(
                                ref_col_series_for_eval, id(df_ref_s)
                            )
                        current_ref_keys, eff_ref_k = build_key(
                            ref_col_series_for_eval,
                            preproc_variant,
                            k_len,
                            ruleset,
                            stats_cache=stats_cache,
                        )
                    except Exception as e:
                        log.warning(
                            f"Failed to build ref keys for {ref_col_actual} with {preproc_variant}: {e}"
                        )
                        if monitor:
                            monitor.update(strategy_info, 0.0)
                        continue
                elif mode == BlockingMode.MATCH and (
                    df_ref_s is None or ref_col_actual not in df_ref_s.columns
                ):
                    log.debug(
                        f"Ref col '{ref_col_actual}' not found in ref sample for match mode. Skipping."
                    )
                    if monitor:
                        monitor.update(strategy_info, 0.0)
                    continue

                # --- Phonetic blocking heuristic ---
                if (
                    preproc_variant.startswith("soundex")
                    or preproc_variant == "metaphone"
                ):
                    meaningful_src_keys = current_src_keys[current_src_keys != ""]
                    unique_block_count = meaningful_src_keys.nunique()
                    sample_size_val = len(df_src_s)

                    if sample_size_val > 0:
                        unique_block_ratio = unique_block_count / sample_size_val
                        if unique_block_ratio < 0.02:
                            log.info(
                                f"Strategy rejected: Phonetic variant produced too few unique blocks "
                                f"({unique_block_count} unique of {sample_size_val} samples, "
                                f"ratio {unique_block_ratio:.4f} < 0.02)."
                            )
                            if monitor:
                                monitor.update(strategy_info, 0.0)
                            continue

                # ---- Calculate Raw Signals ----
                raw_signals_dict = calculate_raw_signals(
                    current_src_keys, eff_src_k, mode, current_ref_keys
                )

                if raw_signals_dict is None:
                    log.debug(
                        "Raw signals returned None (candidate pruned). Skipping."
                    )
                    if monitor:
                        monitor.update(strategy_info, 0.0)
                    continue

                # ---- Early Reject based on raw signals with adaptive thresholds ----
                if early_reject(raw_signals_dict, adaptive_thresholds):
                    log.info(
                        f"Strategy early rejected with adaptive thresholds. Raw: {raw_signals_dict}"
                    )
                    if monitor:
                        monitor.update(strategy_info, 0.0)
                    continue

                # ---- Normalize Signals ----
                normalized_signals_dict = normalize_signals(
                    raw_signals_dict, preproc_variant
                )

                # ---- NEW: Algorithm Fitness Evaluation ----
                if ENGINE_CONFIG.enable_algorithm_fitness:
                    # Sample pairs for fitness evaluation - pass actual column values!
                    sample_pairs = _sample_block_pairs(
                        current_src_keys,
                        current_ref_keys,
                        src_col_series_for_eval,  # Pass original-indexed series
                        ref_col_series_for_eval,  # Pass original-indexed series
                        n=ENGINE_CONFIG.algorithm_fitness_sample_size,
                    )

                    if sample_pairs:
                        # Create column profile for fitness evaluation
                        column_profile = {
                            "script": src_col_metrics.get("script", "unknown"),
                            "semantic_type": src_col_metrics.get("semantic_type"),
                            "avg_length": src_col_metrics.get("avg_length", 10),
                            "has_emoji": src_col_metrics.get("has_emoji", False),
                            "encoding_issues": src_col_metrics.get(
                                "encoding_issues", False
                            ),
                        }

                        # Calculate fitness
                        algo_fitness = _quick_algo_fitness(
                            sample_pairs, algorithm_to_test, column_profile
                        )

                        # Add to normalized signals with lowercase key!
                        normalized_signals_dict["af"] = algo_fitness

                        log.debug(
                            f"Algorithm fitness for {algorithm_to_test} on {src_col}: {algo_fitness:.3f}"
                        )

                # ---- Calculate Quality Score with active weights ----
                current_qs = quality_score(normalized_signals_dict, active_weights)

                # Semantic penalty/bonus heuristics
                src_col_lower = src_col.lower()
                ref_col_lower = ref_col_actual.lower()
                if "domain" in ref_col_lower:
                    # Prefer true domain-to-domain blocking
                    if "domain" in src_col_lower:
                        current_qs *= 1.05
                    elif "email" in src_col_lower and preproc_variant not in (
                        "email_domain",
                        "domain_from_url",
                    ):
                        penalty = (
                            0.25 if preproc_variant.startswith("soundex") else 0.15
                        )
                        current_qs *= max(0.0, 1.0 - penalty)
                        log.info(
                            "Applying email/domain mismatch penalty: src=%s, ref=%s, variant=%s, penalty=%.2f",
                            src_col,
                            ref_col_actual,
                            preproc_variant,
                            penalty,
                        )

                log.debug(
                    f"  Calculated QS: {current_qs:.4f}. Raw: {raw_signals_dict}, Norm: {normalized_signals_dict}"
                )
                log.debug(
                    f"Cand {src_col}-{preproc_variant}-{k_len} | "
                    f"H {raw_signals_dict.get('H',0.0):.2f} "
                    f"RR {raw_signals_dict.get('RR',0.0):.2f} "
                    f"P1 {raw_signals_dict.get('P1',0.0):.2f} "
                    f"QS_eff_k_{eff_src_k:.0f} {current_qs:.2f}"
                )

                strategy_details = {
                    "qs": current_qs,
                    "src_col": src_col,
                    "ref_col": ref_col_actual if mode == BlockingMode.MATCH else None,
                    "k_len": k_len,
                    "eff_k": eff_src_k,
                    "preproc": preproc_variant,
                    "raw_metrics": raw_signals_dict,
                    "normalized_metrics": normalized_signals_dict,
                    "weight_profile": weight_profile,
                    "algorithm": algorithm,  # Include algorithm in strategy
                }

                evaluated_strategies.append(strategy_details)

                if (
                    best_strategy_so_far is None
                    or current_qs > best_strategy_so_far["qs"]
                ):
                    best_strategy_so_far = strategy_details
                    log.info(
                        f"  New best strategy found: QS={current_qs:.4f}, Details: {strategy_details}"
                    )

                # Update progress monitor
                if monitor:
                    monitor.update(strategy_info, current_qs)

                # Early termination if we found an excellent strategy
                if current_qs > 0.95:
                    log.info(
                        f"Found excellent strategy (QS={current_qs:.4f}), stopping search early"
                    )
                    break

    # ---------- 6. Choose / Fallback with adaptive thresholds -------------
    if not best_strategy_so_far:
        log.warning(
            "No strategies were successfully evaluated. Attempting name-prefix fallback."
        )
        name_prefix_strategy = _generate_name_prefix_strategy(
            df_src_sample=df_src_s,
            df_ref_sample=df_ref_s,
            mode=mode,
            mappings=mappings,
            total_src_rows=N_src,
            total_ref_rows=N_ref if mode == BlockingMode.MATCH else None,
        )
        if name_prefix_strategy:
            log.info(
                "Name-prefix fallback strategy selected (source column: %s).",
                name_prefix_strategy.get("source_block_col"),
            )
            log.critical(
                f"DEBUG RETURN PATH: Name-prefix fallback applied. Returning: {name_prefix_strategy}"
            )
            return name_prefix_strategy
        log.warning("Name-prefix fallback unavailable. Applying smart fallback.")
        fallback_result = smart_fallback(
            df_src_s, mode, ruleset, df_ref_s, weight_profile, stats_cache=stats_cache
        )
        log.critical(
            f"DEBUG RETURN PATH: No best_strategy_so_far. Calling smart_fallback. Fallback Result: {fallback_result}"
        )
        return fallback_result

    if best_strategy_so_far["qs"] < min_qs_adaptive:
        log.warning(
            f"Best strategy QS ({best_strategy_so_far['qs']:.4f}) is below adaptive MIN_ACCEPTABLE_QS ({min_qs_adaptive}). "
            f"Attempting fallback."
        )
        log.debug(f"Details of under-performing best strategy: {best_strategy_so_far}")
        name_prefix_strategy = _generate_name_prefix_strategy(
            df_src_sample=df_src_s,
            df_ref_sample=df_ref_s,
            mode=mode,
            mappings=mappings,
            total_src_rows=N_src,
            total_ref_rows=N_ref if mode == BlockingMode.MATCH else None,
        )
        if name_prefix_strategy:
            log.info(
                "Name-prefix fallback strategy selected (source column: %s).",
                name_prefix_strategy.get("source_block_col"),
            )
            log.critical(
                f"DEBUG RETURN PATH: Name-prefix fallback applied after low QS. Returning: {name_prefix_strategy}"
            )
            return name_prefix_strategy
        log.warning("Name-prefix fallback unavailable. Applying smart fallback.")
        fallback_result = smart_fallback(
            df_src_s, mode, ruleset, df_ref_s, weight_profile, stats_cache=stats_cache
        )
        log.critical(
            f"DEBUG RETURN PATH: Best strategy QS below adaptive MIN_ACCEPTABLE_QS. Calling smart_fallback. "
            f"Fallback Result: {fallback_result}"
        )
        return fallback_result

    # ---------- 7. Get Analysis Summary (NEW) ----------
    if monitor:
        summary = monitor.get_summary()
        log.info(
            f"Blocking analysis complete. Summary: Total evaluated: {summary['total_evaluated']}, "
            f"Elapsed: {summary['elapsed_time']:.1f}s, Best score: {summary['best_score']:.4f}"
        )

    log.info(
        f"Optimal strategy chosen: QS={best_strategy_so_far['qs']:.4f}, Details: {best_strategy_so_far}"
    )

    # Prepare final output dictionary with adaptive thresholds
    # Calculate estimated comparisons after blocking
    # RR (reduction ratio) tells us how many comparisons we save
    # estimated_after = estimated_before * (1 - RR)
    estimated_pairs_before = brute_pairs
    reduction_ratio = best_strategy_so_far["raw_metrics"].get("RR", 0.0)
    estimated_pairs_after = int(brute_pairs * (1.0 - reduction_ratio))

    # Extract coverage from raw metrics if available
    # P1 (dominant block proportion) and blank ratios give us coverage info
    coverage_estimate = 1.0 - (
        best_strategy_so_far["raw_metrics"].get("P1", 0.0)
    )  # Rough estimate

    final_decision = {
        "apply_blocking": True,
        "source_block_col": best_strategy_so_far["src_col"],
        "ref_block_col": best_strategy_so_far["ref_col"],
        "block_key_len": best_strategy_so_far["k_len"],
        "preproc": best_strategy_so_far["preproc"],
        "quality_score": round(best_strategy_so_far["qs"], 4),
        "raw_metrics": best_strategy_so_far["raw_metrics"],
        "normalized_metrics": best_strategy_so_far["normalized_metrics"],
        "is_fallback": False,
        "weight_profile": weight_profile,
        "engine_block_strategy": best_strategy_so_far["preproc"],
        "guardrail_triggered": None,
        "strategies_evaluated": len(evaluated_strategies),
        "preprocessing_variants_used": list(preprocessing_variants)
        if preprocessing_variants
        else "auto-detected",
        "adaptive_thresholds": adaptive_thresholds,  # NEW: For debugging/reporting
        "recommended_algorithm": algorithm
        if ENGINE_CONFIG.enable_algorithm_fitness
        else None,  # NEW: Algorithm recommendation
        # RICHER TELEMETRY: Estimated comparison counts and coverage
        "estimated_pairs_before": estimated_pairs_before,
        "estimated_pairs_after": estimated_pairs_after,
        "reduction_ratio": round(reduction_ratio, 4),
        "coverage_estimate": round(coverage_estimate, 4),
    }

    guard = BlockGuardrails()
    guard_limit = getattr(ENGINE_CONFIG, "max_candidate_pairs", guard.max_pairs)
    projected_pairs = _projected_pairs_from_strategy(final_decision, brute_pairs)
    if projected_pairs is None:
        projected_pairs = brute_pairs
    if projected_pairs is not None:
        final_decision["projected_candidate_pairs"] = int(projected_pairs)
    final_decision["guardrail_limit"] = guard_limit

    if projected_pairs is not None and projected_pairs > guard_limit:
        log.warning(
            "Projected comparisons exceed guardrail limit: %s > %s",
            f"{projected_pairs:,}",
            f"{guard_limit:,}",
        )
        if final_decision.get("preproc") != "name_prefix":
            name_guard = _generate_name_prefix_strategy(
                df_src_sample=df_src_s,
                df_ref_sample=df_ref_s,
                mode=mode,
                mappings=mappings,
                total_src_rows=N_src,
                total_ref_rows=N_ref if mode == BlockingMode.MATCH else None,
            )
            fallback_pairs = (
                _projected_pairs_from_strategy(name_guard, brute_pairs)
                if name_guard
                else None
            )
            if (
                name_guard
                and fallback_pairs is not None
                and fallback_pairs <= guard_limit
            ):
                name_guard["guardrail_triggered"] = "forced_name_prefix"
                name_guard["guardrail_limit"] = guard_limit
                if "user_message" in name_guard and name_guard["user_message"]:
                    name_guard["user_message"] += (
                        " Guardrail triggered; prefix fallback applied automatically."
                    )
                else:
                    name_guard["user_message"] = (
                        "Guardrail triggered; prefix fallback applied automatically."
                    )
                log.info(
                    "Guardrail fallback: switching to name-prefix strategy (%s <= %s).",
                    f"{fallback_pairs:,}",
                    f"{guard_limit:,}",
                )
                final_decision = name_guard
                projected_pairs = fallback_pairs
            elif name_guard:
                final_decision = name_guard
                projected_pairs = fallback_pairs
        if projected_pairs is not None and projected_pairs > guard_limit:
            message = (
                f"Projected comparisons {projected_pairs:,} exceed guardrail limit {guard_limit:,}. "
                "Add email/domain columns or pre-filter input to reduce candidate volume."
            )
            log.error(message)
            raise BlockingGuardrailError(message)

    # Add this right before the final return statement:
    # Enhanced with algorithm selection rules if needed
    if (
        mappings
        and validated_candidates
        and not final_decision.get("is_fallback", False)
    ):
        final_decision = enhance_strategy_with_algorithm_rules(
            final_decision, validated_candidates, mappings
        )

    log.critical(
        f"DEBUG RETURN PATH: Optimal strategy found. Returning final_decision: {final_decision}"
    )
    return final_decision


# ------------------------------------------------------------------
# Auto-select the best blocking key
# ------------------------------------------------------------------

REDUCTION_MAX = 0.99  # 99 % maximum reduction we still accept
KEY_LEN_ITER_RANGE = range(2, 7)  # test key-lengths 2,3,4,5,6


# --- Helper for blocking analysis (moved to module level) ---
# This function was previously nested inside suggest_blocking
def _calculate_metrics_for_key_length(
    base_cleaned_series: pd.Series,  # Series after general cleaning (lower, alphanum etc.)
    k_len: int,  # The key length to test
    total_rows_in_sample: int,
    max_possible_pairs_in_sample: int,
    preview_count: int = 3,
) -> Tuple[
    Optional[float], float, float, List[str]
]:  # quality_score, entropy, reduction_pct, preview
    """Calculates blocking metrics for a given series and key length."""
    block_keys = base_cleaned_series.str.slice(0, k_len)
    block_counts = block_keys.value_counts()
    block_counts = block_counts[block_counts.index != ""]  # Exclude empty blocks

    if block_counts.empty:
        return None, 0.0, 0.0, []

    # Calculate Metrics
    distinct_blocks = len(block_counts)
    # avg_block_size = block_counts.mean() # Not directly in new output
    # median_block_size = block_counts.median() # Not directly in new output
    max_block_size = block_counts.max()
    max_block_pct = (
        (max_block_size / total_rows_in_sample) * 100 if total_rows_in_sample > 0 else 0
    )
    gini_coefficient = _calculate_gini(block_counts)
    entropy = _calculate_entropy(block_keys[block_keys != ""])
    estimated_pairs = int(np.sum(block_counts * (block_counts - 1) / 2))
    reduction_factor = (
        1.0 - (estimated_pairs / max_possible_pairs_in_sample)
        if max_possible_pairs_in_sample > 0
        else 0
    )

    # === TUNED Quality Score Heuristic (from existing logic) ===
    reduction_weight = 0.5
    max_block_penalty_weight = 0.35
    gini_penalty_weight = 0.15
    entropy_bonus_weight = 0.05

    quality_score_val = (
        (reduction_factor * reduction_weight)
        - ((max_block_pct / 100) ** 2 * max_block_penalty_weight)
        - (gini_coefficient * gini_penalty_weight)
        + (
            (entropy / max(1, math.log2(distinct_blocks)) if distinct_blocks > 1 else 0)
            * entropy_bonus_weight
        )
    )
    quality_score_val = round(max(quality_score_val, -1.0), 4)

    preview = [f"{k} (count: {c})" for k, c in block_counts.head(preview_count).items()]

    return (
        quality_score_val,
        round(entropy, 3),
        round(reduction_factor * 100, 2),  # This is reduction_pct
        preview,
    )


# ------------------------------------------------------------------
#  Fail-safe wrapper: suggest_blocking
# ------------------------------------------------------------------
# In engine.py, modify _estimate_reduction


def suggest_blocking(
    src_df: pd.DataFrame,  # This should be a sample DataFrame, e.g., src_df.head(some_sample_size)
    ref_df: Optional[pd.DataFrame] = None,  # Also a sample if provided
    *,
    ruleset_name: str = DEFAULT_RULESET_NAME,
    block_key_length_default: int = 3,
    preferred_src: Optional[List[str]] = None,
    preferred_ref: Optional[List[str]] = None,
    restrict_to_pairs: Optional[List[tuple[str, str]]] = None,
    # Params for relaxation logic
    max_reduction: float = 0.99,  # Max acceptable reduction ratio (e.g., 0.99 means at least 1% of pairs remain for comparison)
    min_entropy: float = 4.0,  # Minimum acceptable entropy for a blocking key
    min_meaningful_reduction: float = 0.01,  # Minimum reduction ratio to be considered effective blocking (e.g. 1%)
    **other_auto_decision_kwargs,  # Catch any other kwargs intended for auto_block_decision
) -> Optional[Dict[str, Any]]:
    """
    Run auto_block_decision() then relax key-length until the
    chosen blocking config is satisfactory based on entropy and reduction.
    Returns same dict shape as auto_block_decision() or None.
    """
    # --- Log Function Entry and Key Parameters ---
    log.info(
        f"suggest_blocking: Entered. ruleset='{ruleset_name}', "
        f"block_key_default={block_key_length_default}, "
        f"preferred_src_count={len(preferred_src) if preferred_src else 0}, "
        f"preferred_ref_count={len(preferred_ref) if preferred_ref else 0}, "
        f"restrict_to_pairs_count={len(restrict_to_pairs) if restrict_to_pairs else 0}, "
        f"max_reduction={max_reduction}, min_entropy={min_entropy}, "
        f"min_meaningful_reduction={min_meaningful_reduction}, "
        f"other_auto_decision_kwargs_keys={list(other_auto_decision_kwargs.keys())}"
    )
    # For very verbose debugging of preferred/restrict lists:
    # log.debug(f"suggest_blocking detail: preferred_src={preferred_src}, preferred_ref={preferred_ref}, restrict_to_pairs={restrict_to_pairs}")

    initial_decision = auto_block_decision(
        src_df=src_df,  # Pass the sample
        ref_df=ref_df,  # Pass the sample
        ruleset_name=ruleset_name,
        block_key_length_default=block_key_length_default,
        preferred_src=preferred_src,
        preferred_ref=preferred_ref,
        restrict_to_pairs=restrict_to_pairs,
        **other_auto_decision_kwargs,
    )
    # --- Log Initial Decision ---
    log.info(
        f"suggest_blocking: Initial decision from auto_block_decision: {initial_decision}"
    )

    if initial_decision is None:
        log.info(
            "suggest_blocking: auto_block_decision returned None. No initial configuration found."
        )
        return None

    best_overall_cfg = initial_decision.copy()  # Start with the initial decision
    chosen_col_name = best_overall_cfg.get("original_src_col")

    if not chosen_col_name:
        log.warning(
            "suggest_blocking: Initial decision from auto_block_decision missing 'original_src_col'. Cannot perform relaxation."
        )
        return initial_decision  # Return the potentially incomplete decision

    if chosen_col_name not in src_df.columns:
        log.warning(
            f"suggest_blocking: Chosen column '{chosen_col_name}' not in src_df sample. Cannot perform key length relaxation."
        )
        return best_overall_cfg  # Return initial decision

    original_col_series = src_df[chosen_col_name]  # Series from the input sample
    current_key_len = best_overall_cfg.get("block_key_length", block_key_length_default)
    current_metrics = best_overall_cfg.get(
        "metrics", {}
    )  # This is the rich metrics dict

    log.debug(
        f"suggest_blocking: Initial chosen config for '{chosen_col_name}': key_len={current_key_len}, metrics={current_metrics}"
    )

    MINIMUM_ACCEPTABLE_KEY_LEN = 2  # Don't go below this (or 3)

    # Extract current metric values safely
    current_entropy = current_metrics.get("entropy", 0.0)
    current_reduction_ratio = current_metrics.get(
        "reduction_ratio", 1.0
    )  # Default to 1.0 (max reduction) if not found
    N_in_sample = current_metrics.get("N_in_sample", 0)
    unique_blocks = current_metrics.get(
        "unique_blocks", N_in_sample if N_in_sample > 0 else 0
    )

    needs_relaxation = False
    # Condition 1: Entropy is too low (primary reason to try and adjust)
    if current_entropy < min_entropy:
        # --- Log Relaxation Trigger ---
        log.info(
            f"suggest_blocking: Relaxation triggered for '{chosen_col_name}'. Reason: Entropy ({current_entropy:.2f}) < min_entropy ({min_entropy:.2f})."
        )
        needs_relaxation = True
    # Condition 2: Blocking is too fine-grained (reduction is too low), even if entropy is technically acceptable.
    # This means the blocking isn't effectively reducing pairs.
    elif current_reduction_ratio < min_meaningful_reduction:
        # --- Log Relaxation Trigger ---
        log.info(
            f"suggest_blocking: Relaxation triggered for '{chosen_col_name}'. Reason: Reduction ratio ({current_reduction_ratio:.4f}) is below min_meaningful_reduction ({min_meaningful_reduction:.4f}). Attempting to find a key length with more effective blocking."
        )
        needs_relaxation = True

    if needs_relaxation and current_key_len > MINIMUM_ACCEPTABLE_KEY_LEN:
        log.info(
            f"suggest_blocking: Attempting to relax (shorten) key length for '{chosen_col_name}' from {current_key_len}."
        )

        # Store the best found relaxed configuration so far
        # Initialize with current best in case no shorter key is better or meets criteria
        candidate_relaxed_cfg = best_overall_cfg.copy()
        found_better_relaxation = False

        for test_key_len in range(
            current_key_len - 1, MINIMUM_ACCEPTABLE_KEY_LEN - 1, -1
        ):
            # --- Log Relaxation Loop Iteration ---
            log.debug(
                f"suggest_blocking relaxation loop: Testing key_len={test_key_len} for column '{chosen_col_name}' (ruleset: '{ruleset_name}')"
            )

            new_metrics = _estimate_reduction(
                original_col_series, test_key_len, ruleset_name
            )
            log.debug(
                f"suggest_blocking relaxation loop: Metrics for key_len={test_key_len}: {new_metrics}"
            )

            # Refined Criteria for accepting new_metrics:
            # 1. Entropy must meet the minimum.
            # 2. Reduction ratio must be within an acceptable range (not > max_reduction, which means not too un-aggressive).
            # 3. Reduction ratio must be above the min_meaningful_reduction threshold.
            # 4. Prefer solutions that improve (lower) the number of unique_blocks if current unique_blocks are very high.

            new_entropy = new_metrics.get("entropy", 0.0)
            new_reduction_ratio = new_metrics.get("reduction_ratio", 1.0)
            # new_unique_blocks = new_metrics.get('unique_blocks', N_in_sample) # Not directly used in this version of logic but good to have

            if (
                new_entropy >= min_entropy
                and new_reduction_ratio <= max_reduction
                and new_reduction_ratio > min_meaningful_reduction
            ):
                is_improvement = False
                if (
                    not found_better_relaxation
                ):  # First one that meets criteria is an improvement
                    is_improvement = True
                else:
                    # If already found a relaxed one, is this one even better?
                    # Current logic: if basics met, prefer higher entropy.
                    # Since loop goes from longer to shorter relaxed keys, this will pick the
                    # shortest key that meets criteria and has the best entropy among those.
                    pass

                if is_improvement or new_entropy > candidate_relaxed_cfg["metrics"].get(
                    "entropy", 0.0
                ):  # Prioritize higher entropy if basics are met
                    log.info(
                        f"suggest_blocking: Updating to better/acceptable relaxed metrics for '{chosen_col_name}' with key_len={test_key_len}. New metrics: {new_metrics}"
                    )
                    # --- Log Found Better Relaxation ---
                    candidate_relaxed_cfg["block_key_length"] = test_key_len
                    candidate_relaxed_cfg["metrics"] = new_metrics
                    candidate_relaxed_cfg["metrics"]["reduction_pct"] = round(
                        new_metrics["reduction_ratio"] * 100, 2
                    )
                    found_better_relaxation = True
                    # No break here, to find the shortest key_len that meets criteria and has best entropy.

        if found_better_relaxation:
            best_overall_cfg = candidate_relaxed_cfg
            log.info(
                f"suggest_blocking: Applied relaxation. New best config for '{chosen_col_name}': key_len={best_overall_cfg['block_key_length']}, metrics={best_overall_cfg['metrics']}"
            )
        else:
            # --- Ensure "No suitable relaxation" log is clear ---
            log.info(
                f"suggest_blocking: No suitable relaxation found for '{chosen_col_name}' that met all criteria (min_entropy={min_entropy}, max_reduction={max_reduction}, min_meaningful_reduction={min_meaningful_reduction}). Original/previous best config retained."
            )
            # best_overall_cfg remains the initial_decision if no relaxation was better or possible.

    # --- Log Final Decision ---
    log.info(
        f"suggest_blocking: Final decision after potential relaxation: {best_overall_cfg}"
    )
    return best_overall_cfg


def _estimate_reduction(
    col_values: pd.Series, key_len: int, ruleset_name: str
) -> Dict[str, Union[float, int]]:
    """
    Estimates blocking effectiveness and characteristics for a given column's values.
    1. Applies rules from ruleset_name to col_values.
    2. Generates block keys (standard cleaning + slicing).
    3. Calculates reduction ratio, unique blocks, and entropy.
    """
    global log, _RULESET_LOCK, AVAILABLE_RULESETS, apply_rules, sanitize_column_name, _calculate_entropy  # Ensure these are accessible
    log.debug(
        f"_estimate_reduction called for column: {col_values.name or 'Unnamed Series'}, key_len: {key_len}, ruleset: {ruleset_name}"
    )

    current_series = col_values
    original_col_name = col_values.name
    if original_col_name is None:
        original_col_name = "__temp_col_for_estimate_reduction__"

    if ruleset_name and ruleset_name.lower() != "none":
        with _RULESET_LOCK:
            ruleset_definition = AVAILABLE_RULESETS.get(ruleset_name.lower())
        if ruleset_definition:
            rules_to_apply = ruleset_definition.get("rules", [])
            if rules_to_apply:
                log.debug(
                    f"Applying {len(rules_to_apply)} rules from ruleset '{ruleset_name}' to column '{original_col_name}' for metric estimation."
                )
                temp_df_for_rules = pd.DataFrame({original_col_name: current_series})
                processed_df_by_rules = apply_rules(temp_df_for_rules, rules_to_apply)

                retrieved_processed_series = None
                sanitized_name = sanitize_column_name(original_col_name)
                if sanitized_name in processed_df_by_rules.columns:
                    retrieved_processed_series = processed_df_by_rules[sanitized_name]
                elif original_col_name in processed_df_by_rules.columns:
                    retrieved_processed_series = processed_df_by_rules[
                        original_col_name
                    ]
                elif len(processed_df_by_rules.columns) == 1:
                    retrieved_processed_series = processed_df_by_rules.iloc[:, 0]

                if retrieved_processed_series is not None:
                    current_series = retrieved_processed_series
                else:
                    log.warning(
                        f"For column '{original_col_name}' and ruleset '{ruleset_name}', "
                        f"could not confidently retrieve the transformed series after 'apply_rules' for metrics calculation. "
                        f"Using the series pre-custom_rules for metrics calculation. "
                        f"Processed df columns: {list(processed_df_by_rules.columns)}"
                    )

    block_key_series = (
        current_series.fillna("")
        .astype(str, copy=False)
        .str.lower()
        .str.replace(r"\W+", "", regex=True)
        .str.slice(0, key_len)
    )

    N = len(block_key_series)
    default_metrics = {
        "reduction_ratio": 0.0,
        "unique_blocks": N if N > 0 else 0,
        "entropy": 0.0,
        "N_in_sample": N,
    }
    if N < 2:
        return default_metrics

    max_possible_pairs = (N * (N - 1)) / 2.0
    if max_possible_pairs == 0:  # Should be caught by N < 2, but defensive
        return default_metrics

    block_counts = block_key_series.value_counts()
    block_counts_filtered = block_counts[
        block_counts.index != ""
    ]  # Exclude empty string blocks for unique_blocks count
    meaningful_block_keys = block_key_series[
        block_key_series != ""
    ]  # For entropy calculation

    blocked_pairs = np.sum(
        block_counts * (block_counts - 1) / 2.0
    )  # Use original block_counts for pairs
    reduction_ratio = (
        1.0 - (blocked_pairs / max_possible_pairs) if max_possible_pairs > 0 else 0.0
    )
    reduction_ratio = max(0.0, min(reduction_ratio, 1.0))

    unique_block_count = (
        len(block_counts_filtered) if not block_counts_filtered.empty else 0
    )
    entropy = (
        _calculate_entropy(meaningful_block_keys)
        if not meaningful_block_keys.empty
        else 0.0
    )

    metrics_to_return = {
        "reduction_ratio": reduction_ratio,
        "unique_blocks": unique_block_count,
        "entropy": entropy,
        "N_in_sample": N,
    }
    log.debug(
        f"_estimate_reduction for col '{original_col_name}', len {key_len}: RR={metrics_to_return['reduction_ratio']:.4f}, Unique={metrics_to_return['unique_blocks']}, Entropy={metrics_to_return['entropy']:.2f}, N={metrics_to_return['N_in_sample']}"
    )
    return metrics_to_return


# --- UnionFind Class ---
class UnionFind:
    """Iterative union-find for clustering."""

    def __init__(self):
        self.parent: Dict[Any, Any] = {}
        self.rank: Dict[Any, int] = {}

    def find(self, x):
        if x not in self.parent:
            self.parent[x] = x
            self.rank[x] = 0
            return x
        root = x
        path = []
        while self.parent[root] != root:
            path.append(root)
            root = self.parent[root]
        for node in path:
            self.parent[node] = root
        return root

    def union(self, a, b):
        ra, rb = self.find(a), self.find(b)
        if ra == rb:
            return False
        if self.rank[ra] < self.rank[rb]:
            self.parent[ra] = rb
        elif self.rank[ra] > self.rank[rb]:
            self.parent[rb] = ra
        else:
            self.parent[rb] = ra
            self.rank[ra] += 1
        return True


def should_use_multi_algo(config, mappings: List[Dict[str, Any]]) -> bool:
    """
    Determine if multi-algorithm processing is actually needed.
    Returns True when any mapping has an explicit preferred algorithm override.
    """
    if not mappings:
        return False

    if any(m.get("preferred_algo") for m in mappings):
        log.info("Per-field algorithm overrides detected; enabling multi-algo mode")
        return True

    log.info(
        "No per-field algorithm overrides supplied; single-algorithm mode sufficient"
    )
    return False


def prepare_block_data(
    src_block_df: pd.DataFrame,
    ref_block_df: pd.DataFrame,
    enhanced_mappings: List[EnhancedMapping],
) -> Dict[Tuple[str, str], Dict]:
    """
    Pre-extract and prepare all column data for a block.
    This avoids redundant string extraction and mask computation.
    """
    prepared_data = {}

    for mapping in enhanced_mappings:
        s_col = mapping.source
        r_col = mapping.ref

        if s_col not in src_block_df.columns or r_col not in ref_block_df.columns:
            continue

        # Extract once
        safe_s_col = f"_safe_{s_col}"
        safe_r_col = f"_safe_{r_col}"

        use_raw = getattr(mapping, "preprocessing", None) == "raw"

        if use_raw:
            raw_s_vals_list = [
                val if not pd.isna(val) else None for val in src_block_df[s_col]
            ]
            raw_r_vals_list = [
                val if not pd.isna(val) else None for val in ref_block_df[r_col]
            ]
        else:
            raw_s_series = (
                src_block_df[safe_s_col]
                if safe_s_col in src_block_df.columns
                else src_block_df[s_col].apply(_safe_str)
            )
            raw_r_series = (
                ref_block_df[safe_r_col]
                if safe_r_col in ref_block_df.columns
                else ref_block_df[r_col].apply(_safe_str)
            )
            raw_s_vals_list = raw_s_series.tolist()
            raw_r_vals_list = raw_r_series.tolist()

        transforms = getattr(mapping, "transforms", []) or []
        s_vals_list = apply_transforms_list(raw_s_vals_list, transforms)
        r_vals_list = apply_transforms_list(raw_r_vals_list, transforms)

        if _enhanced_mapping_is_domain(mapping):
            source_values = [v for v in s_vals_list if v]
            ref_values = [v for v in r_vals_list if v]
            source_set = set(source_values)
            ref_set = set(ref_values)
            overlap = len(source_set & ref_set)
            log.info(
                "[Worker] Domain equality overlap (post-transform): %d | S=%d R=%d | sample_s=%s | sample_r=%s",
                overlap,
                len(source_set),
                len(ref_set),
                source_values[:3],
                ref_values[:3],
            )

        s_vals = np.array(s_vals_list, dtype=object)
        r_vals = np.array(r_vals_list, dtype=object)

        # Pre-compute masks
        s_mask = np.array([bool(s) for s in s_vals], dtype=bool)
        r_mask = np.array([bool(r) for r in r_vals], dtype=bool)

        # Only store if we have valid data
        if s_mask.any() and r_mask.any():
            prepared_data[(s_col, r_col)] = {
                "s_vals": s_vals,
                "r_vals": r_vals,
                "s_mask": s_mask,
                "r_mask": r_mask,
                "validity": np.outer(s_mask, r_mask),
            }

    return prepared_data


# --------------------------------------------------------------------------- #
#  worker_match Ã¢â‚¬â€œ with per-process DB connection handling
# --------------------------------------------------------------------------- #
#   ALGORITHM_MAP, DEFAULT_ALGORITHM_KEY, RunStats, cached_score,
#   DEFAULT_BLOCK_COL, log  ...
# Ã¢â€â‚¬Ã¢â€â‚¬ src/fmatch/engine.py Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬
# Replace the existing worker_match function with this one:

# Replace the existing worker_match function with this one:


def worker_match(
    chunk: pd.DataFrame,
    ref: pd.DataFrame,
    config: MatchConfig,
    run_stats: Optional[RunStats] = None,
    progress_q: Optional[mp.Queue] = None,
) -> List[Dict[str, Any]]:
    """
    Worker function for fuzzy matching with multi-core support and soft-widen fallback.

    Features:
    - Multi-algorithm support
    - Soft-widen fallback for unmatched blocks
    - Parallel block processing
    - Enhanced progress reporting

    FIXES APPLIED:
    1. Account for unprocessed source rows (no matching blocks) in progress
    2. Process regular blocks in parallel and soft-widen blocks sequentially
    3. Correct block-level progress reporting with items_in_chunk parameter
    4. Ensure ALL matches from both block types are collected and returned
    """
    pid = os.getpid()
    chunk_timer = time.perf_counter()
    matches: List[Dict[str, Any]] = []
    processed_src_indices: set = set()
    rf_threads_per_worker = 1

    # Set RUN phase for progress tracking
    if run_stats:
        run_stats.set_phase(ProgressPhase.RUN.value)

    # --- Config extractions ---
    src_id_col = config.src_id_col
    ref_id_col = config.ref_id_col
    threshold = float(config.threshold) if config.threshold is not None else 0.0
    if threshold <= 1:
        log.debug(
            f"[Worker {pid}] Converting fractional threshold {threshold} to percentage scale"
        )
        threshold *= 100.0
        config.threshold = threshold
    algo_key_str = config.algorithm
    maps_for_algo = config.maps
    processing_mode = config.processing_mode
    apply_blocking_for_engine = config.apply_blocking
    enable_multi_algo = getattr(config, "enable_multi_algo", False)
    coverage_gate = getattr(config, "coverage_gate", 0.60)
    enable_soft_widen = getattr(config, "enable_soft_widen", True)
    enable_parallel = getattr(config, "enable_parallel", True)
    max_workers = getattr(config, "max_workers", CDIST_WORKERS)

    MAX_TOTAL_WEIGHT = (
        sum(float(m.get("weight", 1.0)) for m in maps_for_algo)
        if maps_for_algo
        else 0.01
    )

    log.debug(
        f"[Worker {pid}] worker_match starting. Chunk shape: {chunk.shape}, Ref shape: {ref.shape}, "
        f"Mode: {processing_mode}, Blocking: {apply_blocking_for_engine}, Algo: {algo_key_str}, "
        f"Threshold: {threshold}, Num Maps: {len(maps_for_algo) if maps_for_algo else 0}, "
        f"MAX_TOTAL_WEIGHT: {MAX_TOTAL_WEIGHT:.2f}, Coverage Gate: {coverage_gate}, "
        f"MultiAlgo: {enable_multi_algo}, SoftWiden: {enable_soft_widen}, "
        f"Parallel: {enable_parallel}, Workers: {max_workers}"
    )

    # Log effective parallelism summary at job start
    if enable_parallel:
        if RUNTIME_AWARE:
            rf_threads_per_worker = max(1, runtime_manager.policy.rf_threads_per_proc)
        else:
            rf_threads_per_worker = max(1, _rf_threads(max_workers))
        effective_parallelism = max_workers * rf_threads_per_worker
        log.info(
            "[policy] workers(procs)=%s rf_threads(per-proc)=%s -> effective=%s",
            max_workers,
            rf_threads_per_worker,
            effective_parallelism,
        )

    # Check time budget if runtime-aware
    if RUNTIME_AWARE:
        degradation = runtime_manager.check_time_budget()
        if degradation:
            log.warning(
                f"[runtime] Applying degradation due to time pressure: {degradation}"
            )
            config = apply_degradation(degradation, config)
            # Re-extract config values that may have changed
            enable_soft_widen = getattr(config, "enable_soft_widen", True)
            enable_multi_algo = getattr(config, "enable_multi_algo", False)
            max_workers = getattr(config, "max_workers", CDIST_WORKERS)

    # Initialize RunStats if not provided
    if run_stats is None and progress_q is not None:
        run_stats = RunStats(
            total_items_to_process=len(chunk), emit_interval_s=0.1, job_id=config.job_id
        )

    # Working copies - shallow copies reduce allocations while keeping isolation
    chunk_copy = chunk.copy(deep=False)
    ref_copy = ref.copy(deep=False)

    # Initialize algorithm
    algo_callable = ALGORITHM_MAP.get(algo_key_str)
    if not algo_callable:
        log.error(
            f"[Worker {pid}] Algorithm {algo_key_str} not in ALGORITHM_MAP. Defaulting."
        )
        algo_callable = ALGORITHM_MAP[DEFAULT_ALGORITHM_KEY]

    # --- Multi-Algorithm Setup ---
    scorer = None
    enhanced_mappings: List[EnhancedMapping] = []

    if maps_for_algo:
        for m in maps_for_algo:
            enhanced_mappings.append(
                EnhancedMapping(
                    source=m.get("source"),
                    ref=m.get("ref"),
                    weight=m.get("weight", 1.0),
                    preferred_algo=m.get("preferred_algo"),
                    algo_confidence=m.get("algo_confidence", 0.0),
                    transforms=m.get("transforms") or [],
                    preprocessing=m.get("preprocessing"),
                )
            )

        enable_multi_algo = True
        domain_exact_count = sum(
            1
            for m in enhanced_mappings
            if _enhanced_mapping_is_domain(m)
            and (m.preferred_algo or "").lower() == "exact"
        )
        effective_threshold = threshold
        if domain_exact_count and effective_threshold > 65:
            display_before = (
                effective_threshold / 100.0 if effective_threshold > 1 else effective_threshold
            )
            log.info(
                f"[Worker {pid}] Domain identity present; lowering match threshold from {display_before:.2f} to 0.65"
            )
            effective_threshold = 65.0

        # Use a local effective threshold; do not mutate the shared config object.
        threshold = effective_threshold

        scorer = MultiAlgorithmScorer(enhanced_mappings, coverage_gate=coverage_gate)
        display_threshold = threshold / 100.0 if threshold > 1 else threshold
        log.info(
            "[Worker %s] ENGINE: per-mapping scoring active | maps=%d | domain_exact=%d | effective_threshold=%.2f",
            pid,
            len(enhanced_mappings),
            domain_exact_count,
            display_threshold,
        )
    else:
        enable_multi_algo = False
        scorer = None

    config.enable_multi_algo = enable_multi_algo

    # --- Safe Column Creation ---
    source_cols_in_maps = set()
    ref_cols_in_maps = set()

    if maps_for_algo:
        for m_map in maps_for_algo:
            s_col = m_map.get("source")
            r_col = m_map.get("ref")
            if s_col:
                source_cols_in_maps.add(s_col)
            if r_col:
                ref_cols_in_maps.add(r_col)

    # Create safe columns for source
    for col_name in source_cols_in_maps:
        if col_name in chunk_copy.columns:
            if PHASE3_VECTORIZED:
                if _is_domain_column(col_name):
                    log.debug(
                        f"Applying domain normalization to source column: {col_name}"
                    )
                    chunk_copy.loc[:, f"_safe_{col_name}"] = chunk_copy[col_name].apply(
                        clean_domain_with_cache
                    )
                else:
                    chunk_copy.loc[:, f"_safe_{col_name}"] = vectorized_safe_str(
                        chunk_copy[col_name]
                    )
            else:
                # Legacy path
                chunk_copy.loc[:, f"_safe_{col_name}"] = chunk_copy[col_name].apply(
                    _safe_str
                )

    # Create safe columns for reference
    for col_name in ref_cols_in_maps:
        if col_name in ref_copy.columns:
            if PHASE3_VECTORIZED:
                if _is_domain_column(col_name):
                    log.debug(
                        f"Applying domain normalization to ref column: {col_name}"
                    )
                    ref_copy.loc[:, f"_safe_{col_name}"] = ref_copy[col_name].apply(
                        clean_domain_with_cache
                    )
                else:
                    ref_copy.loc[:, f"_safe_{col_name}"] = vectorized_safe_str(
                        ref_copy[col_name]
                    )
            else:
                # Legacy path
                ref_copy.loc[:, f"_safe_{col_name}"] = ref_copy[col_name].apply(
                    _safe_str
                )

    # ---- PATH 1: NO BLOCKING, SPEED MODE ----
    if not apply_blocking_for_engine and processing_mode == "speed":
        log.info(f"[Worker {pid}] VECTORIZED NON-BLOCKING (cdist) SPEED mode")
        # Previously this was a silent `pass`, which returned zero matches.
        # Use the same robust full-compare routine as the non-speed fallback.
        matches = process_block(
            src_block_df=chunk_copy,
            ref_block_df=ref_copy,
            block_key_val="__no_block_speed__",
            block_idx=0,
            src_id_col=src_id_col,
            ref_id_col=ref_id_col,
            threshold=threshold,
            algo_key_str=algo_key_str,
            maps_for_algo=maps_for_algo,
            enable_multi_algo=enable_multi_algo,
            scorer=scorer,
            enhanced_mappings=enhanced_mappings,
            algo_callable=algo_callable,
            coverage_gate=coverage_gate,
            MAX_TOTAL_WEIGHT=MAX_TOTAL_WEIGHT,
            pid=pid,
        )
        processed_src_indices = set(chunk_copy.index)
        config.runtime_metrics = {
            "workers_used": 1,
            "rf_threads_per_worker": rf_threads_per_worker,
            "memory_throttle": False,
        }
        log.info(
            f"[Worker {pid}] Runtime metrics: workers=1, rf_threads={rf_threads_per_worker}, memory_throttle=False"
        )

        if run_stats:
            total_ops = len(chunk_copy) * len(ref_copy)
            run_stats.update_blocks(
                blocks_processed=1,
                total_blocks=1,
                items_in_chunk=len(chunk_copy),
                comparisons_in_chunk=total_ops,
                progress_q=progress_q,
            )

    # ---- PATH 2: BLOCKING ENABLED ----
    elif apply_blocking_for_engine:
        log.info(
            f"[Worker {pid}] BLOCKING PATH. Small-block threshold: {SMALL_BLOCK_OPS_THRESHOLD} ops"
        )

        try:
            grouped_src = chunk_copy.groupby(_ACTIVE_BLOCK_KEY_, observed=True)
            grouped_ref = ref_copy.groupby(_ACTIVE_BLOCK_KEY_, observed=True)
        except KeyError as e:
            log.error(
                f"[Worker {pid}] Blocking key '{_ACTIVE_BLOCK_KEY_}' not found: {e}"
            )
            return matches

        # Collect all block keys and their sizes
        block_sizes_list: List[Dict[str, Any]] = []
        raw_common_keys = grouped_src.groups.keys() & grouped_ref.groups.keys()

        for key_val in raw_common_keys:
            try:
                src_len = len(grouped_src.get_group(key_val))
                ref_len = len(grouped_ref.get_group(key_val))
                if src_len * ref_len > 0:
                    block_sizes_list.append(
                        {
                            "key": key_val,
                            "metric": src_len * ref_len,
                            "src_len": src_len,
                            "ref_len": ref_len,
                            "threshold": threshold,
                            "is_soft_widen": False,  # Explicit flag for regular blocks
                        }
                    )
            except KeyError:
                continue

        # Soft-widen: Find unmatched source blocks
        soft_widen_candidates = []
        if enable_soft_widen:
            unmatched_src_keys = set(grouped_src.groups.keys()) - raw_common_keys

            for src_key in unmatched_src_keys:
                if len(str(src_key)) > 2:  # Only try if key is long enough
                    src_len = len(grouped_src.get_group(src_key))

                    # Try progressively shorter keys
                    for fallback_len in range(len(str(src_key)) - 1, 1, -1):
                        fallback_key = str(src_key)[:fallback_len]

                        if fallback_key in grouped_ref.groups:
                            ref_len = len(grouped_ref.get_group(fallback_key))
                            soft_widen_candidates.append(
                                {
                                    "src_key": src_key,
                                    "ref_key": fallback_key,
                                    "metric": src_len * ref_len,
                                    "src_len": src_len,
                                    "ref_len": ref_len,
                                    "threshold": threshold,
                                    "is_soft_widen": True,
                                }
                            )
                            log.debug(
                                f"[Worker {pid}] Soft-widen: '{src_key}' -> '{fallback_key}' "
                                f"({src_len}x{ref_len} ops)"
                            )
                            break

        # Combine all blocks
        all_blocks = block_sizes_list + soft_widen_candidates
        total_blocks = len(all_blocks)

        log.info(
            f"[Worker {pid}] Found {len(block_sizes_list)} exact blocks + "
            f"{len(soft_widen_candidates)} soft-widen blocks = {total_blocks} total"
        )

        # +++ START FIX: Account for unprocessed rows +++
        processed_src_indices = set()
        for block_info in all_blocks:
            # Use 'key' for regular blocks and 'src_key' for soft-widen blocks
            src_key_for_block = (
                block_info.get("key")
                if not block_info.get("is_soft_widen")
                else block_info.get("src_key")
            )
            if src_key_for_block:
                try:
                    # Add all indices from this source block to the set
                    processed_src_indices.update(
                        grouped_src.get_group(src_key_for_block).index
                    )
                except KeyError:
                    continue  # Should not happen if all_blocks is built correctly

        total_rows_in_chunk = len(chunk_copy)
        unprocessed_rows_count = total_rows_in_chunk - len(processed_src_indices)

        if unprocessed_rows_count > 0 and run_stats:
            log.info(
                f"[Worker {pid}] Accounting for {unprocessed_rows_count} source rows in blocks with no reference match."
            )
            # FIX: Properly update progress for unaccounted rows
            run_stats.update(0, unprocessed_rows_count, progress_q)
            # Force emit progress to UI
            if hasattr(run_stats, "emit_progress"):
                run_stats.emit_progress(force=True)
        # +++ END FIX +++

        # Initial progress emit
        if run_stats:
            run_stats.set_phase(ProgressPhase.RUN.value)

        # Better thresholds for parallel vs sequential
        total_src_rows = len(chunk_copy)
        total_ref_rows = len(ref_copy)
        estimated_total_ops = sum(b["metric"] for b in all_blocks)

        # Use runtime manager if available
        if RUNTIME_AWARE:
            use_parallel = enable_parallel and runtime_manager.should_use_parallel(
                total_blocks, estimated_total_ops
            )
            use_memory_mapped = (
                use_parallel
                and runtime_manager.should_use_memory_mapped(
                    total_src_rows, total_ref_rows, estimated_total_ops
                )
            )
        else:
            # Fallback to original logic
            use_parallel = (
                enable_parallel
                and total_blocks > 50  # Raised threshold
                and max_workers > 1
                and estimated_total_ops
                > 100000  # Only parallelize substantial workloads
            )

            # Also decide whether to use memory-mapped files
            # CRITICAL FIX: Respect processing mode and use higher thresholds
            use_memory_mapped = (
                use_parallel
                and config.processing_mode != "speed"  # Never use MM in speed mode
                and (
                    total_src_rows > ADAPTIVE_PARALLEL_THRESHOLD  # 500k threshold
                    or total_ref_rows > ADAPTIVE_PARALLEL_THRESHOLD
                    or estimated_total_ops > MEMORY_MAPPED_OPS_THRESHOLD
                )  # 50M ops
            )

        config.runtime_metrics = {
            "workers_used": max_workers if use_parallel else 1,
            "rf_threads_per_worker": rf_threads_per_worker,
            "memory_throttle": bool(use_memory_mapped),
        }
        log.info(
            f"[Worker {pid}] Runtime metrics: workers={config.runtime_metrics['workers_used']}, rf_threads={rf_threads_per_worker}, memory_throttle={config.runtime_metrics['memory_throttle']}"
        )

        if use_parallel:
            # PARALLEL PATH
            log.info(
                f"[Worker {pid}] Using PARALLEL processing for {total_blocks} blocks"
            )

            # NEW: Switch progress tracking to blocks basis for parallel processing
            if run_stats:
                run_stats.set_progress_basis("blocks", total_blocks)
                log.info(
                    "Progress tracking switched to blocks-based for parallel processing"
                )

            # CRITICAL FIX: Separate regular and soft-widen blocks
            # _match_in_parallel only handles regular blocks, not soft-widen blocks
            regular_blocks = [b for b in all_blocks if not b.get("is_soft_widen")]
            soft_widen_blocks = [b for b in all_blocks if b.get("is_soft_widen")]

            # Process regular blocks in parallel
            if regular_blocks:
                log.info(
                    f"[Worker {pid}] Processing {len(regular_blocks)} regular blocks in parallel"
                )

                # Check if we should use thread-based concurrency (serverless/gsheets)
                if (
                    RUNTIME_AWARE
                    and runtime_manager.policy.concurrency_model
                    == ConcurrencyModel.THREAD
                ):
                    log.info(
                        f"[Worker {pid}] Using thread-based concurrency for {runtime_manager.profile.value}"
                    )
                    from concurrent.futures import ThreadPoolExecutor

                    parallel_matches = []
                    # Thread mode does not run mp.Pool initializers, but _process_one_key
                    # depends on the global tables/groups populated by _pool_init.
                    _pool_init(chunk_copy, ref_copy, config)

                    with ThreadPoolExecutor(
                        max_workers=runtime_manager.policy.max_workers
                    ) as executor:
                        futures = [
                            executor.submit(_process_one_key, block)
                            for block in regular_blocks
                        ]
                        processed_blocks = 0
                        for future in futures:
                            result = future.result()
                            if isinstance(result, tuple):
                                matches, progress = result
                            else:
                                matches, progress = result, None

                            if matches:
                                parallel_matches.extend(matches)

                            if progress:
                                blocks_delta = int(progress.get("blocks", 1) or 1)
                                processed_blocks += blocks_delta
                                if run_stats:
                                    run_stats.update_blocks(
                                        blocks_processed=processed_blocks,
                                        total_blocks=len(regular_blocks),
                                        items_in_chunk=progress.get("items", 0),
                                        comparisons_in_chunk=progress.get(
                                            "comparisons", 0
                                        ),
                                        progress_q=progress_q,
                                    )
                                _emit_progress(
                                    progress.get("block_key"),
                                    progress.get("items", 0),
                                    progress.get("comparisons", 0),
                                    progress.get("matches", 0),
                                    blocks_delta=blocks_delta,
                                )
                            else:
                                processed_blocks += 1
                                if run_stats:
                                    run_stats.update_blocks(
                                        blocks_processed=processed_blocks,
                                        total_blocks=len(regular_blocks),
                                        items_in_chunk=0,
                                        comparisons_in_chunk=0,
                                        progress_q=progress_q,
                                    )
                                _emit_progress(None, 0, 0, 0, blocks_delta=1)

                        _emit_progress(None, 0, 0, 0, blocks_delta=0, flush=True)
                # Use memory-mapped version if calculated above
                elif use_memory_mapped:
                    log.info(
                        f"[Worker {pid}] Using memory-mapped parallel "
                        f"(src={total_src_rows:,} rows, ref={total_ref_rows:,} rows, "
                        f"ops={estimated_total_ops:,}, mode={config.processing_mode})"
                    )

                    # Safety check - should not happen in speed mode
                    if config.processing_mode == "speed":
                        log.warning(
                            "Memory-mapped files triggered in speed mode - this shouldn't happen!"
                        )

                    parallel_matches = _match_in_parallel_mm(
                        config,
                        chunk_copy,
                        ref_copy,
                        regular_blocks,  # Only pass regular blocks
                        run_stats,
                        progress_q,
                    )
                else:
                    parallel_matches = _match_in_parallel(
                        config,
                        chunk_copy,
                        ref_copy,
                        regular_blocks,  # Only pass regular blocks
                        run_stats,
                        progress_q,
                    )
                matches.extend(parallel_matches)
                log.info(
                    f"[Worker {pid}] Parallel processing returned {len(parallel_matches)} matches"
                )

            # Process soft-widen blocks sequentially
            if soft_widen_blocks:
                log.info(
                    f"[Worker {pid}] Processing {len(soft_widen_blocks)} soft-widen blocks sequentially"
                )
                for block_info in soft_widen_blocks:
                    src_key = block_info["src_key"]
                    ref_key = block_info["ref_key"]
                    try:
                        src_block_df = grouped_src.get_group(src_key)
                        ref_block_df = grouped_ref.get_group(ref_key)

                        matches_from_block = process_block(
                            src_block_df,
                            ref_block_df,
                            f"{src_key}->{ref_key}",
                            0,
                            src_id_col,
                            ref_id_col,
                            threshold,
                            algo_key_str,
                            maps_for_algo,
                            enable_multi_algo,
                            scorer,
                            enhanced_mappings,
                            algo_callable,
                            coverage_gate,
                            MAX_TOTAL_WEIGHT,
                            pid,
                        )
                        matches.extend(matches_from_block)

                        _emit_progress(
                            f"{src_key}->{ref_key}",
                            len(src_block_df),
                            len(ref_block_df),
                            len(matches_from_block or []),
                        )

                        if run_stats:
                            run_stats.update(
                                block_info["metric"], block_info["src_len"], progress_q
                            )
                    except KeyError:
                        log.debug(
                            f"[Worker {pid}] KeyError accessing soft-widen block {src_key}->{ref_key}"
                        )
                        continue

                log.info(
                    f"[Worker {pid}] Soft-widen processing added {len(matches) - len(parallel_matches) if regular_blocks else len(matches)} matches"
                )
                _emit_progress(None, 0, 0, 0, blocks_delta=0, flush=True)
        else:
            # SEQUENTIAL PATH
            log.info(
                f"[Worker {pid}] Using SEQUENTIAL processing for {total_blocks} blocks"
            )

            all_blocks_sorted = sorted(
                all_blocks, key=lambda x: x["metric"], reverse=True
            )
            processed_blocks = 0  # Track number of processed blocks

            # Process all blocks sequentially
            for block_idx, block_info in enumerate(all_blocks_sorted):
                # Get block DataFrames based on block type
                try:
                    if block_info.get("is_soft_widen"):
                        src_block_df = grouped_src.get_group(block_info["src_key"])
                        ref_block_df = grouped_ref.get_group(block_info["ref_key"])
                        block_key_val = (
                            f"{block_info['src_key']}->{block_info['ref_key']}"
                        )
                    else:
                        block_key_val = block_info.get("key", "")
                        if not block_key_val or (
                            isinstance(block_key_val, str) and not block_key_val.strip()
                        ):
                            log.debug(
                                f"Skipping empty block key. {block_info.get('src_len', 0)} "
                                f"source records will not be processed."
                            )
                            if run_stats:
                                run_stats.update(
                                    0, block_info.get("src_len", 0), progress_q
                                )
                            continue
                        src_block_df = grouped_src.get_group(block_info["key"])
                        ref_block_df = grouped_ref.get_group(block_info["key"])
                except KeyError:
                    log.debug(
                        f"[Worker {pid}] KeyError accessing block '{block_key_val}', skipping"
                    )
                    continue

                num_src_in_block = len(src_block_df)
                num_ref_in_block = len(ref_block_df)
                estimated_ops = num_src_in_block * num_ref_in_block

                # Periodic logging
                if block_idx % 100 == 0:
                    log.info(
                        f"[Worker {pid}] Processed {block_idx}/{total_blocks} blocks. "
                        f"Current: '{block_key_val}' ({num_src_in_block}x{num_ref_in_block})"
                    )

                if _should_skip_block(
                    block_key_val,
                    num_src_in_block,
                    num_ref_in_block,
                    max_size=MAX_BLOCK_SIZE,
                    max_ops=MAX_BLOCK_OPERATIONS,
                    is_dedup=False,
                ):
                    if run_stats:
                        run_stats.update(0, num_src_in_block, progress_q)
                    continue

                # Process normal-sized blocks
                matches_from_block = process_block(
                    src_block_df,
                    ref_block_df,
                    block_key_val,
                    block_idx,
                    src_id_col,
                    ref_id_col,
                    threshold,
                    algo_key_str,
                    maps_for_algo,
                    enable_multi_algo,
                    scorer,
                    enhanced_mappings,
                    algo_callable,
                    coverage_gate,
                    MAX_TOTAL_WEIGHT,
                    pid,
                )
                matches.extend(matches_from_block)

                # Accurate block-level progress
                if run_stats:
                    processed_blocks += 1
                    run_stats.update_blocks(
                        blocks_processed=processed_blocks,
                        total_blocks=len(all_blocks_sorted),
                        items_in_chunk=num_src_in_block,  # Pass items processed in this block
                        comparisons_in_chunk=estimated_ops,
                        progress_q=progress_q,
                    )

                # Emit heartbeat for long operations
                if estimated_ops > 100000 and run_stats:
                    run_stats.emit_heartbeat(f"Processing large block: {block_key_val}")

    else:
        # Fallback path: run the same block processor over the entire frames.
        log.warning(
            f"[Worker {pid}] Unhandled path. apply_blocking={apply_blocking_for_engine}, mode='{processing_mode}' - falling back to simple mode"
        )
        log.info(
            f"[Worker {pid}] Running full comparison without blocking (may be slow!)"
        )

        # Use the same robust routine we use for blocked processing.
        # Check for normalized copies first, then fall back to originals
        # Avoid using 'or' with DataFrames to prevent truth value errors

        # Helper to safely coalesce DataFrames without truthiness evaluation
        def _first_not_none(*vals):
            for v in vals:
                if v is not None:
                    return v
            return None

        # Get source DataFrame
        src_full = _first_not_none(
            locals().get("chunk_copy"), locals().get("src_df"), chunk
        )

        # Get reference DataFrame
        ref_full = _first_not_none(
            locals().get("ref_copy"),
            locals().get("ref_df"),
            locals().get("reference_df"),
            locals().get("ref"),
        )

        if ref_full is None:
            log.error(
                "[Worker %s] Fallback path could not locate a reference DataFrame in scope. Locals: %s",
                pid,
                [k for k in locals().keys()],
            )
            raise RuntimeError("worker_match fallback: reference DataFrame not found")

        matches = process_block(
            src_block_df=src_full,
            ref_block_df=ref_full,
            block_key_val="__no_block__",
            block_idx=0,
            src_id_col=src_id_col,
            ref_id_col=ref_id_col,
            threshold=threshold,
            algo_key_str=algo_key_str,
            maps_for_algo=maps_for_algo,
            enable_multi_algo=enable_multi_algo,
            scorer=scorer,
            enhanced_mappings=enhanced_mappings,
            algo_callable=algo_callable,
            coverage_gate=coverage_gate,
            MAX_TOTAL_WEIGHT=MAX_TOTAL_WEIGHT,
            pid=pid,
        )
        processed_src_indices = set(chunk_copy.index)
        config.runtime_metrics = {
            "workers_used": 1,
            "rf_threads_per_worker": rf_threads_per_worker,
            "memory_throttle": False,
        }
        log.info(
            f"[Worker {pid}] Runtime metrics: workers=1, rf_threads={rf_threads_per_worker}, memory_throttle=False"
        )

        if run_stats:
            total_ops = len(src_full) * len(ref_full)
            run_stats.update_blocks(
                blocks_processed=1,
                total_blocks=1,
                items_in_chunk=len(src_full),
                comparisons_in_chunk=total_ops,
                progress_q=progress_q,
            )

    residual_cfg = getattr(config, "residual_handling", None)
    if residual_cfg and residual_cfg.get("mode") == "cap_candidates_per_row":
        residual_limit = int(residual_cfg.get("limit", 0) or 0)
        if residual_limit > 0:
            residual_src_indices = (
                chunk_copy.index.difference(processed_src_indices)
                if "processed_src_indices" in locals()
                else chunk_copy.index
            )
            if len(residual_src_indices) > 0:
                log.info(
                    f"[Worker {pid}] Residual handling for {len(residual_src_indices)} rows (cap {residual_limit})."
                )
                residual_block_idx = locals().get("total_blocks", 0)
                for idx in residual_src_indices:
                    src_row_df = chunk_copy.loc[[idx]]
                    if config.mode.lower() == "match":
                        if ref_copy.empty:
                            continue
                        sample_size = min(residual_limit, len(ref_copy))
                        random_seed = abs(hash((idx, sample_size))) % (2**32)
                        sampled_ref = ref_copy.sample(
                            n=sample_size, random_state=random_seed
                        )
                    else:
                        available = chunk_copy.drop(index=idx)
                        if available.empty:
                            continue
                        sample_size = min(residual_limit, len(available))
                        random_seed = abs(hash((idx, sample_size))) % (2**32)
                        sampled_ref = available.sample(
                            n=sample_size, random_state=random_seed
                        )
                    block_key_val = f"_residual_{idx}"
                    matches_from_block = process_block(
                        src_row_df,
                        sampled_ref,
                        block_key_val,
                        residual_block_idx,
                        src_id_col,
                        ref_id_col,
                        threshold,
                        algo_key_str,
                        maps_for_algo,
                        enable_multi_algo,
                        scorer,
                        enhanced_mappings,
                        algo_callable,
                        coverage_gate,
                        MAX_TOTAL_WEIGHT,
                        pid,
                    )
                    residual_block_idx += 1
                    if matches_from_block:
                        matches.extend(matches_from_block)
                    if run_stats:
                        run_stats.update(sample_size, len(src_row_df), progress_q)

    # Clear cache if using multi-algorithm scorer
    if enable_multi_algo and scorer and hasattr(scorer, "clear_cache"):
        scorer.clear_cache()

    # Force final progress emission
    if run_stats:
        # Always emit final progress with latest stats
        run_stats.force_final_emit()
        # Log final summary with both counters
        log.info(
            f"[Worker {pid}] Final progress[basis={run_stats.progress_basis}]: "
            f"items {run_stats.items_processed_count:,}/{run_stats.total_items_to_process:,}, "
            f"blocks {run_stats.blocks_processed_count:,}/{run_stats.total_blocks:,}"
        )

    chunk_elapsed = time.perf_counter() - chunk_timer
    log.info(
        "[TIMING] worker=%s phase=worker_match rows=%d ref_rows=%d matches=%d elapsed=%.2fs blocking=%s parallel=%s mode=%s",
        pid,
        len(chunk),
        len(ref),
        len(matches),
        chunk_elapsed,
        apply_blocking_for_engine,
        enable_parallel,
        processing_mode,
    )

    log.info(f"[Worker {pid}] Completed. Returning {len(matches)} matches.")
    return matches


def process_block(
    src_block_df: pd.DataFrame,
    ref_block_df: Optional[pd.DataFrame],
    block_key_val: str,
    block_idx: int,
    src_id_col: Optional[str],
    ref_id_col: Optional[str],
    threshold: float,
    algo_key_str: str,
    maps_for_algo: List[Dict[str, Any]],
    enable_multi_algo: bool,
    scorer: Optional[MultiAlgorithmScorer],
    enhanced_mappings: List[EnhancedMapping],
    algo_callable: Callable,
    coverage_gate: float,
    MAX_TOTAL_WEIGHT: float,
    pid: int,
) -> List[Dict[str, Any]]:
    """Process a single block and return matches."""
    matches = []

    # Handle dedup mode where ref_block_df might be None
    if ref_block_df is None:
        # Dedup mode - comparing block to itself
        ref_block_df = src_block_df
        is_dedup = True
    else:
        is_dedup = False

    # Apply block guardrails
    guard = BlockGuardrails()

    # Determine secondary columns for sub-blocking
    secondary_src_col = None
    secondary_ref_col = None
    secondary_chars = 4

    def _is_domain_like(col_name: Optional[str]) -> bool:
        lowered = (col_name or "").lower()
        return any(t in lowered for t in ("domain", "website", "url", "email"))

    # Prefer explicit non-domain mapping pair (works even when column names differ).
    for m_map in maps_for_algo or []:
        src_col = m_map.get("source")
        ref_col = m_map.get("reference") or m_map.get("ref")
        if not src_col or not ref_col:
            continue
        if src_col not in src_block_df.columns or ref_col not in ref_block_df.columns:
            continue
        if _is_domain_like(src_col) or _is_domain_like(ref_col):
            continue
        secondary_src_col = src_col
        secondary_ref_col = ref_col
        break

    # Fallbacks for common same-name columns.
    if not secondary_src_col or not secondary_ref_col:
        for col in [
            "company_name",
            "company",
            "name",
            "organization",
            "account_name",
            "account",
        ]:
            if col in src_block_df.columns and col in ref_block_df.columns:
                secondary_src_col = col
                secondary_ref_col = col
                break

    # Last-resort independent detection for cross-schema columns.
    if (not secondary_src_col or not secondary_ref_col) and not is_dedup:
        name_tokens = ("company", "account", "org", "organization", "name")
        src_candidates = [
            c
            for c in src_block_df.columns
            if any(t in str(c).lower() for t in name_tokens)
            and not _is_domain_like(str(c))
        ]
        ref_candidates = [
            c
            for c in ref_block_df.columns
            if any(t in str(c).lower() for t in name_tokens)
            and not _is_domain_like(str(c))
        ]
        if src_candidates and ref_candidates:
            secondary_src_col = src_candidates[0]
            secondary_ref_col = ref_candidates[0]

    if is_dedup and secondary_src_col and not secondary_ref_col:
        secondary_ref_col = secondary_src_col

    # Check if blocks are too large
    num_src = len(src_block_df)
    num_ref = len(ref_block_df)
    estimated_ops = (
        num_src * num_ref if not is_dedup else (num_src * (num_src - 1)) // 2
    )

    # Keep skip behavior consistent across sequential and parallel execution paths.
    if _should_skip_block(
        block_key_val,
        num_src,
        num_ref,
        max_size=MAX_BLOCK_SIZE,
        max_ops=MAX_BLOCK_OPERATIONS,
        is_dedup=is_dedup,
    ):
        return []

    # Treat legacy MAX_* constants as sharding triggers (soft), not skip triggers.
    should_shard = (
        num_src > MAX_BLOCK_SIZE
        or num_ref > MAX_BLOCK_SIZE
        or estimated_ops > MAX_BLOCK_OPERATIONS
        or estimated_ops > guard.max_pairs
    )

    if not should_shard:
        # Small enough to process directly
        if estimated_ops < SMALL_BLOCK_OPS_THRESHOLD:
            # Small block - use tuple iteration
            if log.isEnabledFor(logging.DEBUG):
                log.debug(f"Block '{block_key_val}' SMALL. Tuple iteration.")

            matches = process_small_block_tuples(
                src_block_df,
                ref_block_df,
                block_key_val,
                block_idx,
                src_id_col,
                ref_id_col,
                threshold,
                algo_key_str,
                maps_for_algo,
                enable_multi_algo,
                scorer,
                algo_callable,
                coverage_gate,
                MAX_TOTAL_WEIGHT,
                pid,
                is_dedup=is_dedup,
            )
        else:
            # Large block - use cdist
            if log.isEnabledFor(logging.DEBUG):
                log.debug(
                    f"Block '{block_key_val}' ({estimated_ops} ops). VECTORIZED cdist."
                )

            matches = process_large_block_cdist(
                src_block_df,
                ref_block_df,
                block_key_val,
                block_idx,
                src_id_col,
                ref_id_col,
                threshold,
                algo_key_str,
                maps_for_algo,
                enable_multi_algo,
                scorer,
                enhanced_mappings,
                algo_callable,
                coverage_gate,
                MAX_TOTAL_WEIGHT,
                pid,
                is_dedup=is_dedup,
            )
    else:
        # Too large - need to shard by secondary key
        if not secondary_src_col or not secondary_ref_col:
            log.warning(
                f"Block '{block_key_val}' too large ({estimated_ops} ops) but no secondary column available. Sampling."
            )
            # Sample both sides
            max_sample = int(math.sqrt(guard.max_pairs))
            src_sample = src_block_df.sample(
                n=min(max_sample, len(src_block_df)), random_state=42
            )
            ref_sample = ref_block_df.sample(
                n=min(max_sample, len(ref_block_df)), random_state=42
            )

            matches = process_large_block_cdist(
                src_sample,
                ref_sample,
                block_key_val,
                block_idx,
                src_id_col,
                ref_id_col,
                threshold,
                algo_key_str,
                maps_for_algo,
                enable_multi_algo,
                scorer,
                enhanced_mappings,
                algo_callable,
                coverage_gate,
                MAX_TOTAL_WEIGHT,
                pid,
                is_dedup=is_dedup,
            )
        else:
            # Group by secondary key FIRST, then only compare matching groups.
            # Use adaptive key length because strict prefixes can isolate all rows
            # into non-overlapping shards (especially synthetic test data).
            shard_len_candidates: List[int] = []
            for length in [secondary_chars, 3, 2, 1]:
                if length and length > 0 and length not in shard_len_candidates:
                    shard_len_candidates.append(length)

            # Normalize once to keep shard keys stable across punctuation/casing variants.
            norm_s = (
                src_block_df[secondary_src_col]
                .fillna("")
                .astype(str)
                .str.lower()
                .str.replace(r"[^a-z0-9]+", "", regex=True)
            )
            norm_r = (
                ref_block_df[secondary_ref_col]
                .fillna("")
                .astype(str)
                .str.lower()
                .str.replace(r"[^a-z0-9]+", "", regex=True)
            )

            best_plan = None
            for shard_len in shard_len_candidates:
                key_s = norm_s.str[:shard_len]
                key_r = norm_r.str[:shard_len]
                src_groups = {
                    k: idx
                    for k, idx in src_block_df.groupby(key_s, sort=False).groups.items()
                    if str(k).strip()
                }
                ref_groups = {
                    k: idx
                    for k, idx in ref_block_df.groupby(key_r, sort=False).groups.items()
                    if str(k).strip()
                }
                common_keys = [k for k in src_groups if k in ref_groups]
                if not common_keys:
                    continue

                covered_src = sum(len(src_groups[k]) for k in common_keys)
                covered_ref = sum(len(ref_groups[k]) for k in common_keys)
                covered_src_ratio = covered_src / max(1, num_src)
                covered_ref_ratio = covered_ref / max(1, num_ref)
                max_shard_ops = max(
                    len(src_groups[k]) * len(ref_groups[k]) for k in common_keys
                )
                total_shard_ops = sum(
                    len(src_groups[k]) * len(ref_groups[k]) for k in common_keys
                )

                plan_score = (
                    covered_src_ratio + covered_ref_ratio,
                    -max_shard_ops,
                    -total_shard_ops,
                )
                if best_plan is None or plan_score > best_plan["score"]:
                    best_plan = {
                        "score": plan_score,
                        "len": shard_len,
                        "src_groups": src_groups,
                        "ref_groups": ref_groups,
                        "common_keys": common_keys,
                        "covered_src_ratio": covered_src_ratio,
                        "covered_ref_ratio": covered_ref_ratio,
                    }

            # If shard overlap is too weak, fall back to direct vectorized scoring
            # (or sampled vectorized scoring for very large pair counts) so we
            # don't silently return zero due to key isolation.
            if (
                not best_plan
                or best_plan["covered_src_ratio"] < 0.25
                or best_plan["covered_ref_ratio"] < 0.25
            ):
                if estimated_ops <= guard.max_pairs:
                    log.warning(
                        f"Block '{block_key_val}' too large ({estimated_ops} ops) and "
                        f"secondary-key overlap was weak/empty. Falling back to direct cdist."
                    )
                    matches = process_large_block_cdist(
                        src_block_df,
                        ref_block_df,
                        f"{block_key_val}_fallback",
                        block_idx,
                        src_id_col,
                        ref_id_col,
                        threshold,
                        algo_key_str,
                        maps_for_algo,
                        enable_multi_algo,
                        scorer,
                        enhanced_mappings,
                        algo_callable,
                        coverage_gate,
                        MAX_TOTAL_WEIGHT,
                        pid,
                        is_dedup=is_dedup,
                    )
                else:
                    max_sample = int(math.sqrt(guard.max_pairs))
                    src_sample = src_block_df.sample(
                        n=min(max_sample, len(src_block_df)), random_state=42
                    )
                    ref_sample = ref_block_df.sample(
                        n=min(max_sample, len(ref_block_df)), random_state=42
                    )
                    log.warning(
                        f"Block '{block_key_val}' too large ({estimated_ops} ops), "
                        "secondary-key overlap was weak/empty, and direct cdist exceeds "
                        "guardrails. Falling back to sampled cdist."
                    )
                    matches = process_large_block_cdist(
                        src_sample,
                        ref_sample,
                        f"{block_key_val}_sampled",
                        block_idx,
                        src_id_col,
                        ref_id_col,
                        threshold,
                        algo_key_str,
                        maps_for_algo,
                        enable_multi_algo,
                        scorer,
                        enhanced_mappings,
                        algo_callable,
                        coverage_gate,
                        MAX_TOTAL_WEIGHT,
                        pid,
                        is_dedup=is_dedup,
                    )
            else:
                shard_len = best_plan["len"]
                src_groups = best_plan["src_groups"]
                ref_groups = best_plan["ref_groups"]
                common_keys = best_plan["common_keys"]
                log.info(
                    f"Block '{block_key_val}' too large ({estimated_ops} ops). "
                    f"Sharding by '{secondary_src_col}' -> '{secondary_ref_col}' "
                    f"(prefix_len={shard_len}, common_keys={len(common_keys)}, "
                    f"coverage_src={best_plan['covered_src_ratio']:.1%}, "
                    f"coverage_ref={best_plan['covered_ref_ratio']:.1%})."
                )

                # Only compare matching secondary keys
                for key in common_keys:
                    src_shard_df = src_block_df.loc[src_groups[key]]
                    ref_shard_df = ref_block_df.loc[ref_groups[key]]

                    # Enforce guardrails per shard
                    if (
                        len(src_shard_df) > guard.max_block_rows
                        or len(ref_shard_df) > guard.max_block_rows
                    ):
                        # Sample to max size
                        if len(src_shard_df) > guard.max_block_rows:
                            src_shard_df = src_shard_df.sample(
                                n=guard.max_block_rows, random_state=42
                            )
                        if len(ref_shard_df) > guard.max_block_rows:
                            ref_shard_df = ref_shard_df.sample(
                                n=guard.max_block_rows, random_state=42
                            )

                    num_src_in_shard = len(src_shard_df)
                    num_ref_in_shard = len(ref_shard_df)
                    estimated_ops_in_shard = num_src_in_shard * num_ref_in_shard

                    if estimated_ops_in_shard < SMALL_BLOCK_OPS_THRESHOLD:
                        # Small shard - use tuple iteration
                        shard_matches = process_small_block_tuples(
                            src_shard_df,
                            ref_shard_df,
                            f"{block_key_val}_{key}",
                            block_idx,
                            src_id_col,
                            ref_id_col,
                            threshold,
                            algo_key_str,
                            maps_for_algo,
                            enable_multi_algo,
                            scorer,
                            algo_callable,
                            coverage_gate,
                            MAX_TOTAL_WEIGHT,
                            pid,
                            is_dedup=is_dedup,
                        )
                    else:
                        # Large shard - use cdist
                        shard_matches = process_large_block_cdist(
                            src_shard_df,
                            ref_shard_df,
                            f"{block_key_val}_{key}",
                            block_idx,
                            src_id_col,
                            ref_id_col,
                            threshold,
                            algo_key_str,
                            maps_for_algo,
                            enable_multi_algo,
                            scorer,
                            enhanced_mappings,
                            algo_callable,
                            coverage_gate,
                            MAX_TOTAL_WEIGHT,
                            pid,
                            is_dedup=is_dedup,
                        )

                    # Collect matches from this shard
                    if shard_matches:
                        matches.extend(shard_matches)

    return matches


def process_small_block_tuples(
    src_block_df: pd.DataFrame,
    ref_block_df: pd.DataFrame,
    block_key_val: str,
    block_idx: int,
    src_id_col: Optional[str],
    ref_id_col: Optional[str],
    threshold: float,
    algo_key_str: str,
    maps_for_algo: List[Dict[str, Any]],
    enable_multi_algo: bool,
    scorer: Optional[MultiAlgorithmScorer],
    algo_callable: Callable,
    coverage_gate: float,
    MAX_TOTAL_WEIGHT: float,
    pid: int,
    is_dedup: bool = False,
) -> List[Dict[str, Any]]:
    """Process a small block using tuple iteration."""
    matches = []
    items_processed_tb = 0
    comparisons_tb = 0

    src_col_indices = {col: i_col for i_col, col in enumerate(src_block_df.columns)}
    ref_col_indices = {col: i_col for i_col, col in enumerate(ref_block_df.columns)}

    src_tuples = list(src_block_df.itertuples(index=False, name=None))
    ref_tuples = list(ref_block_df.itertuples(index=False, name=None))

    # Handle dedup mode - only compare upper triangle
    if is_dedup:
        n = len(src_tuples)
        for i in range(n):
            s_row_tuple = src_tuples[i]
            items_processed_tb += 1
            s_id_val_tb = (
                s_row_tuple[src_col_indices[src_id_col]]
                if src_id_col and src_id_col in src_col_indices
                else f"s_sb_{block_idx}_{i}"
            )

            # Only compare with subsequent rows to avoid duplicates (upper triangle)
            for j in range(i + 1, n):
                r_row_tuple = src_tuples[j]  # Note: using src_tuples since it's dedup
                comparisons_tb += 1
                # Use ref_col_indices first, fall back to src_col_indices
                r_idx = ref_col_indices.get(ref_id_col) if ref_id_col else None
                if r_idx is None and ref_id_col:
                    r_idx = src_col_indices.get(ref_id_col)
                r_id_val_tb = (
                    r_row_tuple[r_idx] if r_idx is not None else f"r_sb_{block_idx}_{j}"
                )

                if enable_multi_algo and scorer:
                    # Convert tuples to dicts for scorer
                    s_dict = {
                        col: s_row_tuple[idx] for col, idx in src_col_indices.items()
                    }
                    r_dict = {
                        col: r_row_tuple[idx] for col, idx in ref_col_indices.items()
                    }

                    score_result = scorer.score_pair(s_dict, r_dict)

                    if score_result["confidence_score"] >= threshold:
                        record_tb = {
                            (src_id_col or "s_id"): s_id_val_tb,
                            (ref_id_col or "r_id"): r_id_val_tb,
                            "confidence_score": round(
                                score_result["confidence_score"], 2
                            ),
                            "field_score_details": score_result.get(
                                "field_score_details", []
                            ),
                        }

                        # Add all original columns with prefixes
                        _add_prefixed_row(
                            record_tb, "s", s_row_tuple, src_block_df.columns
                        )
                        _add_prefixed_row(
                            record_tb, "r", r_row_tuple, ref_block_df.columns
                        )

                        matches.append(record_tb)
                else:
                    # Original single-algorithm logic
                    current_pair_field_details = []
                    total_score_tb = 0.0
                    total_weight_tb = 0.0
                    valid_comp_tb = False

                    if maps_for_algo:
                        for m_map_tb in maps_for_algo:
                            s_col_name_tb = m_map_tb.get("source")
                            r_col_name_tb = m_map_tb.get("ref")
                            weight_tb = m_map_tb.get("weight", 0.0)
                            s_col_idx_tb = src_col_indices.get(s_col_name_tb)
                            r_col_idx_tb = ref_col_indices.get(r_col_name_tb)

                            if s_col_idx_tb is not None and r_col_idx_tb is not None:
                                s_val_iter = s_row_tuple[s_col_idx_tb]
                                r_val_iter = r_row_tuple[r_col_idx_tb]
                                valid_comp_tb = True

                                transforms_tb = m_map_tb.get("transforms") or []
                                if transforms_tb:
                                    s_val_iter = apply_transforms(
                                        s_val_iter, transforms_tb
                                    )
                                    r_val_iter = apply_transforms(
                                        r_val_iter, transforms_tb
                                    )
                                else:
                                    s_val_iter = apply_transforms(
                                        s_val_iter, ("strip",)
                                    )
                                    r_val_iter = apply_transforms(
                                        r_val_iter, ("strip",)
                                    )

                                score_iter = robust_score_calculation(
                                    s1=s_val_iter,
                                    s2=r_val_iter,
                                    algo=algo_key_str,
                                    fallback_algo="Ratio",
                                    processing_mode="scale",
                                    ruleset_name="default",
                                    db_path=None,
                                )

                                total_score_tb += score_iter * weight_tb
                                total_weight_tb += weight_tb
                                current_pair_field_details.append(
                                    {
                                        "source_field": s_col_name_tb,
                                        "ref_field": r_col_name_tb,
                                        "score": float(score_iter),
                                        "weight": weight_tb,
                                        "algo_used": algo_key_str,
                                    }
                                )

                    if valid_comp_tb and total_weight_tb > 0:
                        avg_score_tb = total_score_tb / total_weight_tb
                        if maps_for_algo and MAX_TOTAL_WEIGHT > 0:
                            coverage_tb = total_weight_tb / MAX_TOTAL_WEIGHT
                            if coverage_tb < coverage_gate:
                                log.debug(
                                    f"Coverage penalty applied: coverage={coverage_tb:.2f} < gate={coverage_gate}"
                                )
                                avg_score_tb *= coverage_tb

                        if avg_score_tb >= threshold:
                            record_tb = {
                                (src_id_col or "s_id"): s_id_val_tb,
                                (ref_id_col or "r_id"): r_id_val_tb,
                                "confidence_score": round(avg_score_tb, 2),
                            }

                            # Add all original columns with prefixes
                            _add_prefixed_row(
                                record_tb, "s", s_row_tuple, src_block_df.columns
                            )
                            _add_prefixed_row(
                                record_tb, "r", r_row_tuple, ref_block_df.columns
                            )

                            record_tb["field_score_details"] = (
                                current_pair_field_details
                            )
                            matches.append(record_tb)
    else:
        # Non-dedup mode: compare all source vs all reference
        for s_row_tuple in src_tuples:
            items_processed_tb += 1
            s_id_val_tb = (
                s_row_tuple[src_col_indices[src_id_col]]
                if src_id_col and src_id_col in src_col_indices
                else f"s_sb_{block_idx}_{items_processed_tb}"
            )

            for r_row_tuple in ref_tuples:
                comparisons_tb += 1
                r_id_val_tb = (
                    r_row_tuple[ref_col_indices[ref_id_col]]
                    if ref_id_col and ref_id_col in ref_col_indices
                    else f"r_sb_{block_idx}_{comparisons_tb}"
                )

                if enable_multi_algo and scorer:
                    # Convert tuples to dicts for scorer
                    s_dict = {
                        col: s_row_tuple[idx] for col, idx in src_col_indices.items()
                    }
                    r_dict = {
                        col: r_row_tuple[idx] for col, idx in ref_col_indices.items()
                    }

                    score_result = scorer.score_pair(s_dict, r_dict)

                    if score_result["confidence_score"] >= threshold:
                        record_tb = {
                            (src_id_col or "s_id"): s_id_val_tb,
                            (ref_id_col or "r_id"): r_id_val_tb,
                            "confidence_score": round(
                                score_result["confidence_score"], 2
                            ),
                            "field_score_details": score_result.get(
                                "field_score_details", []
                            ),
                        }

                        # Add all original columns with prefixes
                        _add_prefixed_row(
                            record_tb, "s", s_row_tuple, src_block_df.columns
                        )
                        _add_prefixed_row(
                            record_tb, "r", r_row_tuple, ref_block_df.columns
                        )

                        matches.append(record_tb)
                else:
                    # Original single-algorithm logic
                    current_pair_field_details = []
                    total_score_tb = 0.0
                    total_weight_tb = 0.0
                    valid_comp_tb = False

                    if maps_for_algo:
                        for m_map_tb in maps_for_algo:
                            s_col_name_tb = m_map_tb.get("source")
                            r_col_name_tb = m_map_tb.get("ref")
                            weight_tb = m_map_tb.get("weight", 0.0)
                            s_col_idx_tb = src_col_indices.get(s_col_name_tb)
                            r_col_idx_tb = ref_col_indices.get(r_col_name_tb)

                            if s_col_idx_tb is not None and r_col_idx_tb is not None:
                                s_val_iter = s_row_tuple[s_col_idx_tb]
                                r_val_iter = r_row_tuple[r_col_idx_tb]
                                valid_comp_tb = True

                                transforms_tb = m_map_tb.get("transforms") or []
                                if transforms_tb:
                                    s_val_iter = apply_transforms(
                                        s_val_iter, transforms_tb
                                    )
                                    r_val_iter = apply_transforms(
                                        r_val_iter, transforms_tb
                                    )
                                else:
                                    s_val_iter = apply_transforms(
                                        s_val_iter, ("strip",)
                                    )
                                    r_val_iter = apply_transforms(
                                        r_val_iter, ("strip",)
                                    )

                                score_iter = robust_score_calculation(
                                    s1=s_val_iter,
                                    s2=r_val_iter,
                                    algo=algo_key_str,
                                    fallback_algo="Ratio",
                                    processing_mode="scale",
                                    ruleset_name="default",
                                    db_path=None,
                                )

                                total_score_tb += score_iter * weight_tb
                                total_weight_tb += weight_tb
                                current_pair_field_details.append(
                                    {
                                        "source_field": s_col_name_tb,
                                        "ref_field": r_col_name_tb,
                                        "score": float(score_iter),
                                        "weight": weight_tb,
                                        "algo_used": algo_key_str,
                                    }
                                )

                    if valid_comp_tb and total_weight_tb > 0:
                        avg_score_tb = total_score_tb / total_weight_tb
                        if maps_for_algo and MAX_TOTAL_WEIGHT > 0:
                            coverage_tb = total_weight_tb / MAX_TOTAL_WEIGHT
                            if coverage_tb < coverage_gate:
                                log.debug(
                                    f"Coverage penalty applied: coverage={coverage_tb:.2f} < gate={coverage_gate}"
                                )
                                avg_score_tb *= coverage_tb

                        if avg_score_tb >= threshold:
                            record_tb = {
                                (src_id_col or "s_id"): s_id_val_tb,
                                (ref_id_col or "r_id"): r_id_val_tb,
                                "confidence_score": round(avg_score_tb, 2),
                            }

                            # Add all original columns with prefixes
                            _add_prefixed_row(
                                record_tb, "s", s_row_tuple, src_block_df.columns
                            )
                            _add_prefixed_row(
                                record_tb, "r", r_row_tuple, ref_block_df.columns
                            )

                            record_tb["field_score_details"] = (
                                current_pair_field_details
                            )
                            matches.append(record_tb)

    return matches


def process_large_block_cdist(
    src_block_df: pd.DataFrame,
    ref_block_df: pd.DataFrame,
    block_key_val: str,
    block_idx: int,
    src_id_col: Optional[str],
    ref_id_col: Optional[str],
    threshold: float,
    algo_key_str: str,
    maps_for_algo: List[Dict[str, Any]],
    enable_multi_algo: bool,
    scorer: Optional[MultiAlgorithmScorer],
    enhanced_mappings: List[EnhancedMapping],
    algo_callable: Callable,
    coverage_gate: float,
    MAX_TOTAL_WEIGHT: float,
    pid: int,
    is_dedup: bool = False,
) -> List[Dict[str, Any]]:
    """Process a large block using cdist vectorization."""
    matches = []
    S_block = len(src_block_df)
    R_block = len(ref_block_df)

    # Use provided enhanced mappings; if absent, derive them from maps_for_algo.
    effective_mappings = enhanced_mappings
    if not effective_mappings and maps_for_algo:
        effective_mappings = []
        for m in maps_for_algo:
            s_col = m.get("source")
            r_col = m.get("ref") or m.get("source")
            if not s_col or not r_col:
                continue
            effective_mappings.append(
                EnhancedMapping(
                    source=s_col,
                    ref=r_col,
                    weight=m.get("weight", 1.0),
                    preferred_algo=m.get("preferred_algo"),
                    algo_confidence=m.get("algo_confidence", 0.0),
                    transforms=m.get("transforms") or [],
                    preprocessing=m.get("preprocessing"),
                )
            )

    if effective_mappings:
        log.debug(
            f"[Worker {pid}] Using VECTORIZED multi-algorithm approach for large block"
        )

        # Pre-compute all column data once
        prepared_data = prepare_block_data(
            src_block_df, ref_block_df, effective_mappings
        )

        if not prepared_data:
            log.warning(
                f"[Worker {pid}] No valid column pairs for block '{block_key_val}'"
            )
            return matches

        # Collect all unique algorithms needed
        algorithms_needed = set()
        algorithm_weights = defaultdict(float)

        for mapping in effective_mappings:
            algo = mapping.preferred_algo or algo_key_str
            algorithms_needed.add(algo)
            algorithm_weights[algo] += mapping.weight

        # Initialize result matrices
        final_scores = np.zeros((S_block, R_block), dtype=np.float32)
        total_weights_used = np.zeros((S_block, R_block), dtype=np.float32)

        # Process each algorithm separately with vectorization
        for algo in algorithms_needed:
            algo_callable = ALGORITHM_MAP.get(algo)
            if not algo_callable:
                log.warning(
                    f"[Worker {pid}] Algorithm '{algo}' not found in ALGORITHM_MAP"
                )
                continue

            # Calculate scores for all field pairs using this algorithm
            algo_field_scores = np.zeros((S_block, R_block), dtype=np.float32)
            algo_field_weights = np.zeros((S_block, R_block), dtype=np.float32)

            for mapping in effective_mappings:
                # Skip if this mapping doesn't use this algorithm
                if mapping.preferred_algo and mapping.preferred_algo != algo:
                    continue
                if not mapping.preferred_algo and algo != algo_key_str:
                    continue

                s_col = mapping.source
                r_col = mapping.ref
                weight = mapping.weight

                # Use pre-computed data
                col_data = prepared_data.get((s_col, r_col))
                if not col_data:
                    continue

                # Use cdist with pre-extracted values
                try:
                    # Special handling for Exact algorithm
                    if algo == "Exact":
                        scores = rf_process.cdist(
                            col_data["s_vals"],
                            col_data["r_vals"],
                            scorer=algo_callable,
                            workers=1,
                        )
                    else:
                        scores = rf_process.cdist(
                            col_data["s_vals"],
                            col_data["r_vals"],
                            scorer=algo_callable,
                            score_cutoff=0,
                            workers=1,
                        )

                    # Apply pre-computed validity mask
                    valid_scores = scores * col_data["validity"]

                    # Accumulate weighted scores
                    algo_field_scores += valid_scores * weight
                    algo_field_weights += col_data["validity"] * weight

                except Exception as e:
                    log.error(
                        f"[Worker {pid}] cdist failed for {algo} on {s_col}->{r_col}: {e}"
                    )
                    continue

            # Add this algorithm's contribution to final scores
            final_scores += algo_field_scores
            total_weights_used += algo_field_weights

        # Calculate average scores
        with np.errstate(divide="ignore", invalid="ignore"):
            avg_scores = np.divide(
                final_scores,
                total_weights_used,
                out=np.zeros_like(final_scores),
                where=(total_weights_used > 0),
            )

        # Apply coverage penalty
        if MAX_TOTAL_WEIGHT > 0:
            coverage = total_weights_used / MAX_TOTAL_WEIGHT
            penalty_factor = np.where(coverage < coverage_gate, coverage, 1.0)
            final_result_scores = avg_scores * penalty_factor
        else:
            final_result_scores = avg_scores

        # Extract matches above threshold
        match_indices = np.where(final_result_scores >= threshold)

        # If dedup mode, filter to upper triangle only (src_idx < ref_idx)
        if is_dedup:
            log.debug(f"[Worker {pid}] Applying dedup upper-triangle filtering")
            s_indices, r_indices = match_indices
            mask = s_indices < r_indices
            s_indices, r_indices = s_indices[mask], r_indices[mask]
        else:
            s_indices, r_indices = match_indices

        for s_idx, r_idx in zip(s_indices, r_indices):
            s_row = src_block_df.iloc[s_idx]
            r_row = ref_block_df.iloc[r_idx]

            s_id_val = s_row[src_id_col] if src_id_col else f"s_idx_{s_idx}"
            r_id_val = r_row[ref_id_col] if ref_id_col else f"r_idx_{r_idx}"

            record = {
                (src_id_col or "s_id"): s_id_val,
                (ref_id_col or "r_id"): r_id_val,
                "confidence_score": round(float(final_result_scores[s_idx, r_idx]), 2),
                "field_score_details": [],  # Could be populated if needed
            }

            # Add all original columns with prefixes
            for col in src_block_df.columns:
                if not col.startswith("_temp_"):
                    record[f"s_{sanitize_column_name(col)}"] = s_row[col]
            for col in ref_block_df.columns:
                if not col.startswith("_temp_"):
                    record[f"r_{sanitize_column_name(col)}"] = r_row[col]

            matches.append(record)
    else:
        # Original single-algorithm cdist implementation
        # This would contain the existing cdist logic for single algorithm
        # ... [Implementation details omitted for brevity] ...
        pass

    return matches


# --------------------------------------------------------------------------- #
#  worker_duplicate Ã¢â‚¬â€œ with per-process DB connection handling
# --------------------------------------------------------------------------- #
# Replace the existing worker_duplicate function with this one:

# Replace the existing worker_duplicate function with this one:

# In engine.py


def worker_duplicate(
    source_df_processed: pd.DataFrame,
    config: DedupeConfig,
    run_stats: RunStats,
    progress_q: Optional[mp.Queue] = None,
) -> List[Dict[str, Any]]:
    pid = os.getpid()
    duplicates_found: List[Dict[str, Any]] = []
    _pairs_already_processed_fuzzy: set[Tuple[str, str]] = set()

    # CRITICAL: Track total items and processed items
    total_items_to_process = len(source_df_processed)
    items_actually_processed = 0

    rec_id = config.rec_id_col
    threshold = config.threshold
    algo_key_str = config.algorithm
    maps_for_algo = config.maps
    processing_mode = config.processing_mode
    apply_blocking_for_engine = config.apply_blocking
    dup_block_limit = int(config.dup_block_limit)

    coverage_gate = getattr(config, "coverage_gate", 0.60)
    MAX_TOTAL_WEIGHT = (
        sum(float(m.get("weight", 1.0)) for m in maps_for_algo)
        if maps_for_algo
        else 0.01
    )

    log.debug(
        f"[Worker {pid}] worker_duplicate starting. InputDF (preprocessed) shape: {source_df_processed.shape}, "
        f"Mode: {processing_mode}, Blocking: {apply_blocking_for_engine}, Algo: {algo_key_str}, Threshold: {threshold}, "
        f"Num Maps: {len(maps_for_algo) if maps_for_algo else 0}, RecID: {rec_id}"
    )

    algo_callable = ALGORITHM_MAP.get(algo_key_str)
    if not algo_callable:
        log.error(
            f"[Worker {pid}] Algorithm {algo_key_str} not in ALGORITHM_MAP. Defaulting."
        )
        algo_callable = ALGORITHM_MAP[DEFAULT_ALGORITHM_KEY]

    # ---- PATH 1: NO BLOCKING (Non-Blocked Deduplication) ----
    if not apply_blocking_for_engine:
        N = len(source_df_processed)
        log.info(
            f"[Worker {pid}] worker_duplicate: NON-BLOCKING Path. Rows: {N}. "
            f"Potential pairs: {N * (N - 1) // 2 if N > 1 else 0}"
        )
        MEMORY_GUARD_THRESHOLD_ELEMENTS_DEDUPE = 25_000_000

        if N * N > MEMORY_GUARD_THRESHOLD_ELEMENTS_DEDUPE and N > 1000:
            # Tuple-based fallback for large matrices
            log.warning(
                f"[Worker {pid}] Dedupe NB: Matrix too large ({N*N} > {MEMORY_GUARD_THRESHOLD_ELEMENTS_DEDUPE}). Tuple fallback."
            )
            df_as_tuples = list(source_df_processed.itertuples(index=False, name=None))
            col_to_idx = {col: i for i, col in enumerate(source_df_processed.columns)}
            n_tuples = len(df_as_tuples)

            for i in range(n_tuples):
                srow_tuple_bf = df_as_tuples[i]
                src_id_val_bf_raw = (
                    srow_tuple_bf[col_to_idx[rec_id]]
                    if rec_id in col_to_idx
                    else f"idx_{i}"
                )
                src_id_val_bf = _safe_str(src_id_val_bf_raw)
                if not src_id_val_bf:
                    src_id_val_bf = f"__blank_id_idx_{i}__"

                items_processed_bf = 1
                comparisons_bf = 0
                for j in range(i + 1, n_tuples):
                    rrow_tuple_bf = df_as_tuples[j]
                    ref_id_val_bf_raw = (
                        rrow_tuple_bf[col_to_idx[rec_id]]
                        if rec_id in col_to_idx
                        else f"idx_{j}"
                    )
                    ref_id_val_bf = _safe_str(ref_id_val_bf_raw)
                    if not ref_id_val_bf:
                        ref_id_val_bf = f"__blank_id_idx_{j}__"

                    comparisons_bf += 1

                    # Debug logging for self-match investigation
                    if src_id_val_bf == ref_id_val_bf:
                        if not src_id_val_bf.startswith("__blank_id_"):
                            log.debug(
                                f"[Worker {pid}] Skipping self-match: i={i}, j={j}, ID={src_id_val_bf}"
                            )
                        continue

                    pair_key_fuzzy_ids = tuple(sorted((src_id_val_bf, ref_id_val_bf)))
                    if pair_key_fuzzy_ids in _pairs_already_processed_fuzzy:
                        continue

                    current_pair_field_details_bf = []
                    total_score_bf, total_weight_bf, valid_comp_bf = 0.0, 0.0, False

                    if maps_for_algo:
                        for m_dedupe_fuzzy in maps_for_algo:
                            col_fetch = m_dedupe_fuzzy.get("source")
                            weight_bf = m_dedupe_fuzzy.get("weight", 1.0)
                            if not col_fetch or col_fetch not in col_to_idx:
                                continue
                            s_val_int = srow_tuple_bf[col_to_idx[col_fetch]]
                            r_val_int = rrow_tuple_bf[col_to_idx[col_fetch]]
                            safe_col_fetch_name = f"_safe_{col_fetch}"
                            s_val_str = (
                                srow_tuple_bf[col_to_idx[safe_col_fetch_name]]
                                if safe_col_fetch_name in col_to_idx
                                else _safe_str(s_val_int)
                            )
                            r_val_str = (
                                rrow_tuple_bf[col_to_idx[safe_col_fetch_name]]
                                if safe_col_fetch_name in col_to_idx
                                else _safe_str(r_val_int)
                            )

                            if not s_val_str or not r_val_str:
                                continue
                            valid_comp_bf = True
                            score_val = robust_score_calculation(
                                s1=s_val_str,
                                s2=r_val_str,
                                algo=algo_key_str,
                                fallback_algo="Ratio",
                                processing_mode=processing_mode,
                                ruleset_name=config.ruleset_name,
                                db_path=None,
                            )
                            total_score_bf += score_val * weight_bf
                            total_weight_bf += weight_bf
                            current_pair_field_details_bf.append(
                                {
                                    "source_field": col_fetch,
                                    "ref_field": col_fetch,
                                    "score": float(score_val),
                                    "weight": weight_bf,
                                    "algo_used": algo_key_str,
                                }
                            )

                    if valid_comp_bf and total_weight_bf > 0:
                        avg_score_bf = total_score_bf / total_weight_bf
                        if maps_for_algo and MAX_TOTAL_WEIGHT > 0:
                            coverage_d = total_weight_bf / MAX_TOTAL_WEIGHT
                            if coverage_d < coverage_gate:
                                log.debug(
                                    f"Coverage penalty applied: coverage={coverage_d:.2f} < gate={coverage_gate}"
                                )
                                avg_score_bf *= coverage_d
                        if avg_score_bf >= threshold:
                            _pairs_already_processed_fuzzy.add(pair_key_fuzzy_ids)
                            final_id1, final_id2 = pair_key_fuzzy_ids
                            rec = {
                                f"{rec_id}_1": final_id1,
                                f"{rec_id}_2": final_id2,
                                "confidence_score": round(avg_score_bf, 2),
                                "field_score_details": current_pair_field_details_bf,
                            }
                            if maps_for_algo:
                                for m_out in maps_for_algo:
                                    col_out = m_out.get("source")
                                    if not col_out or col_out not in col_to_idx:
                                        continue
                                    val_s_raw, val_r_raw = (
                                        srow_tuple_bf[col_to_idx[col_out]],
                                        rrow_tuple_bf[col_to_idx[col_out]],
                                    )
                                    if src_id_val_bf == final_id1:
                                        rec[
                                            f"REC1_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_s_raw)
                                        rec[
                                            f"REC2_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_r_raw)
                                    else:
                                        rec[
                                            f"REC1_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_r_raw)
                                        rec[
                                            f"REC2_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_s_raw)
                            duplicates_found.append(rec)

                if run_stats:
                    run_stats.update(comparisons_bf, items_processed_bf, progress_q)
                items_actually_processed += items_processed_bf

        else:
            # cdist path for non-blocking dedupe
            log.info(
                f"[Worker {pid}] worker_duplicate: Using VECTORIZED cdist NON-BLOCKING Path. N={N}"
            )
            all_field_score_matrices_nbd = []
            all_pair_validity_masks_nbd = []
            map_weights_list_nbd = []

            if maps_for_algo and N > 0:
                for m_map_idx, m_cfg in enumerate(maps_for_algo):
                    source_col_name = m_cfg.get("source")
                    map_weights_list_nbd.append(m_cfg.get("weight", 1.0))
                    if (
                        not source_col_name
                        or source_col_name not in source_df_processed.columns
                    ):
                        all_field_score_matrices_nbd.append(np.zeros((N, N)))
                        all_pair_validity_masks_nbd.append(np.zeros((N, N), dtype=bool))
                        continue
                    safe_s_col = f"_safe_{source_col_name}"
                    source_strings = (
                        source_df_processed[safe_s_col].to_numpy()
                        if safe_s_col in source_df_processed.columns
                        else source_df_processed[source_col_name]
                        .apply(_safe_str)
                        .to_numpy()
                    )
                    s_mask = np.array(
                        [bool(s_str) for s_str in source_strings], dtype=bool
                    )
                    if not s_mask.any():
                        all_field_score_matrices_nbd.append(np.zeros((N, N)))
                        all_pair_validity_masks_nbd.append(np.zeros((N, N), dtype=bool))
                        continue
                    all_pair_validity_masks_nbd.append(
                        s_mask[:, np.newaxis] & s_mask[np.newaxis, :]
                    )
                    scores_field_nbd = np.zeros((N, N))
                    if source_strings.size > 0:
                        try:
                            # Calculate cutoff for this mapping
                            cutoff = _get_score_cutoff(m_cfg, threshold)

                            # Log RF threads once per worker
                            if not getattr(_GLOBAL_CONFIG, "_rf_logged", False):
                                # Verify environment variable is set
                                env_threads = os.environ.get(
                                    "RAPIDFUZZ_THREAD_COUNT", "NOT SET"
                                )
                                log.info(
                                    f"[Worker {os.getpid()}] RF workers={_RF_WORKERS}, env var={env_threads}, procs={getattr(_GLOBAL_CONFIG,'max_workers',None)}"
                                )
                                try:
                                    _GLOBAL_CONFIG._rf_logged = True
                                except AttributeError:
                                    pass  # Some configs might be frozen/read-only

                            # Instrument cdist call
                            t0 = time.perf_counter()
                            scores_field_nbd = rf_process.cdist(
                                source_strings,
                                source_strings,
                                scorer=algo_callable,
                                score_cutoff=cutoff,
                                workers=_RF_WORKERS,
                            )
                            dt = time.perf_counter() - t0

                            if dt > 0.25:  # Lower threshold to catch more cases
                                safe_dt = dt if dt > 0 else 1e-9
                                ops_per_sec = len(source_strings) ** 2 / safe_dt / 1e6
                                _trace_cdist(
                                    f"cdist {algo_key_str} {len(source_strings)}x{len(source_strings)}={len(source_strings)**2} ops "
                                    f"took {dt:.2f}s ({ops_per_sec:.2f}M ops/s)"
                                )
                        except Exception as e_cdist:
                            log.error(
                                f"[Worker {pid}] Dedupe NB cdist failed map {m_map_idx} (col: {source_col_name}): {e_cdist}",
                                exc_info=True,
                            )
                    all_field_score_matrices_nbd.append(scores_field_nbd)

                if all_field_score_matrices_nbd and map_weights_list_nbd:
                    score_stack_nbd = np.stack(all_field_score_matrices_nbd, axis=0)
                    pair_mask_stack_nbd = np.stack(
                        all_pair_validity_masks_nbd, axis=0
                    ).astype(float)
                    weights_np_nbd = np.array(map_weights_list_nbd).reshape(-1, 1, 1)
                    weighted_scores_nbd = (
                        score_stack_nbd * weights_np_nbd * pair_mask_stack_nbd
                    )
                    sum_weighted_scores_nbd = weighted_scores_nbd.sum(axis=0)
                    actual_weights_nbd = (pair_mask_stack_nbd * weights_np_nbd).sum(
                        axis=0
                    )
                    avg_scores_nbd = np.zeros_like(sum_weighted_scores_nbd)
                    valid_weights_mask_nbd = actual_weights_nbd > 1e-9
                    avg_scores_nbd[valid_weights_mask_nbd] = (
                        sum_weighted_scores_nbd[valid_weights_mask_nbd]
                        / actual_weights_nbd[valid_weights_mask_nbd]
                    )
                    final_scores_nbd = avg_scores_nbd
                    if MAX_TOTAL_WEIGHT > 1e-9:
                        coverage_mat_nbd = actual_weights_nbd / MAX_TOTAL_WEIGHT
                        log.debug(
                            f"Applying vectorized coverage penalty to {len(coverage_mat_nbd)} pairs (non-blocked dedupe)."
                        )
                        penalty_factor_nbd = np.where(
                            coverage_mat_nbd < coverage_gate, coverage_mat_nbd, 1.0
                        )
                        final_scores_nbd = avg_scores_nbd * penalty_factor_nbd

                    s_indices_triu, r_indices_triu = np.triu_indices(N, k=1)
                    if s_indices_triu.size > 0:
                        scores_to_check_nbd = final_scores_nbd[
                            s_indices_triu, r_indices_triu
                        ]
                        match_indices_in_triu_nbd = np.where(
                            scores_to_check_nbd >= threshold
                        )[0]
                        final_s_indices_nbd = s_indices_triu[match_indices_in_triu_nbd]
                        final_r_indices_nbd = r_indices_triu[match_indices_in_triu_nbd]

                        for s_idx, r_idx in zip(
                            final_s_indices_nbd, final_r_indices_nbd
                        ):
                            id1_raw = (
                                source_df_processed.iloc[s_idx][rec_id]
                                if rec_id in source_df_processed.columns
                                else f"idx_{s_idx}"
                            )
                            id2_raw = (
                                source_df_processed.iloc[r_idx][rec_id]
                                if rec_id in source_df_processed.columns
                                else f"idx_{r_idx}"
                            )
                            id1_val, id2_val = _safe_str(id1_raw), _safe_str(id2_raw)
                            if not id1_val:
                                id1_val = f"__blank_id_s_idx_{s_idx}__"
                            if not id2_val:
                                id2_val = f"__blank_id_r_idx_{r_idx}__"
                            if id1_val == id2_val and not id1_val.startswith(
                                "__blank_id_"
                            ):
                                continue
                            pair_key_ids_nbd = tuple(sorted((id1_val, id2_val)))
                            if pair_key_ids_nbd in _pairs_already_processed_fuzzy:
                                continue
                            _pairs_already_processed_fuzzy.add(pair_key_ids_nbd)
                            final_id1, final_id2 = pair_key_ids_nbd
                            confidence_val = final_scores_nbd[s_idx, r_idx]
                            rec_data = {
                                f"{rec_id}_1": final_id1,
                                f"{rec_id}_2": final_id2,
                                "confidence_score": round(confidence_val, 2),
                            }

                            current_pair_field_details_nbd_cdist = []
                            if maps_for_algo:
                                for m_map_idx_val, m_map_config_val in enumerate(
                                    maps_for_algo
                                ):
                                    col_name_detail = m_map_config_val.get("source")
                                    weight_detail = m_map_config_val.get("weight", 0.0)
                                    individual_score_val = 0.0
                                    if m_map_idx_val < len(
                                        all_field_score_matrices_nbd
                                    ):
                                        current_field_score_matrix = (
                                            all_field_score_matrices_nbd[m_map_idx_val]
                                        )
                                        if (
                                            hasattr(current_field_score_matrix, "shape")
                                            and len(current_field_score_matrix.shape)
                                            == 2
                                            and 0
                                            <= s_idx
                                            < current_field_score_matrix.shape[0]
                                            and 0
                                            <= r_idx
                                            < current_field_score_matrix.shape[1]
                                        ):
                                            individual_score_val = (
                                                current_field_score_matrix[s_idx, r_idx]
                                            )
                                        else:
                                            log.error(
                                                f"Index/Shape issue for all_field_score_matrices_nbd map {m_map_idx_val}, s_idx {s_idx}, r_idx {r_idx}."
                                            )
                                    else:
                                        log.error(
                                            f"map_idx {m_map_idx_val} out of bounds for all_field_score_matrices_nbd"
                                        )
                                    current_pair_field_details_nbd_cdist.append(
                                        {
                                            "source_field": col_name_detail,
                                            "ref_field": col_name_detail,
                                            "score": float(individual_score_val),
                                            "weight": weight_detail,
                                            "algo_used": algo_key_str,
                                        }
                                    )
                            rec_data["field_score_details"] = (
                                current_pair_field_details_nbd_cdist
                            )

                            if maps_for_algo:
                                for m_out in maps_for_algo:
                                    col_out = m_out.get("source")
                                    val_s1_out, val_s2_out = None, None
                                    if (
                                        col_out
                                        and col_out in source_df_processed.columns
                                    ):
                                        val_s1_out = source_df_processed.iloc[s_idx][
                                            col_out
                                        ]
                                        val_s2_out = source_df_processed.iloc[r_idx][
                                            col_out
                                        ]
                                    if id1_val == final_id1:
                                        rec_data[
                                            f"REC1_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_s1_out)
                                        rec_data[
                                            f"REC2_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_s2_out)
                                    else:
                                        rec_data[
                                            f"REC1_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_s2_out)
                                        rec_data[
                                            f"REC2_{sanitize_column_name(str(col_out))}"
                                        ] = _safe_str(val_s1_out)
                            duplicates_found.append(rec_data)

            if run_stats:
                comparisons_made = N * (N - 1) // 2 if N > 1 else 0
                run_stats.update(comparisons_made, N, progress_q)
            items_actually_processed = N

    # ---- PATH 2: BLOCKING ENABLED ----
    elif apply_blocking_for_engine:
        log.info(f"[Worker {pid}] worker_duplicate: Using BLOCKING PATH.")

        try:
            grouped_data = source_df_processed.groupby(
                _ACTIVE_BLOCK_KEY_, observed=True
            )
        except KeyError as e:
            log.error(
                f"[Worker {pid}] worker_duplicate (blocked): Blocking key '{_ACTIVE_BLOCK_KEY_}' not found: {e}."
            )
            # Account for all items as "processed" even though we couldn't process them
            if run_stats:
                run_stats.update(0, total_items_to_process, progress_q)
            return duplicates_found

        # Collect block information
        block_sizes_info_list: List[Dict[str, Any]] = []
        total_items_in_blocks = 0

        for key_val_b, group_df_for_size_b in grouped_data:
            block_size = len(group_df_for_size_b)
            block_sizes_info_list.append(
                {
                    "key": key_val_b,
                    "metric": block_size,
                    "group_df_ref": group_df_for_size_b,
                }
            )
            total_items_in_blocks += block_size

        # CRITICAL: Check for orphaned items
        items_not_in_blocks = total_items_to_process - total_items_in_blocks
        if items_not_in_blocks > 0:
            log.warning(
                f"[Worker {pid}] Found {items_not_in_blocks} items not assigned to any block"
            )

        sorted_blocks_info_list = sorted(
            block_sizes_info_list, key=lambda x: x["metric"], reverse=True
        )

        # Process each block
        for block_info_item in sorted_blocks_info_list:
            block_key_val_b = block_info_item["key"]
            block_df_orig_b = block_info_item["group_df_ref"]
            N_block_val = block_info_item["metric"]

            # Skip empty blocks
            if not block_key_val_b or not str(block_key_val_b).strip():
                log.debug(
                    f"Skipping empty block key. {N_block_val} records will be marked as processed."
                )
                if run_stats:
                    run_stats.update(0, N_block_val, progress_q)
                items_actually_processed += N_block_val
                continue

            if N_block_val < 2:
                if run_stats:
                    run_stats.update(0, N_block_val, progress_q)
                items_actually_processed += N_block_val
                continue

            estimated_pairs_in_block_b = N_block_val * (N_block_val - 1) // 2

            # Path 2.1: Small blocks or blocks exceeding dup_block_limit (Tuple Iteration)
            if (
                estimated_pairs_in_block_b < SMALL_BLOCK_DEDUPE_PAIRS_THRESHOLD
                or N_block_val > dup_block_limit
            ):
                log_reason_b = (
                    "SMALL"
                    if estimated_pairs_in_block_b < SMALL_BLOCK_DEDUPE_PAIRS_THRESHOLD
                    else f"LARGE (>{dup_block_limit}, fallback)"
                )
                if log.isEnabledFor(logging.DEBUG):
                    log.debug(
                        f"Block '{block_key_val_b}' ({N_block_val} recs, {estimated_pairs_in_block_b} pairs) -> TUPLE ITERATION ({log_reason_b})."
                    )

                comparisons_block_tuple = 0
                block_col_to_idx_b = {
                    col_b: i_b for i_b, col_b in enumerate(block_df_orig_b.columns)
                }
                block_as_tuples_b = list(
                    block_df_orig_b.itertuples(index=False, name=None)
                )

                for i_b_outer in range(N_block_val):
                    srow_tuple_b_outer = block_as_tuples_b[i_b_outer]
                    src_id_val_b_outer_raw = (
                        srow_tuple_b_outer[block_col_to_idx_b[rec_id]]
                        if rec_id in block_col_to_idx_b
                        else f"b_idx_{i_b_outer}"
                    )
                    src_id_val_b_outer = _safe_str(src_id_val_b_outer_raw)
                    if not src_id_val_b_outer:
                        src_id_val_b_outer = f"__blank_id_b_idx_{i_b_outer}__"

                    for j_b_inner in range(i_b_outer + 1, N_block_val):
                        comparisons_block_tuple += 1
                        rrow_tuple_b_inner = block_as_tuples_b[j_b_inner]
                        ref_id_val_b_inner_raw = (
                            rrow_tuple_b_inner[block_col_to_idx_b[rec_id]]
                            if rec_id in block_col_to_idx_b
                            else f"b_idx_{j_b_inner}"
                        )
                        ref_id_val_b_inner = _safe_str(ref_id_val_b_inner_raw)
                        if not ref_id_val_b_inner:
                            ref_id_val_b_inner = f"__blank_id_b_idx_{j_b_inner}__"

                        if (
                            src_id_val_b_outer == ref_id_val_b_inner
                            and not src_id_val_b_outer.startswith("__blank_id_")
                        ):
                            continue
                        pair_key_block_ids_b_tuple = tuple(
                            sorted((src_id_val_b_outer, ref_id_val_b_inner))
                        )
                        if pair_key_block_ids_b_tuple in _pairs_already_processed_fuzzy:
                            continue

                        current_pair_field_details_b_t = []
                        total_score_b_t, total_weight_b_t, valid_comp_b_t = (
                            0.0,
                            0.0,
                            False,
                        )

                        if maps_for_algo:
                            for m_dedupe_b_t in maps_for_algo:
                                col_fetch_b_t = m_dedupe_b_t.get("source")
                                weight_val_b_t = m_dedupe_b_t.get("weight", 1.0)
                                if (
                                    not col_fetch_b_t
                                    or col_fetch_b_t not in block_col_to_idx_b
                                ):
                                    continue

                                s_val_int_b_t = srow_tuple_b_outer[
                                    block_col_to_idx_b[col_fetch_b_t]
                                ]
                                r_val_int_b_t = rrow_tuple_b_inner[
                                    block_col_to_idx_b[col_fetch_b_t]
                                ]
                                safe_col_name_b_t = f"_safe_{col_fetch_b_t}"
                                s_str_b_t = (
                                    srow_tuple_b_outer[
                                        block_col_to_idx_b[safe_col_name_b_t]
                                    ]
                                    if safe_col_name_b_t in block_col_to_idx_b
                                    else _safe_str(s_val_int_b_t)
                                )
                                r_str_b_t = (
                                    rrow_tuple_b_inner[
                                        block_col_to_idx_b[safe_col_name_b_t]
                                    ]
                                    if safe_col_name_b_t in block_col_to_idx_b
                                    else _safe_str(r_val_int_b_t)
                                )

                                if not s_str_b_t or not r_str_b_t:
                                    continue
                                valid_comp_b_t = True
                                score_v_b_t = robust_score_calculation(
                                    s1=s_str_b_t,
                                    s2=r_str_b_t,
                                    algo=algo_key_str,
                                    fallback_algo="Ratio",
                                    processing_mode=processing_mode,
                                    ruleset_name=config.ruleset_name,
                                    db_path=None,
                                )
                                total_score_b_t += score_v_b_t * weight_val_b_t
                                total_weight_b_t += weight_val_b_t
                                current_pair_field_details_b_t.append(
                                    {
                                        "source_field": col_fetch_b_t,
                                        "ref_field": col_fetch_b_t,
                                        "score": float(score_v_b_t),
                                        "weight": weight_val_b_t,
                                        "algo_used": algo_key_str,
                                    }
                                )

                        if valid_comp_b_t and total_weight_b_t > 0:
                            avg_s_b_t = total_score_b_t / total_weight_b_t
                            if maps_for_algo and MAX_TOTAL_WEIGHT > 0:
                                cov_b_t = total_weight_b_t / MAX_TOTAL_WEIGHT
                                if cov_b_t < coverage_gate:
                                    log.debug(
                                        f"Coverage penalty applied: coverage={cov_b_t:.2f} < gate={coverage_gate}"
                                    )
                                    avg_s_b_t *= cov_b_t

                            if avg_s_b_t >= threshold:
                                _pairs_already_processed_fuzzy.add(
                                    pair_key_block_ids_b_tuple
                                )
                                f_id1_b_t, f_id2_b_t = pair_key_block_ids_b_tuple
                                rec_val_b_t = {
                                    f"{rec_id}_1": f_id1_b_t,
                                    f"{rec_id}_2": f_id2_b_t,
                                    "confidence_score": round(avg_s_b_t, 2),
                                }
                                rec_val_b_t["field_score_details"] = (
                                    current_pair_field_details_b_t
                                )
                                if maps_for_algo:
                                    for m_o_b_t in maps_for_algo:
                                        co_b_t = m_o_b_t.get("source")
                                        if (
                                            not co_b_t
                                            or co_b_t not in block_col_to_idx_b
                                        ):
                                            continue
                                        v_s_b_t, v_r_b_t = (
                                            srow_tuple_b_outer[
                                                block_col_to_idx_b[co_b_t]
                                            ],
                                            rrow_tuple_b_inner[
                                                block_col_to_idx_b[co_b_t]
                                            ],
                                        )
                                        if src_id_val_b_outer == f_id1_b_t:
                                            rec_val_b_t[
                                                f"REC1_{sanitize_column_name(str(co_b_t))}"
                                            ] = _safe_str(v_s_b_t)
                                            rec_val_b_t[
                                                f"REC2_{sanitize_column_name(str(co_b_t))}"
                                            ] = _safe_str(v_r_b_t)
                                        else:
                                            rec_val_b_t[
                                                f"REC1_{sanitize_column_name(str(co_b_t))}"
                                            ] = _safe_str(v_r_b_t)
                                            rec_val_b_t[
                                                f"REC2_{sanitize_column_name(str(co_b_t))}"
                                            ] = _safe_str(v_s_b_t)
                                duplicates_found.append(rec_val_b_t)

                if run_stats:
                    run_stats.update(comparisons_block_tuple, N_block_val, progress_q)
                items_actually_processed += N_block_val

            # Path 2.2: Medium/Large blocks, suitable for cdist
            else:
                if log.isEnabledFor(logging.DEBUG):
                    log.debug(
                        f"Block '{block_key_val_b}' ({N_block_val} recs, {estimated_pairs_in_block_b} pairs) -> VECTORIZED cdist."
                    )

                all_field_score_matrices_b_cdist = []
                all_pair_validity_masks_b_cdist = []
                map_weights_list_b_cdist = []

                if not maps_for_algo or N_block_val == 0:
                    if run_stats:
                        run_stats.update(0, N_block_val, progress_q)
                    items_actually_processed += N_block_val
                    continue

                for m_map_idx_val_b, m_map_config_val_b in enumerate(maps_for_algo):
                    source_col_name_val_b = m_map_config_val_b.get("source")
                    map_weights_list_b_cdist.append(
                        m_map_config_val_b.get("weight", 1.0)
                    )

                    if (
                        not source_col_name_val_b
                        or source_col_name_val_b not in block_df_orig_b.columns
                    ):
                        all_field_score_matrices_b_cdist.append(
                            np.zeros((N_block_val, N_block_val))
                        )
                        all_pair_validity_masks_b_cdist.append(
                            np.zeros((N_block_val, N_block_val), dtype=bool)
                        )
                        continue

                    safe_s_col_val_b = f"_safe_{source_col_name_val_b}"
                    source_strings_val_b = (
                        block_df_orig_b[safe_s_col_val_b].tolist()
                        if safe_s_col_val_b in block_df_orig_b.columns
                        else block_df_orig_b[source_col_name_val_b]
                        .apply(_safe_str)
                        .tolist()
                    )
                    s_mask_val_b = np.array(
                        [bool(s_str) for s_str in source_strings_val_b], dtype=bool
                    )

                    if not s_mask_val_b.any():
                        all_field_score_matrices_b_cdist.append(
                            np.zeros((N_block_val, N_block_val))
                        )
                        all_pair_validity_masks_b_cdist.append(
                            np.zeros((N_block_val, N_block_val), dtype=bool)
                        )
                        continue

                    all_pair_validity_masks_b_cdist.append(
                        s_mask_val_b[:, np.newaxis] & s_mask_val_b[np.newaxis, :]
                    )

                    scores_field_val_b = np.zeros((N_block_val, N_block_val))
                    if source_strings_val_b:
                        try:
                            scores_field_val_b = rf_process.cdist(
                                source_strings_val_b,
                                source_strings_val_b,
                                scorer=algo_callable,
                                score_cutoff=0,
                                workers=1,
                            )
                        except Exception as e_b_cdist:
                            log.error(
                                f"Block '{block_key_val_b}' cdist map {m_map_idx_val_b} fail: {e_b_cdist}",
                                exc_info=True,
                            )
                    all_field_score_matrices_b_cdist.append(scores_field_val_b)

                if not all_field_score_matrices_b_cdist or not map_weights_list_b_cdist:
                    if run_stats:
                        run_stats.update(
                            N_block_val * (N_block_val - 1) // 2
                            if N_block_val > 1
                            else 0,
                            N_block_val,
                            progress_q,
                        )
                    items_actually_processed += N_block_val
                    continue

                score_stack_b_cdist = np.stack(all_field_score_matrices_b_cdist, axis=0)
                pair_mask_stack_b_cdist = np.stack(
                    all_pair_validity_masks_b_cdist, axis=0
                ).astype(float)
                weights_np_b_cdist = np.array(map_weights_list_b_cdist).reshape(
                    -1, 1, 1
                )
                weighted_scores_contrib_b_cdist = (
                    score_stack_b_cdist * weights_np_b_cdist * pair_mask_stack_b_cdist
                )
                sum_weighted_scores_mat_b_cdist = weighted_scores_contrib_b_cdist.sum(
                    axis=0
                )
                actual_weights_used_mat_b_cdist = (
                    pair_mask_stack_b_cdist * weights_np_b_cdist
                ).sum(axis=0)
                avg_scores_mat_b_cdist = np.zeros_like(sum_weighted_scores_mat_b_cdist)
                valid_weights_mask_mat_b_cdist = actual_weights_used_mat_b_cdist > 1e-9
                avg_scores_mat_b_cdist[valid_weights_mask_mat_b_cdist] = (
                    sum_weighted_scores_mat_b_cdist[valid_weights_mask_mat_b_cdist]
                    / actual_weights_used_mat_b_cdist[valid_weights_mask_mat_b_cdist]
                )
                final_scores_mat_b_cdist = avg_scores_mat_b_cdist

                if MAX_TOTAL_WEIGHT > 1e-9:
                    coverage_mat_b_cdist = (
                        actual_weights_used_mat_b_cdist / MAX_TOTAL_WEIGHT
                    )
                    log.debug(
                        f"Applying vectorized coverage penalty to {len(coverage_mat_b_cdist)} pairs (blocked dedupe)."
                    )
                    penalty_factor_mat_b_cdist = np.where(
                        coverage_mat_b_cdist < coverage_gate, coverage_mat_b_cdist, 1.0
                    )
                    final_scores_mat_b_cdist = (
                        avg_scores_mat_b_cdist * penalty_factor_mat_b_cdist
                    )

                s_indices_triu_b_cdist, r_indices_triu_b_cdist = np.triu_indices(
                    N_block_val, k=1
                )
                if s_indices_triu_b_cdist.size > 0:
                    scores_to_check_b_cdist = final_scores_mat_b_cdist[
                        s_indices_triu_b_cdist, r_indices_triu_b_cdist
                    ]
                    match_indices_in_triu_b_cdist = np.where(
                        scores_to_check_b_cdist >= threshold
                    )[0]
                    final_s_indices_b_cdist_match = s_indices_triu_b_cdist[
                        match_indices_in_triu_b_cdist
                    ]
                    final_r_indices_b_cdist_match = r_indices_triu_b_cdist[
                        match_indices_in_triu_b_cdist
                    ]

                    for s_idx_val_bcd, r_idx_val_bcd in zip(
                        final_s_indices_b_cdist_match, final_r_indices_b_cdist_match
                    ):
                        id1_raw_bcd = (
                            block_df_orig_b.iloc[s_idx_val_bcd][rec_id]
                            if rec_id in block_df_orig_b.columns
                            else f"b_cdist_s_{s_idx_val_bcd}"
                        )
                        id2_raw_bcd = (
                            block_df_orig_b.iloc[r_idx_val_bcd][rec_id]
                            if rec_id in block_df_orig_b.columns
                            else f"b_cdist_r_{r_idx_val_bcd}"
                        )
                        id1_val_bcd, id2_val_bcd = (
                            _safe_str(id1_raw_bcd),
                            _safe_str(id2_raw_bcd),
                        )
                        if not id1_val_bcd:
                            id1_val_bcd = f"__blank_b_cdist_s_{s_idx_val_bcd}__"
                        if not id2_val_bcd:
                            id2_val_bcd = f"__blank_b_cdist_r_{r_idx_val_bcd}__"
                        if id1_val_bcd == id2_val_bcd and not id1_val_bcd.startswith(
                            "__blank_id_"
                        ):
                            continue

                        pair_key_ids_bcd = tuple(sorted((id1_val_bcd, id2_val_bcd)))
                        if pair_key_ids_bcd in _pairs_already_processed_fuzzy:
                            continue
                        _pairs_already_processed_fuzzy.add(pair_key_ids_bcd)

                        final_id1_bcd, final_id2_bcd = pair_key_ids_bcd
                        confidence_val_bcd = final_scores_mat_b_cdist[
                            s_idx_val_bcd, r_idx_val_bcd
                        ]
                        rec_val_bcd = {
                            f"{rec_id}_1": final_id1_bcd,
                            f"{rec_id}_2": final_id2_bcd,
                            "confidence_score": round(confidence_val_bcd, 2),
                        }

                        current_pair_field_details_bcd = []
                        if maps_for_algo:
                            for m_map_idx_bcd, m_map_config_bcd_detail in enumerate(
                                maps_for_algo
                            ):
                                col_name_detail_bcd = m_map_config_bcd_detail.get(
                                    "source"
                                )
                                weight_detail_bcd = m_map_config_bcd_detail.get(
                                    "weight", 0.0
                                )
                                individual_field_score_bcd = 0.0
                                if m_map_idx_bcd < len(
                                    all_field_score_matrices_b_cdist
                                ):
                                    current_field_score_matrix_bcd = (
                                        all_field_score_matrices_b_cdist[m_map_idx_bcd]
                                    )
                                    if (
                                        hasattr(current_field_score_matrix_bcd, "shape")
                                        and len(current_field_score_matrix_bcd.shape)
                                        == 2
                                        and 0
                                        <= s_idx_val_bcd
                                        < current_field_score_matrix_bcd.shape[0]
                                        and 0
                                        <= r_idx_val_bcd
                                        < current_field_score_matrix_bcd.shape[1]
                                    ):
                                        individual_field_score_bcd = (
                                            current_field_score_matrix_bcd[
                                                s_idx_val_bcd, r_idx_val_bcd
                                            ]
                                        )
                                    else:
                                        log.error(
                                            f"[Worker {pid}] Index/Shape issue for all_field_score_matrices_block (dedupe blocked cdist) "
                                            f"map {m_map_idx_bcd}, s_idx {s_idx_val_bcd}, r_idx {r_idx_val_bcd}. Shape: {current_field_score_matrix_bcd.shape if hasattr(current_field_score_matrix_bcd, 'shape') else 'N/A'}"
                                        )
                                else:
                                    log.error(
                                        f"[Worker {pid}] map_idx {m_map_idx_bcd} out of bounds for all_field_score_matrices_b_cdist (len: {len(all_field_score_matrices_b_cdist)}) in dedupe blocked cdist"
                                    )

                                current_pair_field_details_bcd.append(
                                    {
                                        "source_field": col_name_detail_bcd,
                                        "ref_field": col_name_detail_bcd,
                                        "score": float(individual_field_score_bcd),
                                        "weight": weight_detail_bcd,
                                        "algo_used": algo_key_str,
                                    }
                                )
                        rec_val_bcd["field_score_details"] = (
                            current_pair_field_details_bcd
                        )

                        if maps_for_algo:
                            for m_o_bcd in maps_for_algo:
                                co_bcd = m_o_bcd.get("source")
                                v1o_bcd, v2o_bcd = None, None
                                if co_bcd and co_bcd in block_df_orig_b.columns:
                                    v1o_bcd = block_df_orig_b.iloc[s_idx_val_bcd][
                                        co_bcd
                                    ]
                                    v2o_bcd = block_df_orig_b.iloc[r_idx_val_bcd][
                                        co_bcd
                                    ]
                                if id1_val_bcd == final_id1_bcd:
                                    rec_val_bcd[
                                        f"REC1_{sanitize_column_name(str(co_bcd))}"
                                    ] = _safe_str(v1o_bcd)
                                    rec_val_bcd[
                                        f"REC2_{sanitize_column_name(str(co_bcd))}"
                                    ] = _safe_str(v2o_bcd)
                                else:
                                    rec_val_bcd[
                                        f"REC1_{sanitize_column_name(str(co_bcd))}"
                                    ] = _safe_str(v2o_bcd)
                                    rec_val_bcd[
                                        f"REC2_{sanitize_column_name(str(co_bcd))}"
                                    ] = _safe_str(v1o_bcd)
                        duplicates_found.append(rec_val_bcd)

                if run_stats:
                    comparisons_this_block_final_cdist = (
                        N_block_val * (N_block_val - 1) // 2 if N_block_val > 1 else 0
                    )
                    run_stats.update(
                        comparisons_this_block_final_cdist, N_block_val, progress_q
                    )
                items_actually_processed += N_block_val

        # CRITICAL: Account for orphaned items
        if items_not_in_blocks > 0 and run_stats:
            log.info(
                f"[Worker {pid}] Accounting for {items_not_in_blocks} items not in any block"
            )
            run_stats.update(0, items_not_in_blocks, progress_q)
            items_actually_processed += items_not_in_blocks

    else:
        # Unhandled combination
        log.warning(
            f"[Worker {pid}] worker_duplicate: Unhandled path. apply_blocking={apply_blocking_for_engine}, mode='{processing_mode}'."
        )
        if run_stats:
            run_stats.update(0, total_items_to_process, progress_q)
        items_actually_processed = total_items_to_process

    # CRITICAL: Final validation and accounting
    if items_actually_processed < total_items_to_process:
        missing_items = total_items_to_process - items_actually_processed
        log.warning(
            f"[Worker {pid}] Final accounting: {missing_items} items were not processed"
        )
        if run_stats:
            run_stats.update(0, missing_items, progress_q)

    log.debug(
        f"[Worker {pid}] worker_duplicate finished chunk. Found {len(duplicates_found)} duplicate pairs. Processed {items_actually_processed}/{total_items_to_process} items."
    )
    return duplicates_found


# --------------------------------------------------------------------------- #
#  (Re)load rulesets
# --------------------------------------------------------------------------- #
def reload_rulesets(
    dir_path: str | Path | None = None, *, verbose: bool = False
) -> dict[str, dict]:
    """
    Clears the in-memory ruleset cache and reloads all .yaml files.
    Uses the 'name' key inside each YAML file as the dictionary key.

    Parameters
    ----------
    dir_path : Path | str | None
        Folder holding your *.yaml* rulesets.
        If *None*, defaults to Ã¢â‚¬Å“<package-root>/rulesetsÃ¢â‚¬Â.

    Returns
    -------
    dict[str, dict]
        The updated global ``AVAILABLE_RULESETS`` mapping.
    """
    global AVAILABLE_RULESETS  # Ensure we modify the global dict
    if dir_path is None:
        # Correctly locate the 'rulesets' directory relative to this engine.py file
        dir_path = Path(__file__).parent.parent / "rulesets"
        log.debug(
            f"Defaulting ruleset directory to: {dir_path.resolve()}"
        )  # Changed from log.info

    # Wipe old entries, then repopulate
    load_ruleset_dir(dir_path)  # type: ignore # This function updates AVAILABLE_RULESETS

    # Ensure AVAILABLE_RULESETS is correctly populated by load_ruleset_dir
    with _RULESET_LOCK:
        if verbose:
            log.info(
                "Rulesets reloaded. Count: %s. Available: %s",
                len(AVAILABLE_RULESETS),
                list(AVAILABLE_RULESETS.keys()),
            )
        else:
            log.debug("Rulesets reloaded (%d)", len(AVAILABLE_RULESETS))
        # Return a copy to prevent external modification
        return AVAILABLE_RULESETS.copy()


def score_half_memory_block(
    small_block: pd.DataFrame,
    large_file_path: str,
    block_key: str,
    large_file_role: str,  # 'source' or 'ref'
    config: dict,
    s_id_col: str,
    r_id_col: str,
) -> Tuple[List[Dict[str, Any]], int]:
    """
    Worker for half-in-memory mode. Receives a small block as a DataFrame
    and reads the corresponding block from the large Parquet file on disk.
    """
    # Read the matching block from the large Parquet file using a filter
    try:
        large_block = pq.read_table(
            large_file_path, filters=[("_block_key", "=", block_key)]
        ).to_pandas(types_mapper=pd.ArrowDtype)
    except Exception as e:
        log.error(f"Failed to read block '{block_key}' from {large_file_path}: {e}")
        return [], 0

    # Assign DataFrames based on their original role
    if large_file_role == "source":
        source_block_df = large_block
        ref_block_df = small_block
    else:
        source_block_df = small_block
        ref_block_df = large_block

    # Now that we have both blocks in memory, we can call the original worker
    return score_block_pair(source_block_df, ref_block_df, config, s_id_col, r_id_col)


def score_block_pair(
    source_block: pd.DataFrame,
    ref_block: pd.DataFrame,
    config: dict,
    s_id_col: str,
    r_id_col: str,
) -> Tuple[List[Dict[str, Any]], int]:
    """
    Compares records in a source block against a reference block using a
    vectorized approach (rapidfuzz.process.cdist) for high performance.
    """
    pid = os.getpid()
    matches: List[Dict[str, Any]] = []

    # --- Configuration ---
    maps = config.get("maps", config.get("columns", []))
    algo_key_str = config.get("algorithm", "WRatio")
    threshold = config.get("threshold", 80)
    algo_callable = ALGORITHM_MAP.get(algo_key_str, rf_fuzz.WRatio)
    block_key_val = config.get("block_key", "")

    if not maps or source_block.empty or ref_block.empty:
        return [], 0

    S, R = len(source_block), len(ref_block)
    comparisons_made = S * R

    # --- Weighted Score Calculation using Matrix Operations---

    # Pre-calculate the total possible weight for coverage calculation
    MAX_TOTAL_WEIGHT = sum(float(m.get("weight", 1.0)) for m in maps)
    NEW_COVERAGE_GATE = 0.60  # The same gate used in the speed mode

    total_weighted_scores = np.zeros((S, R), dtype=np.float32)
    total_weights_used = np.zeros((S, R), dtype=np.float32)

    # Micro-timer for block processing
    t_block_start = time.perf_counter()

    for mapping in maps:
        s_col, r_col = mapping["source"], mapping["ref"]
        weight = mapping.get("weight", 1.0)

        if s_col not in source_block.columns or r_col not in ref_block.columns:
            continue

        t0 = time.perf_counter()

        # Extract series and convert to lists of strings for cdist
        # .tolist() is faster than .apply(_safe_str) for this purpose
        s_list = source_block[s_col].fillna("").astype(str, copy=False).to_numpy()
        r_list = ref_block[r_col].fillna("").astype(str, copy=False).to_numpy()

        # Create boolean masks to identify non-empty strings
        s_mask = np.array([bool(s) for s in s_list], dtype=bool)
        r_mask = np.array([bool(r) for r in r_list], dtype=bool)

        t_build_done = time.perf_counter()

        # Skip if one of the columns has no valid data in this block
        if not np.any(s_mask) or not np.any(r_mask):
            continue

        # Use cdist to get a matrix of scores for the current field
        # Calculate cutoff based on algorithm and threshold
        cutoff = max(threshold - 30, 65) if threshold > 0 else 0

        # Log RF threads once per worker
        if not getattr(_GLOBAL_CONFIG, "_rf_logged", False):
            # Verify environment variable is set
            env_threads = os.environ.get("RAPIDFUZZ_THREAD_COUNT", "NOT SET")
            log.info(
                f"[Worker {os.getpid()}] RF workers={_RF_WORKERS}, env var={env_threads}, procs={getattr(_GLOBAL_CONFIG,'max_workers',None)}"
            )
            try:
                _GLOBAL_CONFIG._rf_logged = True
            except AttributeError:
                pass  # Some configs might be frozen/read-only

        # Instrument cdist call
        score_matrix = rf_process.cdist(
            s_list,
            r_list,
            scorer=algo_callable,
            score_cutoff=cutoff,
            workers=_RF_WORKERS,
        )
        t_cdist_done = time.perf_counter()

        dt = t_cdist_done - t_build_done
        if dt > 0.25:  # Lower threshold to catch more cases
            safe_dt = dt if dt > 0 else 1e-9
            ops_per_sec = len(s_list) * len(r_list) / safe_dt / 1e6
            _trace_cdist(
                f"cdist {algo_key_str} {len(s_list)}x{len(r_list)}={len(s_list)*len(r_list)} ops "
                f"took {dt:.2f}s ({ops_per_sec:.2f}M ops/s)"
            )

        # Create a 2D validity mask for pairs where both strings are non-empty
        pair_validity_mask = np.outer(s_mask, r_mask)

        # Add the weighted scores for valid pairs to the total
        total_weighted_scores += score_matrix * weight * pair_validity_mask
        total_weights_used += weight * pair_validity_mask

        t_post_done = time.perf_counter()

        # Log slow mapping processing
        if TRACE_BLOCKS and (t_post_done - t0) > 1.0:
            log.info(
                f"[TRACE-BLOCK-SLOW] key={block_key_val} mapping={s_col}->{r_col} "
                f"algo={algo_key_str} cutoff={cutoff} "
                f"build={t_build_done-t0:.2f}s cdist={t_cdist_done-t_build_done:.2f}s "
                f"post={t_post_done-t_cdist_done:.2f}s total={t_post_done-t0:.2f}s "
                f"s={len(s_list)} r={len(r_list)}"
            )

    # Calculate the final average score, avoiding division by zero
    # np.divide is safer and faster than manual checking
    final_scores = np.divide(
        total_weighted_scores,
        total_weights_used,
        out=np.zeros_like(total_weighted_scores),
        where=(total_weights_used != 0),
    )

    # Apply the coverage penalty
    if MAX_TOTAL_WEIGHT > 0:
        coverage = total_weights_used / MAX_TOTAL_WEIGHT
        # Apply penalty where coverage is below the gate, otherwise use a factor of 1
        penalty_factor = np.where(coverage < NEW_COVERAGE_GATE, coverage, 1.0)
        final_scores *= penalty_factor

    # Find all pairs that meet the threshold
    match_indices_s, match_indices_r = np.where(final_scores >= threshold)

    if match_indices_s.size == 0:
        return [], comparisons_made

    # --- Efficiently build the results list ---
    source_matches = source_block.iloc[match_indices_s]
    ref_matches = ref_block.iloc[match_indices_r]

    for i in range(len(match_indices_s)):
        s_row = source_matches.iloc[i]
        r_row = ref_matches.iloc[i]

        match_details = {
            s_id_col: s_row[s_id_col],
            r_id_col: r_row[r_id_col],
            "confidence_score": round(
                final_scores[match_indices_s[i], match_indices_r[i]], 2
            ),
            # field_score_details is complex to get here, so we omit it for performance
            # and rely on the overall confidence score.
            "field_score_details": "[]",
        }

        # Add all original columns with prefixes
        for col in source_block.columns:
            if col != "_block_key":
                match_details[f"s_{sanitize_column_name(col)}"] = s_row[col]
        for col in ref_block.columns:
            if col != "_block_key":
                match_details[f"r_{sanitize_column_name(col)}"] = r_row[col]

        matches.append(match_details)

    log.debug(
        f"[Worker {pid}] Processed block ({comparisons_made} comps). Found {len(matches)} matches."
    )
    return matches, comparisons_made


# --------------------------------------------------------------------------- #
# Arrow-friendly adapter used by the new out-of-core scale mode
# --------------------------------------------------------------------------- #


def match_pairs_arrow(
    left_tbl: pa.Table,
    right_tbl: pa.Table,
    mapping_cfg: Dict[str, Any],
    s_id_col: str | None = None,
    r_id_col: str | None = None,
) -> pa.Table:
    """
    Compares records in two Arrow tables using a fully vectorized,
    pandas-free approach for maximum performance.
    """
    # --- 1. Configuration ---
    maps = mapping_cfg.get("maps", mapping_cfg.get("columns", []))
    algo_key_str = mapping_cfg.get("algorithm", "WRatio")
    threshold = mapping_cfg.get("threshold", 80)
    algo_callable = ALGORITHM_MAP.get(algo_key_str, rf_fuzz.WRatio)

    s_id_col_name = s_id_col or "source_id"
    r_id_col_name = r_id_col or "ref_id"

    if not maps or left_tbl.num_rows == 0 or right_tbl.num_rows == 0:
        return pa.Table.from_pydict({})

    if s_id_col_name not in left_tbl.column_names:
        raise KeyError(
            f'Field "{s_id_col_name}" does not exist in source table schema. Available: {left_tbl.schema.names}'
        )
    if r_id_col_name not in right_tbl.column_names:
        raise KeyError(
            f'Field "{r_id_col_name}" does not exist in reference table schema. Available: {right_tbl.schema.names}'
        )

    S, R = left_tbl.num_rows, right_tbl.num_rows

    # --- 2. Vectorized Weighted Score Calculation ---
    MAX_TOTAL_WEIGHT = sum(float(m.get("weight", 1.0)) for m in maps)
    COVERAGE_GATE = 0.60

    total_weighted_scores = np.zeros((S, R), dtype=np.float32)
    total_weights_used = np.zeros((S, R), dtype=np.float32)

    for mapping in maps:
        s_col, r_col = mapping["source"], mapping["ref"]
        weight = mapping.get("weight", 1.0)

        if s_col not in left_tbl.column_names or r_col not in right_tbl.column_names:
            continue

        s_list = np.array(left_tbl.column(s_col).to_pylist())  # Convert once to numpy
        r_list = np.array(right_tbl.column(r_col).to_pylist())

        s_mask = np.array([bool(s) for s in s_list], dtype=bool)
        r_mask = np.array([bool(r) for r in r_list], dtype=bool)

        if not np.any(s_mask) or not np.any(r_mask):
            continue

        # ++ AUDIT FIX: Explicitly set workers=1 to prevent process fan-out on Windows. ++
        s_list = np.ascontiguousarray(s_list)
        r_list = np.ascontiguousarray(r_list)
        # Calculate cutoff based on algorithm and threshold
        cutoff = max(threshold - 30, 65) if threshold > 0 else 0

        # Instrument cdist call
        t0 = time.perf_counter()
        score_matrix = rf_process.cdist(
            s_list,
            r_list,
            scorer=algo_callable,
            score_cutoff=cutoff,
            workers=_RF_WORKERS,
        )
        dt = time.perf_counter() - t0

        if dt > 1.0:  # Only log slow ones
            safe_dt = dt if dt > 0 else 1e-9
            ops_per_sec = len(s_list) * len(r_list) / safe_dt / 1e6
            _trace_cdist(
                f"cdist {algo_key_str} {len(s_list)}x{len(r_list)}={len(s_list)*len(r_list)} ops "
                f"took {dt:.2f}s ({ops_per_sec:.2f}M ops/s)"
            )

        pair_validity_mask = np.outer(s_mask, r_mask)

        weighted_scores_for_map = np.where(pair_validity_mask, score_matrix * weight, 0)
        total_weighted_scores += weighted_scores_for_map

        weights_for_map = np.where(pair_validity_mask, weight, 0)
        total_weights_used += weights_for_map

    # --- 3. Final Score Calculation & Filtering ---
    final_scores = np.divide(
        total_weighted_scores,
        total_weights_used,
        out=np.zeros_like(total_weighted_scores),
        where=(total_weights_used != 0),
    )

    if MAX_TOTAL_WEIGHT > 0:
        coverage = total_weights_used / MAX_TOTAL_WEIGHT
        penalty_factor = np.where(coverage < COVERAGE_GATE, coverage, 1.0)
        final_scores *= penalty_factor

    match_indices_s, match_indices_r = np.where(final_scores >= threshold)

    if match_indices_s.size == 0:
        if left_tbl.num_rows > 0 and right_tbl.num_rows > 0:
            log.warning(
                f"Matcher returned 0 rows for a non-empty block (L={left_tbl.num_rows}, R={right_tbl.num_rows}, threshold={threshold})"
            )
        return pa.Table.from_pydict({})

    # --- 4. Build Result Arrow Table ---
    result_dict = {
        s_id_col_name: left_tbl.column(s_id_col_name).take(
            pa.array(match_indices_s, type=pa.uint64())
        ),
        r_id_col_name: right_tbl.column(r_id_col_name).take(
            pa.array(match_indices_r, type=pa.uint64())
        ),
        "confidence_score": pa.array(final_scores[match_indices_s, match_indices_r]),
        "field_score_details": pa.array([json.dumps([])] * len(match_indices_s)),
    }

    for col_name in left_tbl.column_names:
        if col_name != "_block_key":
            result_dict[f"s_{sanitize_column_name(col_name)}"] = left_tbl.column(
                col_name
            ).take(pa.array(match_indices_s, type=pa.uint64()))

    for col_name in right_tbl.column_names:
        if col_name != "_block_key":
            result_dict[f"r_{sanitize_column_name(col_name)}"] = right_tbl.column(
                col_name
            ).take(pa.array(match_indices_r, type=pa.uint64()))

    return pa.Table.from_pydict(result_dict)


# engine.py - Make this function only gather data, no decisions


def _get_individual_column_stats(column_series_full: pd.Series) -> dict:
    """
    Calculates all required statistics for a single column.
    This function only gathers data, no decision-making.
    """
    from fmatch.preprocessing.core.column_detector import ColumnDetector

    stats = {
        "name": getattr(column_series_full, "name", "Unnamed"),
        "has_data": False,
        "fill_rate": 0.0,
        "distinct_ratio": 0.0,
        "avg_length": 0.0,
        "null_ratio": 0.0,
        "unique_count": 0,
        "total_count": 0,
        "inferred_type": "unknown",
        "id_score": 0,
        "id_confidence": 0.0,
        "detected_system": None,
        "is_primary_id": False,
        # Pattern detection flags
        "has_email_pattern": False,
        "has_phone_pattern": False,
        "has_url_pattern": False,
        "has_date_pattern": False,
        "has_numeric_pattern": False,
        # Quality metrics
        "entropy": 0.0,
        "gini_coefficient": 0.0,
    }

    if column_series_full is None or column_series_full.empty:
        return stats

    total_count = len(column_series_full)
    stats["total_count"] = total_count

    series_non_na = column_series_full.dropna()
    if series_non_na.empty:
        stats["null_ratio"] = 1.0
        return stats

    non_na_count = len(series_non_na)
    unique_count = series_non_na.nunique()

    # Basic statistics
    stats["has_data"] = True
    stats["fill_rate"] = non_na_count / total_count
    stats["null_ratio"] = 1 - stats["fill_rate"]
    stats["unique_count"] = unique_count
    stats["distinct_ratio"] = unique_count / non_na_count if non_na_count > 0 else 0

    # String-specific stats
    if column_series_full.dtype == "object" or pd.api.types.is_string_dtype(
        column_series_full
    ):
        str_series = series_non_na.astype(str)
        stats["avg_length"] = str_series.str.len().mean()

        # Pattern detection (sample for performance)
        sample_size = min(1000, len(str_series))
        str_sample = (
            str_series.sample(n=sample_size, random_state=42)
            if len(str_series) > sample_size
            else str_series
        )

        stats["has_email_pattern"] = (
            str_sample.str.contains(r"[^@]+@[^@]+\.[^@]+", na=False).sum()
            / len(str_sample)
        ) > 0.5
        stats["has_phone_pattern"] = (
            str_sample.str.contains(r"[\d\-\(\)\+\s]{10,}", na=False).sum()
            / len(str_sample)
        ) > 0.3
        stats["has_url_pattern"] = (
            str_sample.str.contains(r"https?://", na=False).sum() / len(str_sample)
        ) > 0.3
        stats["has_date_pattern"] = (
            str_sample.str.contains(r"\d{4}-\d{2}-\d{2}", na=False).sum()
            / len(str_sample)
        ) > 0.3

    # Numeric pattern detection
    try:
        pd.to_numeric(series_non_na, errors="raise")
        stats["has_numeric_pattern"] = True
    except:
        stats["has_numeric_pattern"] = False

    # Semantic type detection
    stats["inferred_type"] = (
        _get_semantic_type(column_series_full.name, series_non_na) or "unknown"
    )

    # Advanced statistics
    if unique_count > 1:
        value_counts = series_non_na.value_counts()
        # Shannon entropy
        probs = value_counts / non_na_count
        stats["entropy"] = -float((probs * np.log2(probs)).sum())
        # Gini coefficient
        stats["gini_coefficient"] = _calculate_gini(value_counts)

    # ID detection using ColumnDetector
    try:
        # Create a temporary DataFrame for detection
        temp_df = pd.DataFrame({column_series_full.name: column_series_full})

        # Get detection result
        detection_result = ColumnDetector.detect_id_column(temp_df)

        if detection_result and detection_result.column_name == column_series_full.name:
            stats["id_score"] = detection_result.score
            stats["id_confidence"] = detection_result.score / 100.0
            stats["detected_system"] = detection_result.detected_system
            stats["is_primary_id"] = detection_result.score >= 70

    except Exception as e:
        log.debug(f"ColumnDetector failed for {stats['name']}: {e}")

    return stats


def auto_select_id_column(
    df: pd.DataFrame, file_source: Optional[str] = None, enable_telemetry: bool = True
) -> Optional[str]:
    """
    Enhanced auto-selection with enterprise awareness and telemetry.
    Uses the ColumnDetector to find the best ID column based on a scoring model.

    Args:
        df: DataFrame to analyze
        file_source: Optional source file path for telemetry
        enable_telemetry: Whether to record telemetry data

    Returns:
        The name of the best ID column, or None if no suitable column found
    """
    try:
        detector_result = ColumnDetector.detect_id_column(df)
    except Exception as e:
        log.error(f"ColumnDetector failed: {e}", exc_info=True)
        return None

    if detector_result is None:
        log.warning("No suitable ID column detected by ColumnDetector")
        return None

    # Generate full report for telemetry
    report = None
    if enable_telemetry and _TELEMETRY:
        try:
            report = ColumnDetector.generate_id_selection_report(df)
        except Exception as e:
            log.debug(f"Failed to generate ID selection report: {e}")

    # Log decision for transparency
    log.info(
        f"Auto-selected ID column: '{detector_result.column_name}' "
        f"(score: {detector_result.score:.1f}, system: {detector_result.detected_system})"
    )

    # CRITICAL: Add uniqueness floor warning
    if detector_result.uniqueness_ratio < ColumnDetector.STRICT_UNIQUENESS_FLOOR:
        log.warning(
            f"WARNING: Auto-selected column '{detector_result.column_name}' has low uniqueness "
            f"({detector_result.uniqueness_ratio:.1%} < {ColumnDetector.STRICT_UNIQUENESS_FLOOR:.0%}). "
            f"This may not be a good ID column. Score: {detector_result.score}"
        )

    # Warn on low confidence
    if detector_result.score < 10:
        log.warning(
            f"Low confidence ID selection. Issues: {', '.join(detector_result.issues)}"
        )

    # Record telemetry if enabled
    if enable_telemetry and _TELEMETRY and file_source:
        try:
            # Generate anonymized dataset hash
            dataset_hash = _generate_dataset_hash(df)

            _TELEMETRY.record_id_detection(
                dataset_hash=dataset_hash,
                detection_result={
                    "selected": detector_result.column_name,
                    "confidence": report.get("confidence", "unknown")
                    if report
                    else "unknown",
                    "score": detector_result.score,
                    "system": detector_result.detected_system,
                    "all_candidates": report.get("all_candidates", [])
                    if report
                    else [],
                },
                user_override=None,
                file_source=file_source,
            )
        except Exception as e:
            log.debug(f"Failed to record telemetry: {e}")

    return detector_result.column_name


def select_id_column_with_override(
    df: pd.DataFrame,
    user_override: Optional[str] = None,
    file_source: Optional[str] = None,
    enable_telemetry: bool = True,
) -> str:
    """
    Select ID column with optional user override and telemetry.

    Args:
        df: DataFrame to analyze
        user_override: User-specified ID column (takes precedence)
        file_source: Optional source file path for telemetry
        enable_telemetry: Whether to record telemetry data

    Returns:
        The ID column name (either user-specified or auto-detected)

    Raises:
        ValueError: If no ID column could be determined
    """
    # Validate user override
    if user_override:
        if user_override not in df.columns:
            log.error(
                f"User-specified ID column '{user_override}' not found in DataFrame"
            )
            raise ValueError(f"Column '{user_override}' not found")

        log.info(f"Using user-specified ID column: '{user_override}'")

        # Still run auto-detection for telemetry/learning
        if enable_telemetry and _TELEMETRY:
            auto_result = auto_select_id_column(df, file_source, enable_telemetry=False)

            # Record the override
            if file_source:
                try:
                    report = ColumnDetector.generate_id_selection_report(df)
                    dataset_hash = _generate_dataset_hash(df)

                    _TELEMETRY.record_id_detection(
                        dataset_hash=dataset_hash,
                        detection_result={
                            "selected": auto_result,
                            "confidence": report.get("confidence", "unknown"),
                            "score": report.get("all_candidates", [{}])[0].get(
                                "score", 0
                            )
                            if report.get("all_candidates")
                            else 0,
                            "system": report.get("all_candidates", [{}])[0].get(
                                "system"
                            )
                            if report.get("all_candidates")
                            else None,
                            "all_candidates": report.get("all_candidates", []),
                        },
                        user_override=user_override,
                        file_source=file_source,
                    )
                except Exception as e:
                    log.debug(f"Failed to record override telemetry: {e}")

        return user_override

    # No override, use auto-detection
    detected = auto_select_id_column(df, file_source, enable_telemetry)
    if detected:
        return detected

    # Fallback
    log.error("No ID column could be detected and no override provided")
    raise ValueError("Unable to determine ID column")


def generate_id_selection_report(df: pd.DataFrame) -> Dict[str, Any]:
    """
    Generate a detailed report on ID column selection for transparency.
    Useful for GUI to show why a column was selected.
    """
    try:
        return ColumnDetector.generate_id_selection_report(df)
    except Exception as e:
        log.error(f"Error generating ID selection report: {e}", exc_info=True)
        return {"error": str(e), "selected": None}


def _generate_dataset_hash(df: pd.DataFrame) -> str:
    """Generate anonymized hash of dataset for telemetry"""
    # Create a fingerprint from shape and column names
    fingerprint = f"{df.shape[0]}_{df.shape[1]}_{'_'.join(sorted(df.columns))}"
    return hashlib.sha256(fingerprint.encode()).hexdigest()[:16]


# --------------------------------------------------------------------------- #
# GUI-compat exports Ã¢â‚¬â€œ single source of truth for every layer
# --------------------------------------------------------------------------- #

# (re)load all rulesets once at import-time
# Rulesets, their names, and the default will be initialized
# within the _INITIALISED block to ensure it happens only once.
AVAILABLE_RULESET_NAMES: list[str] = []  # Initialize as empty
DEFAULT_RULESET_NAME: str = "none"  # Initialize with a safe default


def match_manual(
    source_df: pd.DataFrame,
    reference_df: pd.DataFrame,
    mappings: list,
    anchors: Optional[Dict[str, Any]] = None,
    blocks: Optional[Dict[str, Any]] = None,
    threshold: float = 0.6,
    best_only: bool = True,
    limit: Optional[int] = None,
    strip_b2c: bool = True,
    logger: Optional[logging.Logger] = None,
) -> tuple[pd.DataFrame, Dict[str, Any]]:
    """Execute manual matching via the core engine."""
    from fmatch.core.manual_match_engine import ManualMatchEngine

    engine = ManualMatchEngine(logger=logger)
    return engine.execute(
        source_df=source_df,
        reference_df=reference_df,
        mappings=mappings,
        anchors=anchors,
        blocks=blocks,
        threshold=threshold,
        best_only=best_only,
        limit=limit,
        strip_b2c=strip_b2c,
    )


# --- Define __all__ for explicit exports ---
__all__ = [
    "preprocess",
    "make_blocking_column",
    "soundex",
    "worker_match",
    "worker_duplicate",
    "process_dataframe",
    "validate_dataframe",
    "detect_encoding",
    "reload_rulesets",
    "apply_rules",
    "analyze_blocking_candidates",
    "auto_block_decision",
    "decide_processing_mode",
    "load_ruleset_dir",
    "MatchConfig",  # Exporting the original
    "DedupeConfig",  # Exporting the original
    "MatchJobConfig",  # << EXPORTING THE ALIAS
    "DedupeJobConfig",  # << EXPORTING THE ALIAS
    "EnhancedMatchConfig",
    "EnhancedDedupeConfig",
    "worker_match_enhanced",
    "worker_dedupe_enhanced",
    "match_manual",
    "_init_score_cache_db",
    "ALGORITHM_MAP",
    "ALGORITHM_DISPLAY_NAMES",
    "DEFAULT_ALGORITHM_KEY",
    "DEFAULT_NORMALIZATION_RULES",
    "AVAILABLE_RULESETS",
    "AVAILABLE_RULESET_NAMES",
    "DEFAULT_RULESET_NAME",
    "_ACTIVE_BLOCK_KEY_",
    "_SCORE_CACHE_TABLE",
    "RunStats",
    "_get_individual_column_stats",
    "auto_select_id_column",  # <-- CHANGED: removed underscore
    "select_id_column_with_override",  # <-- NEW: for GUI integration
    "generate_id_selection_report",  # <-- Keep this (you already have it)
    "build_match_config",  # <-- NEW: important public API
    "build_dedupe_config",  # <-- NEW: important public API
    "suggest_mappings",
    "AutoMapper",
    "_get_semantic_type",
    "_check_series_regex_match",
    "RE_PATTERNS",
    "compute_numeric_ratio",
    "compute_value_entropy",
    "VARIANT_CATEGORIES",
    "VARIANT_DESCRIPTIONS",
    "_sanitize_name",
    # Add any other names you want to be publicly importable
]

if not _INITIALISED:
    reload_rulesets(
        verbose=True
    )  # expensive YAML -> dict parse, ensure verbose for initial load
    # Now that AVAILABLE_RULESETS is populated, set the dependent globals
    AVAILABLE_RULESET_NAMES[:] = list(AVAILABLE_RULESETS)
    DEFAULT_RULESET_NAME = "default" if "default" in AVAILABLE_RULESETS else "none"
    log.info("Engine initialized. Available rulesets: %s", AVAILABLE_RULESET_NAMES)
    log.info(f"Default ruleset set to: {DEFAULT_RULESET_NAME}")
    _INITIALISED = True

# ADD THIS FUNCTION TO THE VERY END of src/fmatch/engine.py


def suggest_mappings(
    source_df: pd.DataFrame,
    ref_df: pd.DataFrame,
    threshold: int = 75,
    *,
    stats_cache: ColumnStatsCache | None = None,
) -> list[dict]:
    """
    Suggests column mappings based on fuzzy name matching of column headers.
    """
    if source_df is None or ref_df is None:
        return []

    stats_cache = stats_cache or ColumnStatsCache()

    source_cols = list(source_df.columns)
    ref_cols = list(ref_df.columns)

    processed_source_cols = {
        col: stats_cache.get_processed_header(col) for col in source_cols
    }
    processed_ref_cols = {
        col: stats_cache.get_processed_header(col) for col in ref_cols
    }

    ref_lookup: dict[str, list[str]] = {}
    for col, processed in processed_ref_cols.items():
        ref_lookup.setdefault(processed, []).append(col)

    suggested_mappings = []
    used_ref_cols: set[str] = set()

    for s_col, processed_s_col in processed_source_cols.items():
        available = [
            processed_ref_cols[r_col]
            for r_col in ref_cols
            if r_col not in used_ref_cols
        ]
        if not available:
            break

        best_match = rf_process.extractOne(
            processed_s_col, available, scorer=rf_fuzz.WRatio, processor=None
        )

        if not best_match or best_match[1] < threshold:
            continue

        matched_processed = best_match[0]
        matched_ref_col = None
        for candidate in ref_lookup.get(matched_processed, []):
            if candidate not in used_ref_cols:
                matched_ref_col = candidate
                break

        if matched_ref_col is None:
            continue

        suggested_mappings.append(
            {
                "source": s_col,
                "ref": matched_ref_col,
                "reference": matched_ref_col,
                "weight": 1.0,
            }
        )
        used_ref_cols.add(matched_ref_col)

    return suggested_mappings


def optimize_mapping_algorithms(
    mappings: List[Dict[str, Any]], source_df: pd.DataFrame, ref_df: pd.DataFrame
) -> List[Dict[str, Any]]:
    """Smart algorithm selection with domain-aware fuzzy matching."""
    optimized_mappings = []

    for mapping in mappings:
        optimized_map = mapping.copy()
        source_col = mapping.get("source", "").lower()
        ref_col = mapping.get("ref", "").lower()

        # Domain detection
        domain_keywords = ["domain", "website", "url", "site", "web", ".com"]
        domain_types = {"domain", "website", "url", "profile_url"}
        type_source = (mapping.get("type_source") or "").lower()
        type_ref = (mapping.get("type_reference") or "").lower()
        is_domain = (
            type_source in domain_types
            or type_ref in domain_types
            or any(kw in source_col or kw in ref_col for kw in domain_keywords)
        )

        # Company/name detection
        company_keywords = [
            "company",
            "account",
            "organization",
            "business",
            "firm",
            "name",
        ]
        is_company = any(kw in source_col or kw in ref_col for kw in company_keywords)

        # Email detection
        email_keywords = ["email", "mail", "e-mail"]
        is_email = any(kw in source_col or kw in ref_col for kw in email_keywords)

        # ID/code detection
        id_keywords = ["id", "code", "number", "key", "identifier"]
        is_id = any(kw in source_col or kw in ref_col for kw in id_keywords)

        if is_domain:
            existing_transforms = list(optimized_map.get("transforms") or [])
            remaining_transforms = [
                t for t in existing_transforms if t not in {"to_domain", "lower"}
            ]
            optimized_map["transforms"] = ["to_domain", "lower"] + remaining_transforms
            optimized_map["preferred_algo"] = "Exact"
            optimized_map["weight"] = 1.0
            optimized_map["preprocessing"] = "raw"
            log.info(
                f"Domain column {source_col} -> {ref_col}: using Exact domain matching"
            )
        elif is_company:
            optimized_map["preferred_algo"] = (
                "WRatio"  # CRITICAL: Use WRatio for names!
            )
            log.info(f"Company column {source_col} -> {ref_col}: using WRatio")
        elif is_email:
            optimized_map["preferred_algo"] = "Exact"  # Exact for emails
            log.info(f"Email column {source_col} -> {ref_col}: using Exact")
        elif is_id:
            optimized_map["preferred_algo"] = "Exact"  # Exact for IDs
            log.info(f"ID column {source_col} -> {ref_col}: using Exact")
        else:
            optimized_map["preferred_algo"] = "WRatio"  # Default to WRatio

        optimized_mappings.append(optimized_map)

    return optimized_mappings


def _process_col_name_for_fuzzy(name: str) -> str:
    """Helper to clean a column name for fuzzy name matching."""
    processed_name = str(name).lower()
    processed_name = processed_name.replace(" / ", "").replace("/", "")
    processed_name = processed_name.replace(" & ", "").replace("&", "")
    processed_name = processed_name.replace("_", "").replace(" ", "")
    processed_name = processed_name.replace("-", "")
    return processed_name


def auto_block_decision(*args, **kwargs):
    """
    Shim for backward compatibility. The desktop app calls this,
    and it will now be safely routed to the new, more advanced
    strategy determination logic.
    """
    log.debug(
        "auto_block_decision shim called, redirecting to determine_optimal_blocking_strategy."
    )
    return determine_optimal_blocking_strategy(*args, **kwargs)


# Ã¢â€â‚¬Ã¢â€â‚¬ Batched metrics utilities Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬


def write_metrics_batched(logger, rows, batch=100, progress_cb=None):
    """Write CSV metrics in reasonably-sized batches to avoid UI freeze."""
    total = len(rows)
    for idx in range(0, total, batch):
        slice_ = rows[idx : idx + batch]
        for r in slice_:
            logger.write_row(r)
        if progress_cb:
            progress_cb(
                {
                    "phase": "metrics",
                    "done": min(idx + batch, total),
                    "total": total,
                    "message": f"Writing metrics: {min(idx + batch, total)}/{total}",
                }
            )
    logger.flush()


def ui_progress_bridge(gui_queue):
    """Return a callable suitable for engine progress_cb -> GUI queue."""

    def _callback(update: dict):
        if gui_queue:
            try:
                gui_queue.put_nowait(update)  # non-blocking
            except:
                pass  # Queue full, skip update

    return _callback


# Ã¢â€â‚¬Ã¢â€â‚¬ Debug mode utilities Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬Ã¢â€â‚¬

import os
import json
from pathlib import Path

log = logging.getLogger(__name__)

DEBUG_MODE = False
DEBUG_DIR = None
DEBUG_SAMPLE = 10
_debug_block_counter = 0


def enable_debug(out_dir: Path, sample_blocks: int = 10) -> None:
    """Enable debug mode for detailed block logging."""
    global DEBUG_MODE, DEBUG_DIR, DEBUG_SAMPLE, _debug_block_counter
    DEBUG_MODE = True
    DEBUG_DIR = out_dir
    DEBUG_SAMPLE = sample_blocks
    _debug_block_counter = 0
    DEBUG_DIR.mkdir(parents=True, exist_ok=True)
    log.info(f"Debug mode enabled. Will log first {sample_blocks} blocks to {out_dir}")


def dump_debug_block(block_key, src_block, ref_block, matches, metric):
    """Dump detailed block information for debugging."""
    global _debug_block_counter
    if not DEBUG_MODE or _debug_block_counter >= DEBUG_SAMPLE:
        return

    _debug_block_counter += 1
    debug_file = (
        DEBUG_DIR / f"block_{str(block_key).replace('/','_')}_{os.getpid()}.json"
    )

    debug_data = {
        "block_key": str(block_key),
        "metric": metric,
        "src_shape": src_block.shape,
        "ref_shape": ref_block.shape if ref_block is not None else None,
        "matches_found": len(matches),
        "sample_src_records": src_block.head(5).to_dict("records"),
        "sample_ref_records": ref_block.head(5).to_dict("records")
        if ref_block is not None
        else None,
        "sample_matches": matches[:5] if matches else [],
    }

    with open(debug_file, "w") as f:
        json.dump(debug_data, f, indent=2, default=str)


_COLUMN_SAMPLE_SIZE = 200
_SFDC_ID_RE = re.compile(r"^[A-Za-z0-9]{18}$")
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
_DOMAIN_RE = re.compile(r"^[A-Za-z0-9.-]+\.[A-Za-z]{2,}$")
_URL_RE = re.compile(r"^https?://", re.IGNORECASE)
_PHONE_RE = re.compile(r"^[+0-9][0-9\-().\s]{5,}$")

_ALLOWED_TYPE_PAIRS: set[tuple[str, str]] = {
    ("domain", "url"),
    ("url", "domain"),
    ("domain", "domain"),
    ("email", "email"),
    ("id", "id"),
    ("numeric", "numeric"),
    ("company", "company"),
    ("text", "text"),
    ("phone", "phone"),
    ("city", "city"),
    ("state", "state"),
    ("city", "state"),
    ("state", "city"),
}


class AutoMapper:
    """Automatically suggests and annotates mapping candidates."""

    def __init__(self, stats_cache: ColumnStatsCache | None = None) -> None:
        self.last_warnings: List[Dict[str, Any]] = []
        self._stats_cache = stats_cache or ColumnStatsCache()

    def suggest(
        self,
        source_df: pd.DataFrame,
        ref_df: pd.DataFrame,
        threshold: int = 75,
    ) -> list[dict]:
        initial = suggest_mappings(
            source_df, ref_df, threshold, stats_cache=self._stats_cache
        )
        enriched, warnings = calculate_mapping_weights(
            initial, source_df, ref_df, stats_cache=self._stats_cache
        )
        self.last_warnings = warnings
        return enriched


SYN_GROUPS: tuple[set[str], ...] = (
    {
        "company",
        "business",
        "businessname",
        "account",
        "organization",
        "org",
        "firm",
        "vendor",
        "client",
    },
    {"website", "url", "webaddress", "homepage", "site"},
    {"domain", "emaildomain", "domainappend"},
    {"email", "emailaddress"},
    {"phone", "telephone", "phonenumber", "mobile", "cell"},
    {"city"},
    {"state", "province"},
)

SFDC_HINTS: dict[str, set[str]] = {
    "accountname": {"company", "business", "businessname", "organization"},
    "dhcproviderwebsite": {"website", "url"},
    "domainappend": {"domain", "emaildomain"},
}

_PROFILE_URL_HOSTS: tuple[str, ...] = (
    "linkedin.com",
    "zoominfo.com",
    "crunchbase.com",
    "facebook.com",
    "twitter.com",
    "x.com",
)


def _normalize_header_for_mapping(name: str) -> str:
    norm = re.sub(r"[^a-z0-9]", "", str(name or "").lower())
    for prefix in ("dhc", "zoominfo", "zi", "provider"):
        if norm.startswith(prefix):
            norm = norm[len(prefix) :]
    for suffix in ("append", "id"):
        if norm.endswith(suffix):
            norm = norm[: -len(suffix)]
    return norm


def _header_tokens(label: str, stats_cache: ColumnStatsCache | None = None) -> set[str]:
    if stats_cache is not None:
        return stats_cache.get_header_tokens(label)
    return set(re.findall(r"[a-z]+", str(label or "").lower()))


def _syn_similarity(
    header_a: str, header_b: str, stats_cache: ColumnStatsCache | None = None
) -> float:
    tokens_a = _header_tokens(header_a, stats_cache)
    tokens_b = _header_tokens(header_b, stats_cache)
    for group in SYN_GROUPS:
        if tokens_a & group and tokens_b & group:
            return 0.9
    return 0.0


def _crm_hint_boost(
    header_a: str, header_b: str, stats_cache: ColumnStatsCache | None = None
) -> float:
    norm_a = _normalize_header_for_mapping(header_a)
    norm_b = _normalize_header_for_mapping(header_b)
    for key, synonyms in SFDC_HINTS.items():
        if key in (norm_a, norm_b):
            other_tokens = _header_tokens(
                header_b if key == norm_a else header_a, stats_cache
            )
            if other_tokens & synonyms:
                return 0.15
    return 0.0


def calculate_mapping_weights(
    mappings: list[dict],
    source_df: pd.DataFrame,
    ref_df: pd.DataFrame,
    *,
    stats_cache: ColumnStatsCache | None = None,
) -> tuple[list[dict], list[Dict[str, Any]]]:
    if not mappings:
        return [], []

    stats_cache = stats_cache or ColumnStatsCache()

    warnings: list[Dict[str, Any]] = []
    updated: list[dict] = []

    source_series_cache: dict[str, pd.Series] = {}
    ref_series_cache: dict[str, pd.Series] = {}
    source_df_id = id(source_df)
    ref_df_id = id(ref_df)

    for mapping in mappings:
        s_col = (
            mapping.get("source")
            or mapping.get("source_column")
            or mapping.get("source_field")
            or mapping.get("from")
        )
        r_col = (
            mapping.get("reference")
            or mapping.get("ref")
            or mapping.get("reference_column")
            or mapping.get("reference_field")
            or mapping.get("target")
        )
        if (
            not s_col
            or not r_col
            or s_col not in source_df.columns
            or r_col not in ref_df.columns
        ):
            continue

        if s_col not in source_series_cache:
            source_series_cache[s_col] = stats_cache.remember_series(
                source_df[s_col], source_df_id
            )
        src_series = source_series_cache[s_col]

        if r_col not in ref_series_cache:
            ref_series_cache[r_col] = stats_cache.remember_series(
                ref_df[r_col], ref_df_id
            )
        ref_series = ref_series_cache[r_col]

        name_score = rf_fuzz.WRatio(s_col, r_col) / 100.0
        name_score = min(
            1.0,
            name_score
            + _syn_similarity(s_col, r_col, stats_cache)
            + _crm_hint_boost(s_col, r_col, stats_cache),
        )

        src_type = _detect_column_type(s_col, src_series, stats_cache)
        ref_type = _detect_column_type(r_col, ref_series, stats_cache)

        transforms = list(mapping.get("transforms") or [])
        preprocessing = mapping.get("preprocessing")
        preferred_algo = mapping.get("preferred_algo")

        domainish_types = {"domain", "url", "profile_url"}
        is_domain_pair = src_type in domainish_types and ref_type in domainish_types
        if is_domain_pair:
            src_type = "domain"
            ref_type = "domain"

        if not _types_compatible(src_type, ref_type):
            warnings.append(
                {
                    "source": s_col,
                    "reference": r_col,
                    "reason": "type_mismatch",
                    "details": f"{src_type}->{ref_type}",
                }
            )
            continue

        overlap_score, overlap_has_values = _compute_overlap_score(
            src_series,
            ref_series,
            src_type,
            ref_type,
            stats_cache,
        )

        if is_domain_pair:
            transforms = ["to_domain", "lower"]
            preferred_algo = "Exact"
            preprocessing = "raw"
            if overlap_has_values and overlap_score > 0:
                overlap_score = 1.0

        is_namey = (src_type == ref_type) and (src_type in {"company", "text"})

        if {src_type, ref_type} == {"domain", "url"} and (
            not overlap_has_values or overlap_score < 0.15
        ):
            warnings.append(
                {
                    "source": s_col,
                    "reference": r_col,
                    "reason": "low_overlap_domain_url",
                    "details": f"overlap={overlap_score:.3f}",
                }
            )
            continue

        if is_namey and not is_domain_pair:
            if "normalize_company_name" not in transforms:
                transforms.append("normalize_company_name")
            if not preferred_algo or preferred_algo in {
                "WRatio",
                "Ratio",
                "JaroWinkler",
            }:
                preferred_algo = "TokenSetRatio"

        src_stats = stats_cache.get_column_stats(
            source_df, s_col, _get_individual_column_stats
        )
        ref_stats = stats_cache.get_column_stats(
            ref_df, r_col, _get_individual_column_stats
        )

        s_unique_ratio = float(src_stats.get("distinct_ratio", 0.0))
        r_unique_ratio = float(ref_stats.get("distinct_ratio", 0.0))
        uniqueness_diff = abs(s_unique_ratio - r_unique_ratio)
        uniqueness_score = max(0.0, 1.0 - (uniqueness_diff * 0.5))

        type_score = 1.0 if src_type == ref_type else 0.8

        if is_domain_pair:
            weight = 1.0
            confidence = 1.0
        elif is_namey:
            weight = (
                (name_score * 0.70) + (type_score * 0.15) + (uniqueness_score * 0.15)
            )
            confidence = min(
                1.0, (name_score * 0.35) + (overlap_score * 0.35) + (type_score * 0.2)
            )
        else:
            weight = (
                (name_score * 0.40)
                + (type_score * 0.20)
                + (overlap_score * 0.30)
                + (uniqueness_score * 0.10)
            )
            confidence = min(
                1.0, (name_score * 0.35) + (overlap_score * 0.35) + (type_score * 0.2)
            )

        weight = max(0.1, min(weight, 1.0))

        reasons = [
            {"kind": "header_similarity", "score": round(name_score, 3)},
            {"kind": "overlap", "score": round(overlap_score, 3)},
            {
                "kind": "uniqueness_gap",
                "score": round(1.0 - uniqueness_diff, 3),
            },
        ]

        role = (
            "identity"
            if (
                is_domain_pair
                or src_type in {"company", "id", "email"}
                or ref_type in {"company", "id", "email"}
            )
            else "support"
        )

        updated_mapping = mapping.copy()
        updated_mapping["reference"] = r_col
        updated_mapping["ref"] = r_col  # Backwards compatibility for older consumers
        updated_mapping["weight"] = round(weight, 2)
        updated_mapping["confidence"] = round(confidence, 3)
        updated_mapping["type_source"] = src_type
        updated_mapping["type_reference"] = ref_type
        updated_mapping["sample_overlap"] = round(overlap_score, 3)
        updated_mapping["reasons"] = reasons
        updated_mapping["role"] = role
        updated_mapping["transforms"] = transforms
        if preprocessing:
            updated_mapping["preprocessing"] = preprocessing
        if preferred_algo:
            updated_mapping["preferred_algo"] = preferred_algo

        if is_namey and (overlap_score < 0.05) and (name_score < 0.60):
            warnings.append(
                {
                    "source": s_col,
                    "reference": r_col,
                    "reason": "low_overlap_name_pair",
                    "details": f"name={name_score:.3f}, overlap={overlap_score:.3f}",
                }
            )

        updated.append(updated_mapping)

    if updated:
        identity_types = {"domain", "id", "company", "email"}
        geo_types = {"city", "state"}
        has_identity = any(
            (mapping.get("type_source") in identity_types)
            or (mapping.get("type_reference") in identity_types)
            for mapping in updated
        )
        if not has_identity:
            filtered = [
                m
                for m in updated
                if m.get("type_source") not in geo_types
                and m.get("type_reference") not in geo_types
            ]
            if filtered:
                warnings.append(
                    {
                        "source": None,
                        "reference": None,
                        "reason": "identity_missing",
                        "details": "Recommended mappings exclude geography-only pairs",
                    }
                )
                updated = filtered

        primary = [
            {"source": m.get("source"), "reference": m.get("ref")}
            for m in updated
            if m.get("role") == "identity"
        ]
        secondary = [
            {"source": m.get("source"), "reference": m.get("ref")}
            for m in updated
            if m.get("role") != "identity"
        ]
        if primary:
            warnings.append(
                {
                    "source": None,
                    "reference": None,
                    "reason": "recommended_mapping_set",
                    "details": {"primary": primary, "secondary": secondary},
                }
            )

    return updated, warnings


def _types_compatible(src_type: str, ref_type: str) -> bool:
    if src_type == "profile_url" or ref_type == "profile_url":
        return src_type == ref_type
    if src_type == ref_type:
        return True
    if (src_type, ref_type) in _ALLOWED_TYPE_PAIRS:
        return True
    if (ref_type, src_type) in _ALLOWED_TYPE_PAIRS:
        return True
    return False


def _detect_column_type(
    header: str, series: pd.Series, stats_cache: ColumnStatsCache | None = None
) -> str:
    if stats_cache is not None:
        return stats_cache.get_detected_type(
            series, header, lambda h, s: _detect_column_type(h, s, None)
        )

    header_norm = str(header or "").lower()
    sample_series = series.dropna().astype(str).head(_COLUMN_SAMPLE_SIZE)
    sample_list = sample_series.tolist()

    ratios: Dict[str, float] = {
        "sfdc_id": _ratio(sample_series, _SFDC_ID_RE.match),
        "email": _ratio(sample_series, _EMAIL_RE.match),
        "url": _ratio(sample_series, _URL_RE.match),
        "domain": _ratio(sample_series, lambda v: _DOMAIN_RE.match(v) is not None),
        "phone": _ratio(sample_series, _PHONE_RE.match),
        "numeric": _ratio(sample_series, lambda v: v.replace(".", "", 1).isdigit()),
    }

    if "id" in header_norm and ratios["sfdc_id"] > 0.4:
        return "id"
    if ratios["sfdc_id"] > 0.7:
        return "id"
    if "email" in header_norm or ratios["email"] > 0.6:
        return "email"
    if "profile" in header_norm and ratios["url"] > 0.4:
        return "profile_url"
    if ratios["url"] > 0.4 and sample_list:
        sample_lower = [str(val).lower() for val in sample_list]
        if any(any(host in val for host in _PROFILE_URL_HOSTS) for val in sample_lower):
            return "profile_url"
    if "website" in header_norm or header_norm in {"web", "homepage"}:
        return "url"
    if "domain" in header_norm or (ratios["domain"] > 0.5 and ratios["url"] < 0.4):
        return "domain"
    if "url" in header_norm or ratios["url"] > 0.6:
        return "url"
    if "phone" in header_norm or ratios["phone"] > 0.6:
        return "phone"
    if ratios["numeric"] > 0.6 and "zip" not in header_norm:
        return "numeric"
    if "state" in header_norm or header_norm in {"province", "region"}:
        return "state"
    if "city" in header_norm or header_norm in {"town", "municipality"}:
        return "city"
    if any(tok in header_norm for tok in ("company", "account", "name")):
        return "company"
    return "text"


def _ratio(sample: pd.Series, predicate) -> float:
    if sample.empty:
        return 0.0
    hits = sum(1 for val in sample if predicate(str(val).strip()))
    return hits / float(len(sample)) if len(sample) else 0.0


def _normalize_for_overlap(value: str, detected_type: str) -> str:
    if value is None:
        return ""
    text = str(value).strip()
    if not text:
        return ""
    lower = text.lower()
    if detected_type in {"domain", "url", "profile_url"}:
        normalized = to_domain(text)
        return (normalized or "").strip()
    if detected_type == "email":
        return lower
    if detected_type == "id":
        return text.upper()
    if detected_type == "numeric":
        return re.sub(r"\D", "", text)
    if detected_type == "company":
        return re.sub(r"[^a-z0-9]", "", lower)
    return lower


def _normalized_overlap_unique(series: pd.Series, detected_type: str) -> np.ndarray:
    sample = series.dropna().astype(str).head(_COLUMN_SAMPLE_SIZE)
    if sample.empty:
        return np.array([], dtype=object)
    normalized = sample.map(lambda val: _normalize_for_overlap(val, detected_type))
    if normalized.empty:
        return np.array([], dtype=object)
    values = normalized[normalized != ""].to_numpy(dtype=object)
    if values.size == 0:
        return np.array([], dtype=object)
    return np.unique(values)


def _compute_overlap_score(
    src_series: pd.Series,
    ref_series: pd.Series,
    src_type: str,
    ref_type: str,
    stats_cache: ColumnStatsCache | None = None,
) -> tuple[float, bool]:
    if stats_cache is not None:
        src_unique = stats_cache.get_normalized_overlap(
            src_series, src_type, _normalized_overlap_unique
        )
        ref_unique = stats_cache.get_normalized_overlap(
            ref_series, ref_type, _normalized_overlap_unique
        )
    else:
        src_unique = _normalized_overlap_unique(src_series, src_type)
        ref_unique = _normalized_overlap_unique(ref_series, ref_type)

    if getattr(src_unique, "size", 0) == 0 or getattr(ref_unique, "size", 0) == 0:
        return 0.0, False

    intersection = np.intersect1d(src_unique, ref_unique)
    union = np.union1d(src_unique, ref_unique)
    if union.size == 0:
        return 0.0, False
    return float(len(intersection) / len(union)), True


def batch_cleanse_domains_optimized(
    df: pd.DataFrame, domain_columns: List[str]
) -> pd.DataFrame:
    """
    OPTIMIZED: Batch process with normalized caching.
    """
    global _DOMAIN_CLEANSE_CACHE, _DOMAIN_NORMALIZED_CACHE

    # Collect unique domains across all columns
    all_domains = set()
    for col in domain_columns:
        if col in df.columns:
            all_domains.update(df[col].dropna().astype(str).unique())

    # Process new domains
    new_domains = all_domains - set(_DOMAIN_CLEANSE_CACHE.keys())
    if new_domains:
        log.info(f"Cleansing {len(new_domains)} new domains")
        for domain in new_domains:
            # Normalize once
            normalized = str(domain).lower().strip()
            if normalized not in _DOMAIN_NORMALIZED_CACHE:
                _DOMAIN_NORMALIZED_CACHE[normalized] = cleanse_single_domain(normalized)
            _DOMAIN_CLEANSE_CACHE[domain] = _DOMAIN_NORMALIZED_CACHE[normalized]

    # Apply cached cleanses
    for col in domain_columns:
        if col in df.columns:
            df[f"_clean_{col}"] = df[col].map(_DOMAIN_CLEANSE_CACHE)

    return df


def coalesce_tiny_blocks(all_blocks: list[dict], min_ops: int = 10000) -> list[dict]:
    """
    OPTIONAL: Combine tiny blocks to reduce scheduler overhead.
    """
    coalesced = []
    batch = []
    batch_ops = 0

    for block in sorted(all_blocks, key=lambda x: x["metric"]):
        if block["metric"] >= min_ops:
            # Large enough to process alone
            if batch:
                coalesced.append(
                    {
                        "key": f"batch_{len(coalesced)}",
                        "blocks": batch,
                        "metric": batch_ops,
                        "is_batch": True,
                    }
                )
                batch = []
                batch_ops = 0
            coalesced.append(block)
        else:
            # Add to batch
            batch.append(block)
            batch_ops += block["metric"]

            if batch_ops >= min_ops:
                coalesced.append(
                    {
                        "key": f"batch_{len(coalesced)}",
                        "blocks": batch,
                        "metric": batch_ops,
                        "is_batch": True,
                    }
                )
                batch = []
                batch_ops = 0

    # Add remaining batch
    if batch:
        coalesced.append(
            {
                "key": f"batch_{len(coalesced)}",
                "blocks": batch,
                "metric": batch_ops,
                "is_batch": True,
            }
        )

    log.info(f"Block coalescing: {len(all_blocks)} blocks -> {len(coalesced)} tasks")
    return coalesced
