"""
Worker for v2 API jobs that reads data from storage (not Redis blobs)
"""
print("[WORKER_V2 DEBUG] Starting imports...")

import json
import logging
import time
import io
import os
import asyncio
from typing import Dict, Any, Optional, List, Tuple

print("[WORKER_V2 DEBUG] Basic imports done, importing redis...")
import redis
print("[WORKER_V2 DEBUG] Redis imported, importing pandas...")
import pandas as pd
print("[WORKER_V2 DEBUG] Pandas imported, importing identity_transforms...")
from .preprocessing.identity_transforms import run_identity_transforms
print("[WORKER_V2 DEBUG] identity_transforms imported, importing identity_profiles...")
from .preprocessing.identity_profiles import detect_data_quality, select_profile
print("[WORKER_V2 DEBUG] identity_profiles imported, importing utils_normalize...")
from fmatch.utils_normalize import strip_formerly_known_as
print("[WORKER_V2 DEBUG] All imports complete!")

logger = logging.getLogger(__name__)
log = logger


def _normalize_reason_tokens(raw: str) -> str:
    tokens: List[str] = []
    for token in str(raw or "").split("|"):
        clean = token.strip().upper()
        if clean and clean not in tokens:
            tokens.append(clean)
    if "GEO" in tokens and not any(t.startswith("NAME") for t in tokens):
        geo_index = tokens.index("GEO")
        tokens.insert(max(geo_index, 0), "NAME*")
    return "|".join(tokens)


_TRUTHY_FLAG_VALUES = {"1", "true", "yes", "y", "on"}
_FALSEY_FLAG_VALUES = {"0", "false", "no", "n", "off"}


def _apply_match_policy(
    df: pd.DataFrame, *, threshold: float = 75, status_warnings: Optional[List[str]] = None
) -> pd.DataFrame:
    """Apply match-status guardrails using score, reasons, and policy flags."""
    status_warnings = status_warnings if status_warnings is not None else []
    env = os.getenv("ENV", "development").lower()

    if "reasons" not in df.columns:
        if env != "production":
            raise RuntimeError("Match evidence missing.")
        warning = "Evidence missing; downgraded to Possible Match."
        if warning not in status_warnings:
            status_warnings.append(warning)
        df["match_status"] = "Possible Match"
        return df

    reasons = df["reasons"].fillna("").astype(str)
    if reasons.eq("").all():
        if env != "production":
            raise RuntimeError("Match evidence missing.")
        warning = "Evidence missing; downgraded to Possible Match."
        if warning not in status_warnings:
            status_warnings.append(warning)
        df["match_status"] = "Possible Match"
        df["reasons"] = reasons.apply(_normalize_reason_tokens)
        return df

    df["reasons"] = reasons.apply(_normalize_reason_tokens)
    has_missing_domain = (
        df.get("flags", pd.Series([""] * len(df)))
        .fillna("")
        .astype(str)
        .str.contains("MISSING_DOMAIN_DATA")
    )

    def _derive_status(row: pd.Series) -> str:
        raw_score = row.get("match_score", row.get("score", 0))
        try:
            score = float(raw_score)
        except (TypeError, ValueError):
            score = 0.0
        reason_tokens = str(row.get("reasons") or "").upper().split("|")
        has_domain = "DOM" in reason_tokens or "FAM" in reason_tokens
        fallback = bool(row.get("fallback_marker"))

        if row.get("flags") and "MISSING_DOMAIN_DATA" in str(row.get("flags")):
            return "Possible Match"
        if score >= threshold and has_domain and not fallback:
            return "Account Match"
        if fallback:
            return "Possible Match" if score >= (threshold - 10) else "No Match"
        if score >= (threshold - 10):
            return "Possible Match"
        if score >= threshold and "NAME90" in reason_tokens and not has_domain:
            return "Possible Match"
        return "No Match"

    df["match_status"] = df.apply(_derive_status, axis=1)
    if has_missing_domain.any():
        warning = "Domain evidence missing; downgraded to Possible."
        if warning not in status_warnings:
            status_warnings.append(warning)
    return df


def _coerce_bool(value: Any, default: Optional[bool] = None) -> Optional[bool]:
    if isinstance(value, bool):
        return value
    if value is None:
        return default
    if isinstance(value, (int, float)):
        return bool(value)
    if isinstance(value, str):
        lowered = value.strip().lower()
        if lowered in _TRUTHY_FLAG_VALUES:
            return True
        if lowered in _FALSEY_FLAG_VALUES:
            return False
    return default


def _normalize_profile_override(value: Any) -> Optional[str]:
    if value is None:
        return None
    if isinstance(value, str):
        normalized = value.strip().lower()
        if not normalized or normalized == "auto":
            return None
        return normalized
    return None


def _estimate_domain_coverage(df: pd.DataFrame) -> Optional[float]:
    """Return the fraction of rows that contain a domain-like column."""
    try:
        domain_cols = [col for col in df.columns if "domain" in str(col).lower()]
        if not domain_cols:
            return None
        coverage = df[domain_cols[0]].notna().mean()
        if pd.isna(coverage):
            return None
        return float(coverage)
    except Exception:
        return None


def _build_summary_snapshot(
    source_rows: int,
    reference_rows: Optional[int],
    domain_coverage: Optional[float],
    blocking_info: Optional[Dict[str, Any]],
    progress_fraction: Optional[float],
    config: Dict[str, Any],
) -> Dict[str, Any]:
    """Seed a lightweight summary payload for the Sheets progress dialog."""
    blocking_info = blocking_info or {}
    summary: Dict[str, Any] = {}

    if source_rows:
        summary["source_rows"] = int(source_rows)
        summary["rows_total"] = int(source_rows)

    if reference_rows is not None:
        summary["reference_rows"] = int(reference_rows)
        if source_rows:
            summary["raw_comparisons"] = int(source_rows * reference_rows)

    if domain_coverage is not None:
        summary["domain_coverage"] = float(domain_coverage)

    threshold = config.get("threshold") or config.get("match_threshold")
    if threshold is not None:
        try:
            summary["threshold"] = float(threshold)
        except (TypeError, ValueError):
            pass

    est_pairs = blocking_info.get("pairs_after") or blocking_info.get("estimated_pairs")
    if est_pairs is not None:
        try:
            summary["estimated_comparisons"] = int(float(est_pairs))
        except (TypeError, ValueError):
            pass

    avg_candidates = blocking_info.get("avg_candidates") or blocking_info.get(
        "candidates_per_row"
    )
    if avg_candidates is not None:
        try:
            summary["avg_candidates"] = float(avg_candidates)
        except (TypeError, ValueError):
            pass

    progress_fraction = progress_fraction or 0.0
    try:
        progress_fraction = max(0.0, min(float(progress_fraction), 1.0))
    except (TypeError, ValueError):
        progress_fraction = 0.0
    if source_rows and progress_fraction > 0:
        summary["rows_processed"] = int(progress_fraction * source_rows)

    return summary


# Use centralized storage helpers
from .storage import resolve_object_path, resolve_uploads_base
from .worker import emit_progress, ProgressPhase, _set_job_meta, update_job_status_sync

from .services.manual_match_service import (
    execute as manual_match_execute,
    MappingConfig as ManualMappingConfig,
    MappingOptions as ManualMappingOptions,
)

# Import the EngineClient for real matching
try:
    from .services.engine_client import EngineClient

    log.info("[IMPORT] EngineClient imported successfully")
except ImportError as e:
    log.error(f"[IMPORT] Failed to import EngineClient: {e}")
    EngineClient = None


def _get_redis() -> redis.Redis:
    """Get Redis connection"""
    redis_url = os.getenv("REDIS_URL", "redis://127.0.0.1:6379/0")
    return redis.from_url(redis_url)


def _open_storage_stream(object_id: str):
    """
    Returns a binary file-like for the given uploads/<id>.csv
    Uses centralized storage path resolution.
    """
    full_path = resolve_object_path(object_id)
    log.info(f"Opening storage file: {full_path}")

    if not os.path.exists(full_path):
        log.error(f"File not found: {full_path}")
        # List what files are in the uploads directory for debugging
        base = resolve_uploads_base()
        if os.path.exists(base):
            files = os.listdir(base)[:10]  # First 10 files
            log.info(f"Files in {base}: {files}")

    return open(full_path, "rb")


def _read_csv_from_object(object_id: str) -> pd.DataFrame:
    """Read CSV from storage by object_id"""
    try:
        with _open_storage_stream(object_id) as f:
            data = f.read()
        # Robust CSV reading with BOM handling and bad lines skipping
        df = pd.read_csv(
            io.BytesIO(data),
            dtype=str,
            keep_default_na=False,
            na_values=[],
            encoding="utf-8",
            encoding_errors="ignore",
            on_bad_lines="skip",  # pandas >=1.3
        )
        log.info(
            f"Loaded CSV from {object_id}: {len(df)} rows, {len(df.columns)} columns"
        )
        return df
    except Exception as e:
        log.error(f"Failed to read CSV from {object_id}: {e}")
        raise


def _write_results_csv(job_id: str, df: pd.DataFrame) -> str:
    """Write results DataFrame to CSV and return object_id"""
    object_id = f"uploads/results/{job_id}.csv"
    path = resolve_object_path(object_id)
    os.makedirs(os.path.dirname(path), exist_ok=True)

    csv_df = _prepare_results_csv_frame(df)
    csv_df.to_csv(path, index=False, encoding="utf-8")
    log.info(f"Wrote results to {path}")

    # Return canonical object_id path the API expects
    return object_id


def _prepare_results_csv_frame(df: pd.DataFrame) -> pd.DataFrame:
    """
    Remove verbose fields (like explanation JSON) that bloat disk/network size
    without improving the Sheets experience.
    """
    drop_columns: List[str] = []
    approx_bytes = 0
    for column in df.columns:
        lower = str(column).lower()
        if lower == "explanation" or lower == "b2c_explanation":
            drop_columns.append(column)
            try:
                approx_bytes += int(
                    df[column]
                    .astype(str)
                    .map(lambda val: len(val.encode("utf-8", "ignore")))
                    .sum()
                )
            except Exception:
                # Best-effort size estimate; ignore conversion errors
                pass

    if not drop_columns:
        return df

    mb = approx_bytes / (1024 * 1024) if approx_bytes else 0
    log.info(
        "[results_csv] Dropping verbose columns before export: %s (approx %.1f MB)",
        drop_columns,
        mb,
    )
    return df.drop(columns=drop_columns, errors="ignore")


def _build_reference_name_maps(
    reference_df: pd.DataFrame, ref_id_col: Optional[str]
) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, bool]]:
    """Return lookup dictionaries for display names and historical flags."""
    if reference_df is None or reference_df.empty or not ref_id_col:
        return {}, {}, {}
    if ref_id_col not in reference_df.columns:
        return {}, {}, {}

    lower_to_actual = {str(col).lower(): col for col in reference_df.columns}
    priority_keys = [
        "display_name",
        "company",
        "company_name",
        "account name",
        "account_name",
        "name",
        "legal_name",
        "organization",
        "org_name",
    ]

    name_columns: List[str] = []
    for key in priority_keys:
        actual = lower_to_actual.get(key)
        if actual and actual not in name_columns:
            name_columns.append(actual)

    if not name_columns:
        for col in reference_df.columns:
            col_lower = str(col).lower()
            if (
                "name" in col_lower
                and "domain" not in col_lower
                and "score" not in col_lower
            ):
                if col not in name_columns:
                    name_columns.append(col)

    display_map: Dict[str, str] = {}
    family_display_map: Dict[str, str] = {}
    historical_lookup: Dict[str, bool] = {}

    for _, row in reference_df.iterrows():
        ref_id_val = row.get(ref_id_col)
        if pd.isna(ref_id_val):
            continue
        ref_key = str(ref_id_val).strip()
        if not ref_key:
            continue

        display_val: Optional[str] = None
        for col in name_columns:
            val = row.get(col)
            if isinstance(val, str) and val.strip():
                candidate = strip_formerly_known_as(val)
                if candidate:
                    display_val = candidate
                    break

        family_val = row.get("family_id")
        family_key: Optional[str] = None
        if family_val is not None and not pd.isna(family_val):
            family_str = str(family_val).strip()
            if family_str:
                family_key = family_str.lower()

        if display_val:
            display_map[ref_key] = display_val
            if family_key:
                family_display_map[family_key] = display_val
        elif family_key and family_key not in family_display_map:
            family_display_map.setdefault(family_key, None)

        hist_val = row.get("is_historical")
        hist_bool: Optional[bool] = None
        if isinstance(hist_val, str):
            hist_bool = hist_val.strip().lower() in {"1", "true", "yes", "y", "t"}
        elif hist_val is not None:
            hist_bool = bool(hist_val)
        if hist_bool is not None:
            historical_lookup[ref_key] = hist_bool

    family_display_map = {k: v for k, v in family_display_map.items() if v}

    return display_map, family_display_map, historical_lookup


def process_job_v2(job_id: str, job_data: Dict[str, Any] = None, **kwargs) -> None:
    """
    Process a job from the v2 API by reading data from storage (not Redis blobs).
    This is called either by RQ or inline from the API.
    """
    log.info(f"[WORKER_V2] Processing job {job_id}")

    redis_conn = _get_redis()

    MATCH_PROGRESS_BASE = 40.0
    MATCH_PROGRESS_RANGE = 50.0
    MATCH_PROGRESS_MAX = MATCH_PROGRESS_BASE + MATCH_PROGRESS_RANGE

    # Initialize job variable early to ensure it's always in scope
    job = {}
    job_key = f"job:{job_id}"
    summary_key = f"job:summary:{job_id}"

    try:
        # Load job from Redis (or use provided job_data)
        raw = redis_conn.get(job_key)

        if raw:
            job = json.loads(raw)
            log.info(
                f"Loaded job from Redis: state={job.get('state')}, mode={job.get('mode')}"
            )
        elif job_data:
            job = job_data
            log.info(f"Using provided job_data: mode={job.get('mode')}")
        else:
            raise ValueError(
                f"Job {job_id} not found in Redis and no job_data provided"
            )

        # Update job state to processing
        job["state"] = "processing"
        job["status"] = "processing"
        job["progress"] = 0.1
        job["started_at"] = time.time()
        redis_conn.setex(job_key, 86400, json.dumps(job))

        # Extract input configuration
        input_cfg = job.get("input", {})
        source_object_id = (
            input_cfg.get("object_id")
            or input_cfg.get("source_object_id")
            or job.get("source_object_id")
        )
        reference_object_id = (
            input_cfg.get("reference_object_id")
            or input_cfg.get("reference_id")
            or job.get("reference_object_id")
        )

        # Get mode and configuration
        mode = job.get("mode", "match")
        config = job.get("config", {})
        config_hash_submitted = (
            job.get("config_hash_submitted")
            or config.get("config_hash")
            or config.get("hash")
        )

        log.info(
            f"Processing {mode} job with source={source_object_id}, ref={reference_object_id}"
        )

        if not source_object_id:
            raise ValueError(f"Missing source object_id in job {job_id}")

        # Load source data from storage
        log.info(f"Loading source data from storage: {source_object_id}")
        source_df = _read_csv_from_object(source_object_id)
        source_rows = len(source_df)
        source_cols = len(source_df.columns)
        message = f"Loaded source data ({source_rows:,} rows, {source_cols} columns)"
        job["progress"] = 0.10
        job["message"] = message
        redis_conn.setex(job_key, 86400, json.dumps(job))
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.LOADING_DATA,
            10.0,
            message,
            rows=source_rows,
            columns=source_cols,
        )
        _set_job_meta(
            redis_conn,
            job_id,
            progress=0.10,
            message=message,
            rows_processed=0,
        )

        # Load reference data if match mode
        reference_df = None
        if mode in ("match", "manual") and reference_object_id:
            log.info(f"Loading reference data from storage: {reference_object_id}")
            reference_df = _read_csv_from_object(reference_object_id)
            ref_rows = len(reference_df)
            ref_cols = len(reference_df.columns)
            message = f"Loaded reference data ({ref_rows:,} rows, {ref_cols} columns)"
            job["progress"] = 0.20
            job["message"] = message
            redis_conn.setex(job_key, 86400, json.dumps(job))
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.LOADING_DATA,
                20.0,
                message,
                rows=ref_rows,
                columns=ref_cols,
            )
            _set_job_meta(
                redis_conn,
                job_id,
                progress=0.20,
                message=message,
                rows_processed=0,
            )
        elif mode in ("match", "manual"):
            log.warning(f"No reference object_id for {mode} job {job_id}")
            message = (
                "Reference dataset not provided; continuing with source-only processing"
            )
            job["progress"] = 0.20
            job["message"] = message
            redis_conn.setex(job_key, 86400, json.dumps(job))
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.LOADING_DATA,
                20.0,
                message,
            )
            _set_job_meta(
                redis_conn,
                job_id,
                progress=0.20,
                message=message,
                rows_processed=0,
            )

        summary_config = config if isinstance(config, dict) else {}
        summary_blocking = summary_config.get("blocking_strategy") or {}
        domain_coverage_estimate = _estimate_domain_coverage(source_df)
        summary_snapshot = _build_summary_snapshot(
            source_rows=source_rows,
            reference_rows=len(reference_df) if reference_df is not None else None,
            domain_coverage=domain_coverage_estimate,
            blocking_info=summary_blocking,
            progress_fraction=job.get("progress"),
            config=summary_config,
        )
        summary_template = dict(summary_snapshot) if summary_snapshot else {}
        if summary_snapshot:
            job["summary"] = summary_snapshot
            job.setdefault("metrics", {})
            job["metrics"]["summary"] = summary_snapshot
            redis_conn.setex(summary_key, 86400, json.dumps(summary_snapshot))
            log.info(
                "[WORKER_V2] Seeded summary snapshot for job %s: %s",
                job_id,
                summary_snapshot,
            )

        flag_scopes: List[Dict[str, Any]] = []
        if isinstance(config, dict):
            flag_scopes.append(config)
            cfg_options = config.get("options")
            if isinstance(cfg_options, dict):
                flag_scopes.append(cfg_options)
            cfg_intelligence = config.get("intelligence")
            if isinstance(cfg_intelligence, dict):
                flag_scopes.append(cfg_intelligence)
                cfg_intel_options = cfg_intelligence.get("options")
                if isinstance(cfg_intel_options, dict):
                    flag_scopes.append(cfg_intel_options)
        if isinstance(job, dict):
            job_options = job.get("options")
            if isinstance(job_options, dict):
                flag_scopes.append(job_options)
            job_intelligence = job.get("intelligence")
            if isinstance(job_intelligence, dict):
                flag_scopes.append(job_intelligence)
                job_intel_options = job_intelligence.get("options")
                if isinstance(job_intel_options, dict):
                    flag_scopes.append(job_intel_options)

        flag_value = None
        for scope in flag_scopes:
            if isinstance(scope, dict) and "enable_identity_transforms" in scope:
                flag_value = scope.get("enable_identity_transforms")
                break

        enable_identity_transforms = _coerce_bool(flag_value, default=None)
        if enable_identity_transforms is None:
            env_default = os.getenv("FM_IDENTITY_TRANSFORMS_DEFAULT")
            enable_identity_transforms = _coerce_bool(env_default, default=False)

        # Check for selective source/reference transform settings
        transforms_config = config.get("transforms", {})
        if isinstance(transforms_config, dict):
            flag_scopes.append(transforms_config)
        enable_source_transforms = (
            transforms_config.get("enable_source", True)
            if enable_identity_transforms
            else False
        )
        enable_ref_transforms = (
            transforms_config.get("enable_reference", False)
            if enable_identity_transforms
            else False
        )

        # Check if this is a preview request (skip expensive transforms for speed)
        is_preview = config.get("preview", False) or config.get("limit") == 50

        identity_profiles_info: Dict[str, Any] = {}
        identity_metrics_reference: Dict[str, Any] = {}

        preprocess_message = (
            f"Preprocessing source data ({source_rows:,} rows) for identity signals"
        )
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.PREPROCESSING,
            25.0,
            preprocess_message,
            rows=source_rows,
        )
        _set_job_meta(
            redis_conn,
            job_id,
            progress=0.25,
            message=preprocess_message,
            rows_processed=0,
        )

        if is_preview:
            # Preview mode: Skip actual transforms, return estimates only
            log.info("[IDENTITY] Skipping transforms for preview (speed optimization)")
            source_row_count = len(source_df)
            identity_metrics_source = {
                "transforms": {
                    "enabled": bool(enable_source_transforms),
                    "dataset": "source",
                    "rows": source_row_count,
                    "estimated_credits": source_row_count
                    if enable_source_transforms
                    else 0,
                    "message": "Preview mode - transforms will run on full match",
                    "identity_ready_rows": 0,
                    "identity_ready_pct": 0.0,
                    "identity_missing_rows": source_row_count,
                    "identity_missing_company_rows": 0,
                    "identity_no_domain_rows": 0,
                    "domain_backfilled": 0,
                    "company_backfilled": 0,
                }
            }
            identity_profiles_info["source"] = {"profile": None, "preview": True}
            if reference_df is not None:
                ref_row_count = len(reference_df)
                identity_metrics_reference = {
                    "transforms": {
                        "enabled": bool(enable_ref_transforms),
                        "dataset": "reference",
                        "rows": ref_row_count,
                        "estimated_credits": ref_row_count
                        if enable_ref_transforms
                        else 0,
                        "message": "Preview mode - transforms will run on full match",
                        "identity_ready_rows": 0,
                        "identity_ready_pct": 0.0,
                        "identity_missing_rows": ref_row_count,
                        "identity_missing_company_rows": 0,
                        "identity_no_domain_rows": 0,
                        "domain_backfilled": 0,
                        "company_backfilled": 0,
                    }
                }
                identity_profiles_info["reference"] = {"profile": None, "preview": True}
        else:
            profile_override = _normalize_profile_override(
                transforms_config.get("profile")
            )
            dns_override = _coerce_bool(
                transforms_config.get("enable_dns_validation"), default=None
            )

            source_transform_options: Optional[Dict[str, Any]] = None
            source_signals = None
            if enable_source_transforms:
                source_signals = detect_data_quality(source_df)
                selected_profile = select_profile(
                    source_signals, override=profile_override
                )
                source_transform_options = {
                    "profile": selected_profile.name,
                    "transforms_enabled": selected_profile.transforms_enabled,
                    "company_to_domain": selected_profile.company_to_domain_options(),
                }
                circuit_opts = dict(selected_profile.circuit_breaker_options() or {})
                if dns_override is not None:
                    dns_forced = bool(dns_override)
                    source_transform_options["company_to_domain"]["use_dns"] = (
                        dns_forced
                    )
                    if not dns_forced:
                        source_transform_options["company_to_domain"][
                            "max_candidates"
                        ] = min(
                            source_transform_options["company_to_domain"].get(
                                "max_candidates", 5
                            ),
                            2,
                        )
                effective_dns = bool(
                    source_transform_options["company_to_domain"].get("use_dns", False)
                )
                if effective_dns and not circuit_opts:
                    circuit_opts = {
                        "threshold": selected_profile.circuit_threshold,
                        "sample_size": selected_profile.circuit_sample_size,
                    }
                if circuit_opts:
                    source_transform_options["circuit_breaker"] = circuit_opts
                identity_profiles_info["source"] = {
                    "profile": selected_profile.name,
                    "signals": source_signals.__dict__,
                    "override": profile_override,
                }
                log.info(
                    "[IDENTITY] Selected profile '%s' for source (signals=%s)",
                    selected_profile.name,
                    source_signals.__dict__,
                )
                enable_source_transforms = (
                    enable_source_transforms and selected_profile.transforms_enabled
                )
                if not selected_profile.transforms_enabled:
                    identity_profiles_info["source"]["disabled_by_profile"] = True
            else:
                identity_profiles_info["source"] = {"profile": None, "disabled": True}

            identity_metrics_source = run_identity_transforms(
                source_df,
                dataset="source",
                enabled=bool(enable_source_transforms),
                options=source_transform_options,
            )
            if source_signals is not None:
                identity_metrics_source.setdefault(
                    "quality_signals", source_signals.__dict__
                )

            if reference_df is not None:
                reference_transform_options: Optional[Dict[str, Any]] = None
                reference_signals = None
                if enable_ref_transforms:
                    reference_signals = detect_data_quality(reference_df)
                    reference_profile = select_profile(
                        reference_signals, override=profile_override
                    )
                    reference_transform_options = {
                        "profile": reference_profile.name,
                        "transforms_enabled": reference_profile.transforms_enabled,
                        "company_to_domain": reference_profile.company_to_domain_options(),
                    }
                    circuit_opts = dict(
                        reference_profile.circuit_breaker_options() or {}
                    )
                    if dns_override is not None:
                        dns_forced = bool(dns_override)
                        reference_transform_options["company_to_domain"]["use_dns"] = (
                            dns_forced
                        )
                        if not dns_forced:
                            reference_transform_options["company_to_domain"][
                                "max_candidates"
                            ] = min(
                                reference_transform_options["company_to_domain"].get(
                                    "max_candidates", 5
                                ),
                                2,
                            )
                    effective_dns_ref = bool(
                        reference_transform_options["company_to_domain"].get(
                            "use_dns", False
                        )
                    )
                    if effective_dns_ref and not circuit_opts:
                        circuit_opts = {
                            "threshold": reference_profile.circuit_threshold,
                            "sample_size": reference_profile.circuit_sample_size,
                        }
                    if circuit_opts:
                        reference_transform_options["circuit_breaker"] = circuit_opts
                    identity_profiles_info["reference"] = {
                        "profile": reference_profile.name,
                        "signals": reference_signals.__dict__,
                        "override": profile_override,
                    }
                    log.info(
                        "[IDENTITY] Selected profile '%s' for reference (signals=%s)",
                        reference_profile.name,
                        reference_signals.__dict__,
                    )
                    enable_ref_transforms = (
                        enable_ref_transforms and reference_profile.transforms_enabled
                    )
                    if not reference_profile.transforms_enabled:
                        identity_profiles_info["reference"]["disabled_by_profile"] = (
                            True
                        )
                    identity_metrics_reference = run_identity_transforms(
                        reference_df,
                        dataset="reference",
                        enabled=bool(enable_ref_transforms),
                        options=reference_transform_options,
                    )
                    if reference_signals is not None:
                        identity_metrics_reference.setdefault(
                            "quality_signals", reference_signals.__dict__
                        )
                else:
                    identity_profiles_info["reference"] = {
                        "profile": None,
                        "disabled": True,
                    }

        if reference_df is None and "reference" not in identity_profiles_info:
            identity_profiles_info["reference"] = None

        source_summary = identity_metrics_source.get("transforms", {})
        log.info(
            "[IDENTITY] Source transforms enabled=%s estimated_credits=%s",
            bool(enable_source_transforms),
            source_summary.get("estimated_credits"),
        )
        if identity_metrics_reference:
            ref_summary = identity_metrics_reference.get("transforms", {})
            log.info(
                "[IDENTITY] Reference transforms enabled=%s estimated_credits=%s",
                bool(enable_ref_transforms),
                ref_summary.get("estimated_credits"),
            )
        else:
            ref_summary = {}

        def _pct_value(raw: Any) -> Optional[float]:
            if isinstance(raw, (int, float)):
                return max(0.0, min(raw * 100.0 if raw <= 1 else raw, 100.0))
            return None

        source_ready_pct = _pct_value(source_summary.get("identity_ready_pct"))
        ref_ready_pct = _pct_value(ref_summary.get("identity_ready_pct"))

        prep_complete_message_parts = ["Preprocessing complete"]
        if source_ready_pct is not None:
            prep_complete_message_parts.append(
                f"source identity ready {source_ready_pct:.1f}%"
            )
        if ref_ready_pct is not None:
            prep_complete_message_parts.append(
                f"reference identity ready {ref_ready_pct:.1f}%"
            )
        prep_complete_message = " – ".join(prep_complete_message_parts)

        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.PREPROCESSING,
            35.0,
            prep_complete_message,
            source_identity_ready_pct=source_ready_pct,
            reference_identity_ready_pct=ref_ready_pct,
        )
        job["progress"] = 0.35
        job["message"] = prep_complete_message
        _set_job_meta(
            redis_conn,
            job_id,
            progress=0.35,
            message=prep_complete_message,
        )

        transforms_summary: Dict[str, Any] = {
            "source": identity_metrics_source.get(
                "transforms", identity_metrics_source
            ),
        }
        if identity_metrics_reference:
            transforms_summary["reference"] = identity_metrics_reference.get(
                "transforms", identity_metrics_reference
            )

        job.setdefault("metrics", {})
        job["metrics"].setdefault("identity_transforms", transforms_summary)
        job["identity_profiles"] = identity_profiles_info
        job["identity_transforms"] = transforms_summary
        job["source_auto_metrics"] = identity_metrics_source
        if identity_metrics_reference:
            job["reference_auto_metrics"] = identity_metrics_reference

        # Extract configuration from job
        threshold = config.get("threshold", 80)  # Default to 80 (0-100 scale)

        # Convert threshold if it's in 0-1 scale
        if isinstance(threshold, float) and threshold <= 1.0:
            threshold = int(threshold * 100)
        else:
            threshold = int(threshold)

        log.info(f"Using threshold: {threshold} (0-100 scale)")
        log.info(f"Source columns: {list(source_df.columns)[:10]}")
        log.info(
            f"Reference columns: {list(reference_df.columns)[:10] if reference_df is not None else 'None'}"
        )

        matches_found = 0
        status_warnings: List[str] = []

        if mode == "match":
            # PHASE 1 FIX: Extract email domains if needed
            # Check if we need to extract domain from email
            mappings_from_config = config.get("maps") or config.get("mappings", [])
            for mapping in mappings_from_config:
                if mapping.get("extract_from"):
                    # We need to extract domain from email column
                    email_col = mapping["extract_from"]
                    domain_col = mapping["source"]

                    if (
                        email_col in source_df.columns
                        and domain_col not in source_df.columns
                    ):
                        log.info(
                            f"[DOMAIN EXTRACTION] Extracting {domain_col} from {email_col}"
                        )

                        # B2C email domains to filter/flag
                        B2C_DOMAINS = {
                            "gmail.com",
                            "yahoo.com",
                            "aol.com",
                            "hotmail.com",
                            "outlook.com",
                            "live.com",
                            "icloud.com",
                            "me.com",
                            "msn.com",
                            "protonmail.com",
                            "proton.me",
                            "zoho.com",
                            "ymail.com",
                            "rocketmail.com",
                            "mail.com",
                            "gmx.com",
                            "fastmail.com",
                            "tutanota.com",
                            "hushmail.com",
                            "yandex.com",
                            "mail.ru",
                            "qq.com",
                            "163.com",
                            "comcast.net",
                            "verizon.net",
                            "att.net",
                            "cox.net",
                            "earthlink.net",
                            "charter.net",
                            "shaw.ca",
                            "rogers.com",
                            "btinternet.com",
                            "virginmedia.com",
                            "sky.com",
                            "telus.net",
                            "sympatico.ca",
                            "bellsouth.net",
                        }

                        def extract_domain(email):
                            """Extract domain from email address"""
                            if pd.isna(email) or not email or "@" not in str(email):
                                return None
                            try:
                                domain = str(email).strip().lower().split("@")[1]
                                # Clean up common issues
                                domain = domain.strip().rstrip(".")
                                # Validate it looks like a domain
                                if "." in domain and len(domain) > 3:
                                    return domain
                            except:
                                pass
                            return None

                        # Create the domain column
                        source_df[domain_col] = source_df[email_col].apply(
                            extract_domain
                        )

                        # Also create B2C flag column for filtering
                        source_df["_is_b2c_email"] = source_df[domain_col].apply(
                            lambda d: d in B2C_DOMAINS if d else False
                        )

                        # Log statistics
                        domain_fill_rate = source_df[domain_col].notna().mean()
                        unique_domains = source_df[domain_col].nunique()
                        b2c_count = source_df["_is_b2c_email"].sum()
                        b2c_rate = (
                            b2c_count / len(source_df) if len(source_df) > 0 else 0
                        )

                        log.info(
                            f"[DOMAIN EXTRACTION] Created {domain_col}: {domain_fill_rate:.1%} fill rate, {unique_domains} unique domains"
                        )
                        log.info(
                            f"[B2C FILTER] Found {b2c_count} B2C emails ({b2c_rate:.1%} of total)"
                        )

                        # Show sample domains
                        sample_domains = (
                            source_df[domain_col].dropna().value_counts().head(5)
                        )
                        # Log domain statistics without exposing actual domains
                        log.info(
                            f"[DOMAIN EXTRACTION] Unique domains: {source_df[domain_col].nunique()}, Coverage: {source_df[domain_col].notna().mean():.1%}"
                        )

        if mode == "match" and reference_df is not None:
            # Use the real EngineClient for matching
            log.info("[MATCHING MODE] Starting real engine matching")
            log.info(f"[MATCHING MODE] Source shape: {source_df.shape}")
            log.info(f"[MATCHING MODE] Reference shape: {reference_df.shape}")

            match_start_message = (
                f"Starting matching phase ({len(source_df):,} source rows vs "
                f"{len(reference_df):,} reference rows)"
            )
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.MATCHING,
                MATCH_PROGRESS_BASE,
                match_start_message,
                source_rows=len(source_df),
                reference_rows=len(reference_df),
            )
            job["progress"] = MATCH_PROGRESS_BASE / 100
            job["message"] = match_start_message
            _set_job_meta(
                redis_conn,
                job_id,
                progress=MATCH_PROGRESS_BASE / 100,
                message=match_start_message,
            )

            engine_progress_state = {
                "processed_items": 0,
                "comparisons": 0,
                "total_items": max(len(source_df), 1),
                "last_percent": MATCH_PROGRESS_BASE - 1.0,
            }

            async def engine_progress_callback(event) -> None:
                if not event:
                    return

                phase = str(getattr(event, "phase", "") or "").lower()
                if phase == "heartbeat":
                    return

                total_items = getattr(event, "total_items", None)
                if isinstance(total_items, (int, float)) and total_items > 0:
                    engine_progress_state["total_items"] = int(total_items)
                total_items = engine_progress_state["total_items"]

                items_delta = getattr(event, "items_delta", 0) or 0
                if items_delta < 0:
                    items_delta = 0
                engine_progress_state["processed_items"] += items_delta
                processed = engine_progress_state["processed_items"]
                processed = min(processed, total_items)

                comparisons_delta = getattr(event, "comparisons_delta", 0) or 0
                if comparisons_delta < 0:
                    comparisons_delta = 0
                engine_progress_state["comparisons"] += comparisons_delta

                ratio = processed / total_items if total_items else 0.0
                percent = MATCH_PROGRESS_BASE + ratio * MATCH_PROGRESS_RANGE
                percent = min(percent, MATCH_PROGRESS_MAX)

                if phase == "init":
                    percent = MATCH_PROGRESS_BASE
                elif phase == "final":
                    percent = min(MATCH_PROGRESS_MAX, max(percent, MATCH_PROGRESS_BASE))

                if (
                    percent - engine_progress_state["last_percent"] < 0.5
                    and phase == "run"
                ):
                    return

                engine_progress_state["last_percent"] = percent

                message = getattr(event, "message", "") or "Matching records..."
                extras = getattr(event, "extras", None)
                eta_seconds = None
                blocks_delta = None
                if isinstance(extras, dict):
                    eta_seconds = extras.get("eta_seconds")
                    blocks_delta = extras.get("blocks_delta")

                emit_progress(
                    redis_conn,
                    job_id,
                    ProgressPhase.MATCHING,
                    percent,
                    message,
                    items_delta=items_delta,
                    items_processed=processed,
                    total_items=total_items,
                    comparisons_delta=comparisons_delta,
                    comparisons_processed=engine_progress_state["comparisons"],
                    eta_seconds=eta_seconds,
                    blocks_delta=blocks_delta,
                )
                _set_job_meta(
                    redis_conn,
                    job_id,
                    progress=percent / 100,
                    message=message,
                    processed_rows=processed,
                )

            try:
                # Get mappings from config - prefer "maps" for EngineClient
                mappings = config.get("maps") or config.get("mappings", [])
                log.info(f"[MATCHING MODE] Initial mappings from config: {mappings}")

                # Get Phase-2 options from config
                options = config.get("options", {})
                log.info(f"[MATCHING MODE] Phase-2 B2B intelligence options: {options}")

                # Convert mappings to EngineClient format
                # Ensure we have "ref" instead of "reference" for EngineClient
                normalized_mappings = []
                for m in mappings:
                    # Skip mappings that require extraction (handled separately)
                    if "extract_from" in m:
                        continue

                    # Build clean map with only required fields for EngineClient
                    # Only include: source, ref, preferred_algo, weight
                    normalized_map = {
                        "source": m.get("source"),
                        "ref": m.get("ref")
                        or m.get("reference"),  # Handle both formats
                        "preferred_algo": m.get("preferred_algo")
                        or m.get("similarity_measure", "TokenSetRatio"),
                        "weight": m.get("weight", 0.5),
                    }

                    # Only add if we have valid source and ref columns
                    # This excludes any mappings with helper fields like apply_blocking
                    if normalized_map["source"] and normalized_map["ref"]:
                        normalized_mappings.append(normalized_map)

                mappings = normalized_mappings
                log.info(f"[MATCHING MODE] Normalized mappings for engine: {mappings}")

                # If no mappings provided, create basic ones from intelligence config
                if not mappings:
                    intelligence = config.get("intelligence", {})

                    # Extract anchor columns from intelligence
                    source_domain = intelligence.get("source_domain")
                    reference_domain = intelligence.get("reference_domain")
                    source_company = intelligence.get("source_company")
                    reference_company = intelligence.get("reference_company")

                    # Create mappings based on available columns
                    mappings = []

                    # Domain mapping (highest priority for B2B matching)
                    if source_domain and reference_domain:
                        mappings.append(
                            {
                                "source": source_domain,
                                "ref": reference_domain,
                                "preferred_algo": "Domain",  # EngineClient expects preferred_algo
                                "weight": 0.8,
                            }
                        )
                        log.info(
                            f"[MATCHING MODE] Added domain mapping: {source_domain} <-> {reference_domain}"
                        )

                    # Company name mapping
                    if source_company and reference_company:
                        mappings.append(
                            {
                                "source": source_company,
                                "ref": reference_company,
                                "preferred_algo": "TokenSetRatio",  # EngineClient expects preferred_algo
                                "weight": 0.6,
                            }
                        )
                        log.info(
                            f"[MATCHING MODE] Added company mapping: {source_company} <-> {reference_company}"
                        )

                    # Add any additional mappings from config
                    if config.get("additional_mappings"):
                        mappings.extend(config["additional_mappings"])

                    log.info(f"[MATCHING MODE] Generated mappings: {mappings}")

                    if not mappings:
                        log.warning("[MATCHING MODE] No mappings could be generated!")
                        raise ValueError("No mappings available for matching")

                # Find ID columns
                # [DIAGNOSTIC] Log what we have before ID detection
                log.info(f"[ID DETECTION] job_id={job_id}")
                log.info(
                    f"[ID DETECTION] Source columns ({len(source_df.columns)}): {list(source_df.columns)}"
                )
                log.info(
                    f"[ID DETECTION] Config source_id_column: {config.get('source_id_column')}"
                )
                log.info(
                    f"[ID DETECTION] Config reference_id_column: {config.get('reference_id_column')}"
                )

                # Identify which columns contain "ID"
                id_columns_in_source = [
                    col for col in source_df.columns if "ID" in col.upper()
                ]
                log.info(
                    f"[ID DETECTION] Source columns containing 'ID': {id_columns_in_source}"
                )

                # Show pattern for each ID column (length + type, not actual values)
                for col in id_columns_in_source[:5]:  # Limit to first 5
                    sample_vals = source_df[col].dropna().head(3)
                    if len(sample_vals) > 0:
                        patterns = []
                        for val in sample_vals:
                            val_str = str(val).strip()
                            char_types = (
                                "alpha"
                                if val_str.isalpha()
                                else ("alnum" if val_str.isalnum() else "mixed")
                            )
                            patterns.append(f"{len(val_str)}ch_{char_types}")
                        log.info(f"[ID DETECTION] Column '{col}' patterns: {patterns}")

                src_id_col = None
                ref_id_col = None

                # Look for ID columns in source
                for col in source_df.columns:
                    if "ID" in col.upper() and src_id_col is None:
                        src_id_col = col
                        break

                log.info(f"[ID DETECTION] Detected src_id_col: '{src_id_col}'")

                # Look for ID columns in reference
                ref_id_columns = [
                    col for col in reference_df.columns if "ID" in col.upper()
                ]
                log.info(
                    f"[ID DETECTION] Reference columns containing 'ID': {ref_id_columns}"
                )
                for col in reference_df.columns:
                    if "ID" in col.upper() and ref_id_col is None:
                        ref_id_col = col
                        break

                log.info(f"[ID DETECTION] Detected ref_id_col: '{ref_id_col}'")

                if src_id_col:
                    config["source_id_column"] = src_id_col
                if ref_id_col:
                    config["reference_id_column"] = ref_id_col

                # Create EngineClient
                if EngineClient is None:
                    raise ImportError("EngineClient could not be imported")

                log.info("[MATCHING MODE] Creating EngineClient instance...")
                engine_client = EngineClient()
                log.info("[MATCHING MODE] EngineClient created successfully")

                log.info("[MATCHING MODE] Running real matching with EngineClient...")
                log.info(f"[MATCHING MODE] Final mappings: {mappings}")
                log.info(
                    f"[MATCHING MODE] Threshold: {threshold} (will be {threshold/100.0} in engine)"
                )
                log.info(f"[MATCHING MODE] Source ID col: {src_id_col}")
                log.info(f"[MATCHING MODE] Reference ID col: {ref_id_col}")

                match_kwargs = dict(
                    source_data=source_df,
                    reference_data=reference_df,
                    mappings=mappings,
                    threshold=threshold / 100.0,
                    source_id_column=src_id_col,
                    reference_id_column=ref_id_col,
                    job_id=job_id,
                    options=options,
                    progress_callback=engine_progress_callback,
                )

                # Run async function in sync context
                # Save existing loop to avoid disrupting the environment
                try:
                    existing_loop = asyncio.get_event_loop()
                    loop_was_running = existing_loop.is_running()
                except RuntimeError:
                    existing_loop = None
                    loop_was_running = False

                if existing_loop and loop_was_running:
                    # We're already in an async context, this shouldn't happen in worker
                    log.warning(
                        "Unexpected: worker_v2 called with existing running loop"
                    )
                    # Create a thread to run the async function
                    import concurrent.futures

                    with concurrent.futures.ThreadPoolExecutor() as executor:
                        future = executor.submit(
                            asyncio.run,
                            engine_client.match_records(**match_kwargs),
                        )
                        match_result = future.result(timeout=300)  # 5 min timeout
                else:
                    # Normal case: no existing loop or not running
                    # BLOCKING VALIDATION: Log what blocking strategy we're using
                    blocking_info = config.get("blocking_strategy", {})
                    if blocking_info:
                        src_block_col = (
                            blocking_info.get("source_block_col")
                            or blocking_info.get("source_column")
                            or "none"
                        )
                        ref_block_col = (
                            blocking_info.get("ref_block_col")
                            or blocking_info.get("reference_column")
                            or "none"
                        )
                        est_pairs = blocking_info.get("pairs_after", "unknown")
                        domain_first = (
                            "domain" in src_block_col.lower()
                            or "domain" in ref_block_col.lower()
                        )
                        log.info(
                            f"[BLOCKING] domain_first={domain_first}, src_block={src_block_col}, ref_block={ref_block_col}, est_pairs={est_pairs}"
                        )
                    else:
                        log.info("[BLOCKING] No blocking strategy configured")

                    # Check if we have domain columns for blocking
                    domain_cols = [
                        col for col in source_df.columns if "domain" in col.lower()
                    ]
                    if domain_cols:
                        log.info(
                            f"[BLOCKING] Domain columns available for blocking: {domain_cols}"
                        )
                        domain_coverage = (
                            domain_coverage_estimate
                            if domain_coverage_estimate is not None
                            else _estimate_domain_coverage(source_df)
                        )
                        if domain_coverage is not None:
                            log.info(
                                f"[BLOCKING] Domain coverage: {float(domain_coverage):.1%}"
                            )

                    # Use asyncio.run() for clean execution
                    match_result = asyncio.run(
                        engine_client.match_records(**match_kwargs)
                    )

                # Extract results
                result_df = match_result.matches
                matches_found = len(result_df)
                status_warnings = []

                # Extract Phase-2 metrics if available
                if hasattr(match_result, "metrics") and match_result.metrics:
                    log.info(f"[MATCHING MODE] Phase-2 metrics: {match_result.metrics}")
                    # Store metrics in job metadata for status endpoint
                    job["metrics"] = match_result.metrics
                else:
                    job["metrics"] = {}

                if isinstance(job.get("metrics"), dict) and job.get("summary"):
                    job["metrics"].setdefault("summary", job["summary"])

                identity_summary = job.get("identity_transforms")
                if identity_summary:
                    job.setdefault("metrics", {})
                    job["metrics"].setdefault("identity_transforms", identity_summary)

                log.info(
                    f"[MATCHING MODE] ✅ REAL MATCHING COMPLETED: {matches_found} matches found"
                )
                log.info(f"[MATCHING MODE] Result columns: {list(result_df.columns)}")
                # Log score statistics without exposing row-level data
                if "score" in result_df.columns:
                    score_stats = result_df["score"].describe()
                    log.info(
                        f"[MATCHING MODE] Score stats - mean: {score_stats['mean']:.2f}, median: {score_stats['50%']:.2f}, min: {score_stats['min']:.2f}, max: {score_stats['max']:.2f}"
                    )
                else:
                    log.info("[MATCHING MODE] No score column found")
                log.info("[MATCHING MODE] Using REAL ENGINE results")

                # Ensure we have the expected columns for the UI
                # The engine returns scores in 0-1 scale, convert to 0-100 for display
                if "score" in result_df.columns:
                    result_df["match_score"] = (result_df["score"] * 100).round(1)
                elif (
                    "match_score" in result_df.columns
                    and result_df["match_score"].max() <= 1
                ):
                    result_df["match_score"] = (result_df["match_score"] * 100).round(1)

                # Add status column based on score (will be overridden by reason codes if available)
                if "match_status" not in result_df.columns:
                    if "match_score" in result_df.columns:
                        # Use simpler status names that align with B2B requirements
                        result_df["match_status"] = result_df["match_score"].apply(
                            lambda x: "Account Match"
                            if x >= 75
                            else "Possible Match"
                            if x >= 65
                            else "No Match"
                        )
                    elif "score" in result_df.columns:
                        result_df["match_status"] = result_df["score"].apply(
                            lambda x: "Account Match"
                            if x >= 0.75
                            else "Possible Match"
                            if x >= 0.65
                            else "No Match"
                        )
                    else:
                        result_df["match_status"] = "Possible Match"

                # Ensure reference_id column exists
                if "reference_id" not in result_df.columns:
                    if ref_id_col and ref_id_col in result_df.columns:
                        result_df["reference_id"] = result_df[ref_id_col]
                    elif "ref_id" in result_df.columns:
                        result_df["reference_id"] = result_df["ref_id"]

                # Add match reason/explanation if available
                if "match_reason" not in result_df.columns:
                    if "explanation" in result_df.columns:
                        result_df["match_reason"] = result_df["explanation"]
                    elif "match_explanation" in result_df.columns:
                        result_df["match_reason"] = result_df["match_explanation"]
                    else:
                        # Generate meaningful explanations based on score and mappings
                        def generate_match_reason(score):
                            if score >= 95:
                                return "Exact or near-exact match on key fields"
                            elif score >= 85:
                                return "Strong match with high confidence"
                            elif score >= 75:
                                return "Good match with minor variations"
                            elif score >= 60:
                                return "Moderate match, review recommended"
                            else:
                                return "Potential match, manual verification needed"

                        if "match_score" in result_df.columns:
                            result_df["match_reason"] = result_df["match_score"].apply(
                                generate_match_reason
                            )
                        elif "score" in result_df.columns:
                            result_df["match_reason"] = result_df["score"].apply(
                                lambda x: generate_match_reason(
                                    x * 100 if x <= 1 else x
                                )
                            )
                        else:
                            result_df["match_reason"] = (
                                "Matched based on configured criteria"
                            )

            except Exception as e:
                log.error(
                    f"[MATCHING MODE] ❌ Real matching failed, falling back to mock: {e}",
                    exc_info=True,
                )
                log.error(f"[MATCHING MODE] Error type: {type(e).__name__}")
                log.error(f"[MATCHING MODE] Error details: {str(e)}")

                # Fallback to mock data if engine fails
                log.warning("[MATCHING MODE] ⚠️ USING MOCK DATA DUE TO ENGINE FAILURE")
                result_df = source_df.head(10).copy()
                result_df["match_score"] = 85.0
                result_df["match_status"] = "Mock Match (Engine Error)"
                result_df["reference_id"] = "MOCK-001"
                result_df["match_reason"] = f"Engine error: {str(e)}"
                matches_found = len(result_df)

        elif mode == "manual":
            if reference_df is None or reference_df.empty:
                raise ValueError(
                    f"Manual mode requires reference data for job {job_id}"
                )

            # Reference-side hygiene: Normalize domain columns to eTLD+1 and lowercase
            # This ensures consistent join space (*.ibm.com → ibm.com)
            manual_mappings = config.get("mappings") or config.get("maps") or []
            for mapping in manual_mappings:
                if not isinstance(mapping, dict):
                    continue
                algo = str(mapping.get("algo") or "").strip().lower()
                if algo in ("domain", "email_domain"):
                    ref_col = str(mapping.get("reference") or "").strip()
                    if ref_col and ref_col in reference_df.columns:
                        # Normalize reference domain column (collapse subdomains, lowercase)
                        original_values = reference_df[ref_col].copy()
                        normalized_count = 0

                        def normalize_domain(domain):
                            nonlocal normalized_count
                            if pd.isna(domain) or not domain:
                                return domain
                            domain_str = str(domain).strip().lower()
                            if not domain_str:
                                return domain

                            # Remove protocol if present
                            if "://" in domain_str:
                                domain_str = domain_str.split("://", 1)[1]

                            # Remove path/query/fragment
                            domain_str = (
                                domain_str.split("/")[0].split("?")[0].split("#")[0]
                            )

                            # Extract eTLD+1 (simple heuristic: keep last 2 parts)
                            # For more robust handling, could use tldextract library
                            parts = domain_str.split(".")
                            if len(parts) > 2:
                                # Collapse subdomains: www.ibm.com → ibm.com
                                domain_str = ".".join(parts[-2:])
                                if domain != domain_str:
                                    normalized_count += 1

                            return domain_str

                        reference_df[ref_col] = reference_df[ref_col].apply(
                            normalize_domain
                        )
                        if normalized_count > 0:
                            log.info(
                                f"[REFERENCE DOMAIN HYGIENE] Normalized {normalized_count} domains in column '{ref_col}' (collapsed subdomains to eTLD+1)"
                            )
            manual_mapping_configs: List[ManualMappingConfig] = []
            for mapping in manual_mappings:
                if not isinstance(mapping, dict):
                    continue
                source_col = str(mapping.get("source") or "").strip()
                reference_col = str(mapping.get("reference") or "").strip()
                if not source_col or not reference_col:
                    continue
                algo = str(mapping.get("algo") or "token_set").strip().lower()
                try:
                    weight = float(mapping.get("weight", 0.0))
                except (TypeError, ValueError):
                    weight = 0.0
                if weight > 1:
                    weight = weight / 100.0
                weight = min(max(weight, 0.0), 1.0)
                opts = mapping.get("opts") or {}
                options = ManualMappingOptions(
                    casefold=bool(opts.get("casefold")),
                    strip_punctuation=bool(opts.get("strip_punctuation")),
                )
                manual_mapping_configs.append(
                    ManualMappingConfig(
                        source=source_col,
                        reference=reference_col,
                        algo=algo,
                        weight=weight,
                        opts=options,
                    )
                )

            # CRITICAL FIX: If identity transforms enriched domains from Email, use the enriched column
            # Identity transforms extracts domain from Email and writes to "domain_for_blocking" column
            # But user mappings may reference empty "Domain" column instead of Email
            # This causes rows with email domains to get zero candidates from domain blocking
            from .preprocessing.domain_autoclean import AUTO_DOMAIN_FOR_BLOCKING_COL

            # IMPORTANT: domain_for_blocking is extracted from EMAIL, not from Website/URL columns
            # So we should ONLY replace "Domain" or "Email" based mappings, never "Website" mappings

            if AUTO_DOMAIN_FOR_BLOCKING_COL in source_df.columns:
                # Guard 1: Only flip if enriched column has meaningful coverage (>1%)
                enriched_coverage = (
                    source_df[AUTO_DOMAIN_FOR_BLOCKING_COL].notna().mean()
                )
                if enriched_coverage >= 0.01:
                    log.info(
                        f"[DOMAIN ENRICHMENT] Found email-derived domain column '{AUTO_DOMAIN_FOR_BLOCKING_COL}' with {enriched_coverage:.1%} coverage"
                    )

                    # Find if there's an Email column to identify which mappings benefit from enrichment
                    email_col_candidates = [
                        "Email",
                        "email",
                        "Email Address",
                        "email_address",
                        "E-mail",
                    ]
                    email_col = None
                    for candidate in email_col_candidates:
                        if candidate in source_df.columns:
                            email_col = candidate
                            break

                    for mapping in manual_mapping_configs:
                        if mapping.algo not in ("domain", "email_domain"):
                            continue

                        # Recover the true original source column if this mapping was enriched in a prior step
                        original_source = (
                            mapping.meta.get("original_source") or mapping.source
                        )

                        # If a previous run already swapped the column, but we now have access to the
                        # original data again, revert before re-applying any decisions.
                        if (
                            mapping.source == AUTO_DOMAIN_FOR_BLOCKING_COL
                            and mapping.meta.get("original_source")
                        ):
                            # Restore for evaluation; we will decide below whether enrichment is still needed
                            mapping.source = mapping.meta["original_source"]
                            original_source = mapping.source
                            mapping.meta.pop("original_source", None)
                            mapping.meta.pop("enriched_by_identity", None)
                            log.info(
                                f"[DOMAIN ENRICHMENT] Restored mapping to original column '{original_source}' before evaluation"
                            )

                        # Guard 2: Don't double-flip - if the original column is already the enrichment column, skip
                        if original_source == AUTO_DOMAIN_FOR_BLOCKING_COL:
                            continue

                        # Guard 3: ONLY replace mappings that would benefit from email-derived enrichment.
                        # Keep explicit Website/URL mappings untouched (they're different data sources).
                        source_lower = original_source.lower()
                        is_website_mapping = any(
                            keyword in source_lower
                            for keyword in [
                                "website",
                                "url",
                                "web",
                                "site",
                                "link",
                                "homepage",
                            ]
                        )

                        if is_website_mapping:
                            log.info(
                                f"[DOMAIN ENRICHMENT] Preserving Website mapping '{original_source}' (different source than email-derived enrichment)"
                            )
                            continue

                        # For non-website mappings, check if the original column actually has data.
                        original_coverage = 0.0
                        if original_source in source_df.columns:
                            has_data = source_df[original_source].notna() & (
                                source_df[original_source].astype(str).str.strip() != ""
                            )
                            original_coverage = has_data.mean()

                        if (
                            original_coverage >= 0.10
                        ):  # Original column has >=10% real data
                            log.info(
                                f"[DOMAIN ENRICHMENT] Keeping domain mapping '{original_source}' (has {original_coverage:.1%} coverage)"
                            )
                            continue

                        # Original is empty/missing and not a website field - safe to replace with email-derived domain
                        mapping.meta["original_source"] = original_source
                        mapping.meta["enriched_by_identity"] = True
                        mapping.source = AUTO_DOMAIN_FOR_BLOCKING_COL
                        log.info(
                            f"[DOMAIN ENRICHMENT] Replaced '{original_source}' -> '{AUTO_DOMAIN_FOR_BLOCKING_COL}' (original coverage {original_coverage:.1%})"
                        )
                else:
                    log.info(
                        f"[DOMAIN ENRICHMENT] Skipping '{AUTO_DOMAIN_FOR_BLOCKING_COL}' - low coverage ({enriched_coverage:.1%})"
                    )

            if not manual_mapping_configs:
                raise ValueError(
                    f"Manual mode requires at least one mapping for job {job_id}"
                )

            anchors_cfg = config.get("anchors")
            if isinstance(anchors_cfg, dict):
                anchor_value = (anchors_cfg.get("require_exact_on") or "").strip()
                anchors_cfg = (
                    {"require_exact_on": anchor_value} if anchor_value else None
                )
            else:
                anchors_cfg = None

            blocks_cfg = config.get("blocks")
            if isinstance(blocks_cfg, dict):
                blocks_cfg = {
                    "domain": bool(blocks_cfg.get("domain")),
                    "state": bool(blocks_cfg.get("state")),
                }
            else:
                blocks_cfg = None

            manual_threshold = config.get("threshold", 0.8)
            try:
                manual_threshold = float(manual_threshold)
            except (TypeError, ValueError):
                manual_threshold = 0.8
            if manual_threshold > 1:
                manual_threshold = manual_threshold / 100.0

            manual_limit = config.get("limit")
            try:
                manual_limit = int(manual_limit) if manual_limit is not None else None
            except (TypeError, ValueError):
                manual_limit = None
            if manual_limit is not None and manual_limit <= 0:
                manual_limit = None

            best_only = bool(config.get("best_match_only", True))
            strip_b2c = bool(config.get("strip_b2c", True))

            log.info("[MANUAL MODE] Starting manual matching")

            # [DIAGNOSTIC] Log ID detection for manual mode
            log.info(f"[ID DETECTION MANUAL] job_id={job_id}")
            log.info(
                f"[ID DETECTION MANUAL] Source columns ({len(source_df.columns)}): {list(source_df.columns)}"
            )
            log.info(
                f"[ID DETECTION MANUAL] Config source_id_column: {config.get('source_id_column')}"
            )
            log.info(
                f"[ID DETECTION MANUAL] Config reference_id_column: {config.get('reference_id_column')}"
            )

            # Detect ID columns - prefer config, then sniff
            src_id_col = config.get("source_id_column")
            ref_id_col = config.get("reference_id_column")

            if src_id_col:
                log.info(
                    f"[ID DETECTION MANUAL] Using config-provided source ID: '{src_id_col}'"
                )
            else:
                # Identify which columns contain "ID"
                id_columns_in_source = [
                    col for col in source_df.columns if "ID" in col.upper()
                ]
                log.info(
                    f"[ID DETECTION MANUAL] Source columns containing 'ID': {id_columns_in_source}"
                )

                # Show pattern for each ID column (length + type, not actual values)
                for col in id_columns_in_source[:5]:  # Limit to first 5
                    sample_vals = source_df[col].dropna().head(3)
                    if len(sample_vals) > 0:
                        patterns = []
                        for val in sample_vals:
                            val_str = str(val).strip()
                            char_types = (
                                "alpha"
                                if val_str.isalpha()
                                else ("alnum" if val_str.isalnum() else "mixed")
                            )
                            patterns.append(f"{len(val_str)}ch_{char_types}")
                        log.info(
                            f"[ID DETECTION MANUAL] Column '{col}' patterns: {patterns}"
                        )

                # Sniff for ID column (first with 'ID' in name)
                for col in source_df.columns:
                    if "ID" in col.upper():
                        src_id_col = col
                        log.info(
                            f"[ID DETECTION MANUAL] Detected source ID by sniffing: '{src_id_col}'"
                        )
                        break

            if ref_id_col:
                log.info(
                    f"[ID DETECTION MANUAL] Using config-provided reference ID: '{ref_id_col}'"
                )
            elif reference_df is not None:
                ref_id_columns = [
                    col for col in reference_df.columns if "ID" in col.upper()
                ]
                log.info(
                    f"[ID DETECTION MANUAL] Reference columns containing 'ID': {ref_id_columns}"
                )
                for col in reference_df.columns:
                    if "ID" in col.upper():
                        ref_id_col = col
                        log.info(
                            f"[ID DETECTION MANUAL] Detected reference ID by sniffing: '{ref_id_col}'"
                        )
                        break

            if src_id_col:
                config["source_id_column"] = src_id_col
            if ref_id_col:
                config["reference_id_column"] = ref_id_col
            log.info(
                f"[ID DETECTION MANUAL] Final ID columns - source: '{src_id_col}', reference: '{ref_id_col}'"
            )

            _last_progress_emit = {"percent": MATCH_PROGRESS_BASE - 1.0}

            # Create progress callback to emit incremental updates
            def progress_callback(processed_rows: int, total_rows: int) -> None:
                """Emit progress updates as chunks complete"""
                if total_rows <= 0:
                    return

                ratio = max(0.0, min(processed_rows / total_rows, 1.0))
                percent = MATCH_PROGRESS_BASE + ratio * MATCH_PROGRESS_RANGE
                percent = min(percent, MATCH_PROGRESS_MAX)

                # Throttle to avoid flooding (emit on >=0.5% increments or completion)
                if (
                    percent - _last_progress_emit["percent"] < 0.5
                    and processed_rows < total_rows
                ):
                    return
                _last_progress_emit["percent"] = percent

                message = f"Processed {processed_rows:,} / {total_rows:,} rows"
                emit_progress(
                    redis_conn,
                    job_id,
                    ProgressPhase.MATCHING,
                    percent,
                    message,
                    processed_rows=processed_rows,
                    total_rows=total_rows,
                )
                # Also update job metadata for backwards compatibility
                _set_job_meta(
                    redis_conn,
                    job_id,
                    progress=percent / 100,
                    message=message,
                    processed_rows=processed_rows,
                )
                # Refresh summary snapshot so UI sees up-to-date counts
                try:
                    summary_update = dict(summary_template)
                    summary_update["rows_processed"] = processed_rows
                    summary_update["result_rows"] = processed_rows
                    summary_update["rows_total"] = (
                        summary_update.get("rows_total") or total_rows
                    )
                    summary_update["source_rows"] = (
                        summary_update.get("source_rows") or total_rows
                    )
                    summary_update["progress_pct"] = percent
                    redis_conn.setex(summary_key, 86400, json.dumps(summary_update))
                except Exception:
                    pass

            # Initialize domain registry for family matching
            domain_registry = None
            try:
                from .services.domain_family import get_registry

                domain_registry = get_registry()
                log.info(
                    f"[DOMAIN FAMILY] Domain registry initialized: {domain_registry.to_dict()['total_families']} families, {domain_registry.to_dict()['total_domains']} domains"
                )
            except Exception as e:
                log.warning(
                    f"[DOMAIN FAMILY] Could not initialize domain registry: {e}"
                )

            match_start_message = (
                f"Starting matching phase ({len(source_df):,} source rows)"
            )
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.MATCHING,
                MATCH_PROGRESS_BASE,
                match_start_message,
                total_rows=len(source_df),
            )
            job["progress"] = MATCH_PROGRESS_BASE / 100
            job["message"] = match_start_message
            _set_job_meta(
                redis_conn,
                job_id,
                progress=MATCH_PROGRESS_BASE / 100,
                message=match_start_message,
            )

            result_df, metrics = manual_match_execute(
                source_df=source_df,
                reference_df=reference_df,
                mappings=manual_mapping_configs,
                anchors=anchors_cfg,
                blocks=blocks_cfg,
                threshold=manual_threshold,
                best_only=best_only,
                strip_b2c=strip_b2c,
                limit=manual_limit,
                logger=log,
                progress_callback=progress_callback,
                domain_registry=domain_registry,
                source_id_column=src_id_col,
                reference_id_column=ref_id_col,
            )

            matches_found = int(metrics.get("matches_found", 0))

            # Build row_map (row_index -> real source ID) so downstream consumers can align rows
            if src_id_col and src_id_col in source_df.columns:
                try:
                    id_series = source_df[src_id_col]
                    if hasattr(id_series, "items"):
                        row_map: Dict[str, str] = {}
                        for row_idx, raw_value in id_series.items():
                            if pd.isna(raw_value):
                                continue
                            value = str(raw_value).strip()
                            if not value:
                                continue
                            row_map[str(row_idx)] = value
                        if row_map:
                            job["row_map"] = row_map
                            job.setdefault("metrics", {}).setdefault(
                                "row_map_rows", len(row_map)
                            )
                            log.info(
                                "[MANUAL MODE] Built row_map with %d entries from column '%s'",
                                len(row_map),
                                src_id_col,
                            )
                except Exception as row_map_err:
                    log.warning(
                        "[ID DETECTION MANUAL] Failed to build row_map for job %s using '%s': %s",
                        job_id,
                        src_id_col,
                        row_map_err,
                    )
            status_warnings = []
            job["manual_mode"] = True
            job["metrics"] = metrics
            identity_summary = job.get("identity_transforms")
            if identity_summary:
                job["metrics"].setdefault("identity_transforms", identity_summary)

        else:
            # Dedupe mode: mark some rows as duplicates
            result_df = source_df.copy()
            result_df["duplicate_group"] = 0

            # Mock: mark every 3rd row as a duplicate
            for i in range(0, min(10, len(result_df)), 3):
                result_df.at[i, "duplicate_group"] = 1

            matches_found = len(result_df[result_df["duplicate_group"] > 0])

        # Guard score scale before writeback (ensure 0-100 scale)
        if "score" in result_df.columns and result_df["score"].max() <= 1.00001:
            result_df["score"] = (result_df["score"] * 100).round(2)
        if (
            "match_score" in result_df.columns
            and result_df["match_score"].max() <= 1.00001
        ):
            result_df["match_score"] = (result_df["match_score"] * 100).round(2)

        # Add reason codes and evidence to results
        update_match_status_with_reasons = None
        if mode == "match" and not result_df.empty:
            try:
                from ..utils.reason_codes import (
                    add_reason_codes_to_results,
                    update_match_status_with_reasons as _update_match_status,
                )

                result_df = add_reason_codes_to_results(
                    result_df=result_df,
                    source_df=source_df,
                    reference_df=reference_df,
                    mappings=mappings
                    if "mappings" in locals()
                    else config.get("mappings", []),
                    source_id_col=src_id_col,
                    ref_id_col=ref_id_col,
                )
                if "reasons" in result_df.columns:
                    result_df["reasons"] = (
                        result_df["reasons"].fillna("").apply(_normalize_reason_tokens)
                    )
                update_match_status_with_reasons = _update_match_status
                log.info(
                    f"[REASON CODES] Added reason codes to {len(result_df)} matches"
                )
            except Exception as e:
                log.warning(f"[REASON CODES] Failed to add reason codes: {e}")
                # Ensure columns exist even if generation fails
                if "reasons" not in result_df.columns:
                    result_df["reasons"] = ""
                else:
                    result_df["reasons"] = (
                        result_df["reasons"].fillna("").apply(_normalize_reason_tokens)
                    )
                if "field_score_details" not in result_df.columns:
                    result_df["field_score_details"] = "{}"
                else:
                    # Ensure JSON integrity - convert any non-JSON to valid JSON
                    result_df["field_score_details"] = result_df[
                        "field_score_details"
                    ].apply(
                        lambda x: x
                        if isinstance(x, str) and x.startswith("{")
                        else "{}"
                    )

        if mode == "match" and not result_df.empty:
            reasons_series = result_df.get("reasons")
            if reasons_series is None:
                reasons_series = pd.Series([""] * len(result_df))
            else:
                reasons_series = reasons_series.fillna("").astype(str)
            if reasons_series.eq("").all():
                warning_msg = "MATCH_EVIDENCE_MISSING"
                if warning_msg not in status_warnings:
                    status_warnings.append(warning_msg)
        if (
            mode == "match"
            and not result_df.empty
            and "match_status" in result_df.columns
            and "reasons" in result_df.columns
        ):
            reasons_upper = reasons_series.str.upper()
            status_upper = result_df["match_status"].fillna("").astype(str).str.upper()
            guardrail_mask = status_upper.eq("ACCOUNT MATCH") & ~(
                reasons_upper.str.contains("DOM") | reasons_upper.str.contains("FAM")
            )
            if guardrail_mask.any():
                result_df.loc[guardrail_mask, "match_status"] = "Possible Match"
                if "flags" in result_df.columns:

                    def _append_flag(value: str, token: str) -> str:
                        parts = [
                            piece for piece in str(value or "").split("|") if piece
                        ]
                        if token not in parts:
                            parts.append(token)
                        return "|".join(parts)

                    result_df.loc[guardrail_mask, "flags"] = result_df.loc[
                        guardrail_mask, "flags"
                    ].apply(lambda v: _append_flag(v, "[NO_DOM_FAM]"))
                if "NO_DOM_FAM_GUARDRAIL" not in status_warnings:
                    status_warnings.append("NO_DOM_FAM_GUARDRAIL")
                log.warning(
                    "[GUARDRAIL] Downgraded %s account matches lacking DOM/FAM evidence",
                    int(guardrail_mask.sum()),
                )
            result_df["reasons"] = reasons_series.apply(_normalize_reason_tokens)

        if mode == "match" and reference_df is not None and not result_df.empty:
            display_map, family_display_map, historical_lookup = (
                _build_reference_name_maps(reference_df, ref_id_col)
            )

            if "matched_account_name" not in result_df.columns:
                result_df["matched_account_name"] = pd.Series([None] * len(result_df))

            if "reference_id" in result_df.columns:
                result_df["reference_id"] = result_df["reference_id"].astype(str)
                if display_map:
                    result_df["matched_account_name"] = result_df[
                        "matched_account_name"
                    ].fillna(result_df["reference_id"].map(display_map))

            if family_display_map and "family_id" in result_df.columns:
                result_df["matched_account_name"] = result_df[
                    "matched_account_name"
                ].fillna(
                    result_df["family_id"]
                    .astype(str)
                    .str.lower()
                    .map(family_display_map)
                )

            if result_df["matched_account_name"].isna().any():
                fallback_cols = [
                    "ref_display_name",
                    "ref_name",
                    "reference_name",
                    "r_name",
                    "ref_company",
                    "reference_company",
                    "r_company",
                    "company",
                    "account_name",
                ]
                for candidate in fallback_cols:
                    if candidate in result_df.columns:
                        result_df["matched_account_name"] = result_df[
                            "matched_account_name"
                        ].fillna(
                            result_df[candidate].apply(
                                lambda v: strip_formerly_known_as(v)
                                if isinstance(v, str)
                                else v
                            )
                        )
                        if not result_df["matched_account_name"].isna().any():
                            break

            result_df["matched_account_name"] = (
                result_df["matched_account_name"].fillna("").astype(str)
            )

            if historical_lookup and "reference_id" in result_df.columns:
                historical_flags = (
                    result_df["reference_id"].map(historical_lookup).fillna(False)
                )
                if historical_flags.any():
                    if "score" in result_df.columns:
                        penalty = 5.0 if result_df["score"].max() > 1 else 0.05
                        result_df.loc[historical_flags, "score"] = (
                            result_df.loc[historical_flags, "score"] - penalty
                        ).clip(lower=0)
                        if "match_score" in result_df.columns:
                            if penalty < 1:
                                result_df["match_score"] = (
                                    result_df["score"] * 100
                                ).round(1)
                            else:
                                result_df["match_score"] = result_df["score"].round(1)
                    elif "match_score" in result_df.columns:
                        result_df.loc[historical_flags, "match_score"] = (
                            result_df.loc[historical_flags, "match_score"] - 5
                        ).clip(lower=0)

                    if "field_score_details" in result_df.columns:
                        updated_details = []
                        for details, flag in zip(
                            result_df["field_score_details"], historical_flags.tolist()
                        ):
                            if not flag:
                                updated_details.append(details)
                                continue
                            try:
                                payload = (
                                    json.loads(details)
                                    if isinstance(details, str) and details
                                    else {}
                                )
                            except Exception:
                                payload = {}
                            name_section = payload.get("name")
                            if isinstance(name_section, dict):
                                name_section.setdefault("type", "historical")
                                name_section["historical"] = True
                            else:
                                payload["name"] = {
                                    "type": "historical",
                                    "historical": True,
                                }
                            updated_details.append(
                                json.dumps(payload, separators=(",", ":"))
                            )
                        result_df["field_score_details"] = updated_details

                    if update_match_status_with_reasons:
                        try:
                            result_df = update_match_status_with_reasons(result_df)
                        except Exception as status_exc:
                            log.debug(
                                f"[REASON CODES] status recompute skipped: {status_exc}"
                            )

                    job.setdefault("metrics", {})
                    job["metrics"]["historical_matches"] = int(historical_flags.sum())

        # Write results to storage
        saving_message = f"Saving {len(result_df):,} results to workspace storage..."
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.SAVING_RESULTS,
            90.0,
            saving_message,
            result_rows=len(result_df),
        )
        job["progress"] = 0.90
        job["message"] = saving_message
        job["state"] = "processing"
        job["status"] = "processing"
        _set_job_meta(
            redis_conn,
            job_id,
            progress=0.90,
            message=saving_message,
            status="processing",
            phase="saving_results",
            matches_found=matches_found,
            rows_processed=len(source_df),
            result_rows=len(result_df),
        )
        log.info(
            "[WORKER_V2] Preparing to write CSV for job %s (rows=%s, matches=%s)",
            job_id,
            len(source_df),
            matches_found,
        )
        log.info(
            "[WORKER_V2] Writing %s result rows to CSV for job %s",
            len(result_df),
            job_id,
        )
        result_object_id = _write_results_csv(job_id, result_df)
        log.info(
            "[WORKER_V2] Results artifact for job %s available at %s",
            job_id,
            result_object_id,
        )
        post_save_message = "Results saved successfully. Preparing download link..."
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.SAVING_RESULTS,
            95.0,
            post_save_message,
            result_rows=len(result_df),
        )
        job["progress"] = 0.95
        job["message"] = post_save_message
        _set_job_meta(
            redis_conn,
            job_id,
            progress=0.95,
            message=post_save_message,
        )

        effective_hash = (
            job.get("config_hash_effective")
            or config.get("config_hash")
            or config.get("hash")
        )
        if effective_hash:
            job["config_hash_effective"] = effective_hash
        if (
            config_hash_submitted
            and effective_hash
            and str(config_hash_submitted) != str(effective_hash)
        ):
            if "CONFIG_MISMATCH" not in status_warnings:
                status_warnings.append("CONFIG_MISMATCH")
            log.warning(
                "[CONFIG] Submitted hash %s differs from effective hash %s for job %s",
                config_hash_submitted,
                effective_hash,
                job_id,
            )

        if status_warnings:
            existing_warnings = job.get("warnings")
            combined_warnings: List[str] = []
            if isinstance(existing_warnings, list):
                combined_warnings.extend(existing_warnings)
            elif isinstance(existing_warnings, str) and existing_warnings:
                combined_warnings.append(existing_warnings)
            for warning in status_warnings:
                if warning not in combined_warnings:
                    combined_warnings.append(warning)
            if combined_warnings:
                job["warnings"] = combined_warnings
                metrics_ref = job.get("metrics")
                if isinstance(metrics_ref, dict):
                    metrics_ref.setdefault("warnings", combined_warnings)

        # Add match status counts to metrics for status endpoint
        if "match_status" in result_df.columns:
            account_matches = len(
                result_df[result_df["match_status"] == "Account Match"]
            )
            possible_matches = len(
                result_df[result_df["match_status"] == "Possible Match"]
            )
            no_matches = len(result_df[result_df["match_status"] == "No Match"])

            # Update metrics with match counts
            if "metrics" not in job:
                job["metrics"] = {}
            job["metrics"]["account_matches"] = account_matches
            job["metrics"]["possible_matches"] = possible_matches
            job["metrics"]["no_matches"] = no_matches

            log.info(
                f"[MATCH COUNTS] Account Matches: {account_matches}, Possible: {possible_matches}, No Match: {no_matches}"
            )

        # Update job to completed
        job["state"] = "completed"
        job["status"] = "completed"
        job["progress"] = 1.0
        job["completed_at"] = time.time()
        job["result_object_id"] = result_object_id
        job["results_path"] = result_object_id
        job["result_rows"] = len(result_df)
        job["matches_found"] = matches_found
        job["rows_processed"] = len(source_df)
        job["message"] = "Match complete! Results are ready to review."
        if not isinstance(job.get("metrics"), dict):
            job["metrics"] = {}
        metrics_ref = job["metrics"]
        metrics_ref["rows_processed"] = len(source_df)
        metrics_ref["result_rows"] = len(result_df)
        metrics_ref["matches_found"] = matches_found

        try:
            started_at = job.get("started_at")
            elapsed_s = (
                max(0.0, time.time() - float(started_at))
                if isinstance(started_at, (int, float))
                else 0.0
            )
        except Exception:
            elapsed_s = 0.0

        if job.get("summary"):
            final_summary = dict(job["summary"])
            final_summary.update(
                {
                    "rows_processed": len(source_df),
                    "result_rows": len(result_df),
                    "matches_found": matches_found,
                    "possible_matches": job.get("metrics", {}).get("possible_matches"),
                    "elapsed_seconds": elapsed_s,
                }
            )
            job["summary"] = final_summary
            if isinstance(job.get("metrics"), dict):
                job["metrics"]["summary"] = final_summary
            redis_conn.setex(summary_key, 86400, json.dumps(final_summary))

        log.info(
            "[WORKER_V2] Persisting database status for job %s (results_path=%s, rows=%s, matches=%s)",
            job_id,
            result_object_id,
            len(source_df),
            matches_found,
        )
        try:
            update_job_status_sync(
                job_id,
                "completed",
                results_path=result_object_id,
                matches_found=matches_found,
                rows_processed=len(source_df),
                elapsed_s=elapsed_s,
            )
        except Exception as db_err:
            log.exception(
                "[WORKER_V2] Unable to persist results_path for job %s: %s",
                job_id,
                db_err,
            )

        log.info(
            "[WORKER_V2] Persisting completed job %s to Redis with result_object_id=%s",
            job_id,
            result_object_id,
        )
        redis_conn.setex(job_key, 86400, json.dumps(job))
        log.info(
            f"[WORKER_V2] Job {job_id} completed: {job['result_rows']} rows -> {result_object_id}"
        )
        completion_message = "Match complete! Results are ready to review."
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.COMPLETED,
            100.0,
            completion_message,
            result_rows=len(result_df),
            matches_found=matches_found,
        )
        _set_job_meta(
            redis_conn,
            job_id,
            progress=1.0,
            message=completion_message,
            matches_found=matches_found,
            processed_rows=len(source_df),
            rows_processed=len(source_df),
            result_rows=len(result_df),
            result_object_id=result_object_id,
            results_path=result_object_id,
            status="completed",
        )

    except Exception as e:
        log.error(f"[WORKER_V2] Job {job_id} failed: {e}", exc_info=True)

        # Update job to failed state
        try:
            failure_message = f"Processing failed: {e}"
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.FAILED,
                100.0,
                failure_message,
            )
            _set_job_meta(
                redis_conn,
                job_id,
                progress=1.0,
                status="failed",
                message=failure_message,
            )
            job["state"] = "failed"
            job["status"] = "failed"
            job["progress"] = 1.0
            job["error"] = str(e)
            job["message"] = failure_message
            job["failed_at"] = time.time()
            redis_conn.setex(f"job:{job_id}", 86400, json.dumps(job))
        except:
            pass

        raise
