# fmatch/saas/api/v2/jobs.py
import os
import sys
import json
import re
import logging
import pandas as pd
import numpy as np
import pyarrow as pa
from fmatch.utils_normalize import strip_formerly_known_as
import pyarrow.parquet as pq
import hashlib
import io
import time
import gzip
import tempfile
from pathlib import Path
from fastapi import (
    APIRouter,
    Depends,
    HTTPException,
    Body,
    Query,
    UploadFile,
    File,
    Form,
    BackgroundTasks,
    Header,
    Response,
    Request,
)
from fastapi.responses import StreamingResponse, FileResponse, Response
from sse_starlette.sse import EventSourceResponse
import asyncio
from datetime import datetime
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text, select as sa_select
from typing import Optional, Dict, Any, List, Set, Tuple, Union
from uuid import UUID, uuid4
from dataclasses import dataclass

# Windows-compatible emojis
CHECK_MARK = "[OK]" if sys.platform.startswith("win") else "âœ…"
CROSS_MARK = "[X]" if sys.platform.startswith("win") else "âŒ"
WARNING_SIGN = "[!]" if sys.platform.startswith("win") else "âš ï¸"
INFO_SIGN = "[i]" if sys.platform.startswith("win") else "â„¹ï¸"

# Set up logger
log = logging.getLogger(__name__)

# Try to import chardet for encoding detection
try:
    import chardet

    CHARDET_AVAILABLE = True
except ImportError:
    log.warning("chardet not installed. Falling back to basic encoding detection.")
    CHARDET_AVAILABLE = False

from ...models import Job  # Note: three dots for proper import
from ...db import get_session, AsyncSessionLocal
from ...auth import get_current_account
from ..progress import get_redis
from ...models import Integration, IntegrationProvider, IntegrationStatus
from ...services.salesforce_gateway import SalesforceGateway, get_salesforce_gateway
from ...services.domain_utils import (
    enrich_with_domain_families,
    extract_domain_from_url,
)
from ...services.domain_family import get_registry
from ...services.utils.data_profile import normalise_series
from ...storage import get_storage_backend
from ...utils import apply_best_match_per_source
from ...services.confidence_bands import band_for_score, count_confidence_bands
from ...services.manual_match_service import (
    execute as manual_match_execute,
    validate_configuration as validate_manual_match_configuration,
    MappingConfig as ManualMappingConfig,
    MappingOptions as ManualMappingOptions,
    ManualMatchValidationError,
    ManualMatchServiceError,
)
# Lazy import: worker_v2 has heavy dependencies (GCS, DuckDB) that slow startup
# Import moved to _get_process_job_v2() helper below

# Use the exact same hash function as intelligence.py to avoid drift
from .intelligence import _stable_intelligence_hash

# Import from engine bridge ONLY
from fmatch.saas.engine_bridge import (
    match,
    dedupe,
    MatchConfig,
    DedupeConfig,
    ProgressEvent,
)


# Lazy loader for worker_v2 to avoid slow startup
_process_job_v2_cached = None

def _get_process_job_v2():
    """Lazy import process_job_v2 to avoid slow GCS/DuckDB init at startup."""
    global _process_job_v2_cached
    if _process_job_v2_cached is None:
        from ...worker_v2 import process_job_v2
        _process_job_v2_cached = process_job_v2
    return _process_job_v2_cached


# Helper function to get proper temp directory
def get_temp_dir(job_id: str) -> Path:
    """Get a temporary directory for job files that works on Windows and Unix."""
    if sys.platform.startswith("win"):
        # Use Windows temp directory
        base_dir = Path(tempfile.gettempdir()) / "fmatch" / job_id
    else:
        # Use /tmp on Unix systems
        base_dir = Path("/tmp") / "fmatch" / job_id
    base_dir.mkdir(parents=True, exist_ok=True)
    return base_dir


def _normalize_results_path(value: Optional[str]) -> Optional[str]:
    """Ensure storage keys use POSIX separators for portability."""
    if not value:
        return value
    return str(value).replace("\\", "/")


def _coerce_progress_percent(value: Any) -> Optional[int]:
    """Normalize progress values to an integer percent (0-100)."""
    if value is None:
        return None
    try:
        if isinstance(value, str):
            text = value.strip()
            if not text:
                return None
            if text.endswith("%"):
                text = text[:-1].strip()
            numeric = float(text)
        else:
            numeric = float(value)
    except Exception:
        return None

    # Some workers emit progress as fractions (0.0-1.0).
    if 0.0 <= numeric <= 1.0:
        numeric *= 100.0
    return int(max(0.0, min(100.0, round(numeric))))


def _load_domain_registry_safe() -> Optional[Any]:
    """Attempt to load the domain family registry used for manual matching."""
    try:
        registry = get_registry()
        stats = registry.to_dict()
        log.info(
            "[DOMAIN FAMILY] Registry initialized: %s families, %s domains",
            stats.get("total_families"),
            stats.get("total_domains"),
        )
        return registry
    except Exception as exc:
        log.warning(f"[DOMAIN FAMILY] Could not initialize domain registry: {exc}")
        return None


def normalize_reason_tokens(raw: str) -> str:
    tokens: List[str] = []
    for token in str(raw or "").split("|"):
        clean = token.strip().upper()
        if clean and clean not in tokens:
            tokens.append(clean)
    if "GEO" in tokens and not any(t.startswith("NAME") for t in tokens):
        geo_index = tokens.index("GEO")
        tokens.insert(max(geo_index, 0), "NAME*")
    return "|".join(tokens)


# Backwards compatibility for legacy references
_normalize_reason_tokens = normalize_reason_tokens


def extract_domain_from_url(url):
    """Extract domain from URL or website field"""
    if not url or pd.isna(url) or str(url).lower() in ("nan", "none", ""):
        return None

    url = str(url).strip().lower()

    # Skip invalid URLs
    if url in ("-", "n/a", "na", "null"):
        return None

    # Already a clean domain (e.g., "example.com")
    if "/" not in url and "." in url and not url.startswith("http"):
        return url

    # Parse URL
    from urllib.parse import urlparse

    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    try:
        parsed = urlparse(url)
        domain = parsed.netloc or parsed.path.split("/")[0]

        # Remove www. prefix
        if domain.startswith("www."):
            domain = domain[4:]

        # Validate it looks like a domain
        if domain and "." in domain and len(domain) > 3:
            return domain
        return None
    except:
        return None


# Salesforce integration helpers
async def get_sf_integration(db: AsyncSession, account_id: str):
    """Get active Salesforce integration for account - tolerant to enum variations"""

    # Build status values list - handle both CONNECTED and legacy ACTIVE
    status_values = []
    if hasattr(IntegrationStatus, "CONNECTED"):
        status_values.append(IntegrationStatus.CONNECTED)
    if hasattr(IntegrationStatus, "ACTIVE"):
        status_values.append(IntegrationStatus.ACTIVE)

    # Handle provider enum variations
    provider_value = getattr(
        IntegrationProvider,
        "SALESFORCE",
        getattr(IntegrationProvider, "salesforce", None),
    )

    if not status_values or not provider_value:
        # Log and return None if enums are broken
        import logging

        logging.warning(
            "IntegrationStatus or IntegrationProvider enums may be misconfigured"
        )
        return None

    result = await db.execute(
        sa_select(Integration).where(
            Integration.account_id == str(account_id),
            Integration.provider == provider_value,
            Integration.status.in_(status_values),
        )
    )
    return result.scalar_one_or_none()


async def materialize_salesforce(
    sf, soql: str, obj: str, use_bulk: bool, job_id: str
) -> pd.DataFrame:
    """Materialize Salesforce data to DataFrame via Parquet for memory efficiency"""
    # Use proper temp directory
    temp_dir = get_temp_dir(job_id)
    temp_path = temp_dir / f"sf_{uuid4().hex}.parquet"
    temp_path_str = str(temp_path)
    wrote_any = False
    writer = None

    try:
        if use_bulk:
            # Check if bulk_query_stream exists, otherwise use bulk_query with temp file
            log.info(f"Using Bulk API for {obj}")

            if hasattr(sf, "bulk_query_stream"):
                # Stream directly if available
                async for chunk_bytes in sf.bulk_query_stream(soql, object_type=obj):
                    df_chunk = pd.read_csv(
                        io.BytesIO(chunk_bytes), header=0 if not wrote_any else None
                    )
                    table = pa.Table.from_pandas(df_chunk, preserve_index=False)
                    if writer is None:
                        writer = pq.ParquetWriter(temp_path_str, table.schema)
                    writer.write_table(table)
                    wrote_any = True
            else:
                # Fall back to bulk_query
                bulk_res = await sf.bulk_query(soql, obj)
                if not bulk_res:
                    if writer:
                        writer.close()
                    return pd.DataFrame()
                # bulk_res may be a CSV string or a list of dicts
                if isinstance(bulk_res, str):
                    csv_path = temp_dir / f"bulk_{uuid4().hex}.csv"
                    with open(csv_path, "w", encoding="utf-8", newline="") as f:
                        f.write(bulk_res)
                        for chunk in pd.read_csv(csv_path, chunksize=100000):
                            table = pa.Table.from_pandas(chunk, preserve_index=False)
                            if writer is None:
                                writer = pq.ParquetWriter(temp_path_str, table.schema)
                            writer.write_table(table)
                            wrote_any = True
                    if os.path.exists(csv_path):
                        os.unlink(csv_path)
                elif isinstance(bulk_res, list):
                    # Write list-of-dicts in chunks directly
                    CHUNK = 100000
                    for i in range(0, len(bulk_res), CHUNK):
                        df_chunk = pd.DataFrame(bulk_res[i : i + CHUNK])
                        table = pa.Table.from_pandas(df_chunk, preserve_index=False)
                        if writer is None:
                            writer = pq.ParquetWriter(temp_path_str, table.schema)
                        writer.write_table(table)
                        wrote_any = True
        else:
            # REST API with pagination
            log.info(f"Using REST API for {obj}")
            async for page in sf.paginate_soql(soql):
                # Handle both list and dict response formats
                if isinstance(page, list):
                    records = page
                else:
                    records = page.get("records", [])
                if records:
                    df_page = pd.DataFrame(records)
                    # Remove Salesforce metadata columns
                    df_page = df_page.drop(columns=["attributes"], errors="ignore")

                    table = pa.Table.from_pandas(df_page, preserve_index=False)
                    if writer is None:
                        writer = pq.ParquetWriter(temp_path_str, table.schema)
                    writer.write_table(table)
                    wrote_any = True

        if writer:
            writer.close()

        # Read back the complete DataFrame if we wrote anything
        df = pd.read_parquet(temp_path_str) if wrote_any else pd.DataFrame()

    finally:
        # Clean up parquet file
        if os.path.exists(temp_path_str):
            os.unlink(temp_path_str)

    return df


async def stash_dataframe(
    job_id: str,
    df: pd.DataFrame,
    db: AsyncSession,
    account_id: UUID,
    kind: str = "source",
) -> None:
    """Store DataFrame in temp directory as parquet (consistent with CSV flow)"""
    output_dir = get_temp_dir(job_id)
    file_path = output_dir / f"{kind}.parquet"
    df.to_parquet(file_path, index=False)

    # Also store in Redis for quick access
    # Ownership check (hide existence on mismatch)
    try:
        rec = await db.get(Job, job_id)
    except Exception:
        rec = None
    if not rec or (rec.account_id and str(rec.account_id) != str(account_id)):
        raise HTTPException(404, "job not found")

    redis_conn = await get_redis()
    if redis_conn:
        # Store a sample for preview
        sample_data = df.head(100).to_json(orient="records")
        await redis_conn.set(f"job:{job_id}:{kind}_sample", sample_data, ex=3600)

        # Store metadata
        meta = {
            "rows": len(df),
            "columns": list(df.columns),
            "dtypes": {col: str(dtype) for col, dtype in df.dtypes.items()},
        }
        await redis_conn.set(f"job:{job_id}:{kind}_meta", json.dumps(meta), ex=3600)


# Request models
from pydantic import BaseModel, Field, ConfigDict, ValidationError


class ExecuteRequest(BaseModel):
    configuration: Dict[str, Any]


# Models for JSON job creation endpoint
class Mapping(BaseModel):
    model_config = ConfigDict(extra="allow")
    role: str = ""
    type_source: str = ""
    type_reference: str = ""
    source: str
    reference: str
    preferred_algo: Optional[str] = None
    weight: Optional[float] = None
    confidence: Optional[float] = None


class JobsInput(BaseModel):
    model_config = ConfigDict(extra="allow")
    object_id: Optional[str] = None
    reference_object_id: Optional[str] = None
    uri: Optional[str] = None
    reference_uri: Optional[str] = None
    has_header: Optional[bool] = True
    reference_has_header: Optional[bool] = True
    union: Optional[dict] = None


class JobsIntelligence(BaseModel):
    model_config = ConfigDict(extra="allow")
    profile: str = "balanced"
    mappings: List[Mapping] = Field(default_factory=list)
    threshold: Optional[float] = 0.8
    apply_blocking: Optional[bool] = True
    b2c_filter: Optional[bool] = True
    use_domain_family: Optional[bool] = True
    config_hash: Optional[str] = None


class CreateJobRequest(BaseModel):
    model_config = ConfigDict(extra="allow")
    input: JobsInput
    intelligence: JobsIntelligence


router = APIRouter(prefix="/api/v2/jobs", tags=["jobs-v2"])
manual_router = APIRouter(prefix="/api/v2/manual", tags=["manual-match"])

__all__ = ["router", "manual_router"]

# Cloud run minimal endpoints (presigned upload + single-run jobs)
from ... import settings as _settings
from ...models import JobStatus, JobType
from ...services.usage_gateway import UsageGateway
from ...services.justcall_models import JustCallConfig
from ...services.heyreach_models import HeyReachConfig
from ...workers.justcall_worker import (
    JustCallWorker,
    META_KEY as JUSTCALL_META_KEY,
    STATE_KEY as JUSTCALL_STATE_KEY,
    PROGRESS_KEY as JUSTCALL_PROGRESS_KEY,
    SUMMARY_KEY as JUSTCALL_SUMMARY_KEY,
    ERRORS_KEY as JUSTCALL_ERRORS_KEY,
    RESULTS_KEY as JUSTCALL_RESULTS_KEY,
)
from ...workers.heyreach_worker import (
    HeyReachWorker,
    META_KEY as HEYREACH_META_KEY,
    STATE_KEY as HEYREACH_STATE_KEY,
    PROGRESS_KEY as HEYREACH_PROGRESS_KEY,
    SUMMARY_KEY as HEYREACH_SUMMARY_KEY,
    ERRORS_KEY as HEYREACH_ERRORS_KEY,
    RESULTS_KEY as HEYREACH_RESULTS_KEY,
)


class InputSpec(BaseModel):
    object_id: Optional[str] = None
    uri: Optional[str] = None
    reference_object_id: Optional[str] = None
    reference_uri: Optional[str] = None
    union: Optional[List[str]] = (
        None  # optional union-mode dedupe across multiple objects
    )


def _resolve_upload_path(object_id: Optional[str]) -> Optional[str]:
    if not object_id:
        return None
    normalized = str(object_id).strip()
    if not normalized:
        return None
    normalized = normalized.replace("\\", "/").lstrip("/")
    if _settings.STORAGE_BACKEND == "local":
        base_path = Path(_settings.LOCAL_STORAGE_PATH).resolve()
        rel_path = Path(normalized)
        if rel_path.is_absolute():
            return str(rel_path)
        try:
            if rel_path.parts and Path(rel_path.parts[0]).name == base_path.name:
                rel_path = Path(*rel_path.parts[1:])
        except Exception:
            pass
        return str((base_path / rel_path).resolve())
    return normalized


def _mask_value_for_log(value: Optional[str]) -> Optional[str]:
    if not value:
        return value
    text = str(value)
    if text.startswith("http://") or text.startswith("https://"):
        try:
            from urllib.parse import urlparse

            parsed = urlparse(text)
            return f"{parsed.scheme}://{parsed.netloc}/..."
        except Exception:
            pass
    if len(text) <= 12:
        return text
    return f"{text[:6]}...{text[-4:]}"


def _mask_config_for_log(config: Dict[str, Any]) -> Dict[str, Any]:
    masked = {}
    for key, value in config.items():
        if key in {"source_url", "reference_url"}:
            masked[key] = _mask_value_for_log(value)
        else:
            masked[key] = value
    return masked


class IntelligenceSpec(BaseModel):
    profile: str
    threshold: float
    options: Dict[str, Any] = Field(default_factory=dict)
    hash: Optional[str] = None
    config_hash: Optional[str] = None

    model_config = ConfigDict(extra="allow")


class RunReq(BaseModel):
    input: Optional[InputSpec] = None
    intelligence: Optional[IntelligenceSpec] = None
    mode: Optional[str] = None  # "match" (default) or "dedupe"
    job_type: Optional[str] = None  # alias for mode to support Sheets payloads
    upload_id: Optional[str] = None
    rows: Optional[List[Dict[str, Any]]] = None
    config: Optional[Dict[str, Any]] = None


class RunResp(BaseModel):
    job_id: str
    correlation_id: Optional[str] = None
    echo_config_hash: Optional[str] = None


class ManualRangeSpec(BaseModel):
    model_config = ConfigDict(extra="allow")
    upload_id: Optional[str] = None
    id_column: Optional[str] = None


class ManualMappingOptionsSpec(BaseModel):
    casefold: bool = False
    strip_punctuation: bool = False


class ManualMappingSpec(BaseModel):
    model_config = ConfigDict(extra="allow")
    source: str
    reference: str
    algo: str = "token_set"
    weight: float = 0.2
    opts: ManualMappingOptionsSpec = ManualMappingOptionsSpec()


class ManualAnchorsSpec(BaseModel):
    model_config = ConfigDict(extra="allow")
    require_exact_on: Optional[str] = None


class ManualBlocksSpec(BaseModel):
    model_config = ConfigDict(extra="allow")
    domain: bool = False
    state: bool = False


class ManualExecuteRequest(BaseModel):
    model_config = ConfigDict(extra="allow")
    mode: Optional[str] = None
    source: ManualRangeSpec
    reference: ManualRangeSpec
    mappings: List[ManualMappingSpec]
    anchors: Optional[ManualAnchorsSpec] = None
    blocks: Optional[ManualBlocksSpec] = None
    threshold: Optional[float] = 0.8
    best_match_only: bool = True
    strip_b2c: bool = True
    limit: Optional[int] = None
    preview: Optional[bool] = False
    enable_identity_transforms: Optional[bool] = None
    transforms: Optional[Dict[str, Any]] = None
    source_id_column: Optional[str] = None
    reference_id_column: Optional[str] = None


def _normalize_manual_threshold(value: Optional[float]) -> float:
    if value is None:
        return 0.8
    try:
        num = float(value)
    except (TypeError, ValueError):
        return 0.8
    if num > 1:
        num = num / 100.0
    return min(max(num, 0.0), 1.0)


def _load_manual_dataframe(object_id: str, limit: Optional[int] = None) -> pd.DataFrame:
    from ...storage import resolve_object_path, exists_object  # type: ignore

    if not object_id:
        raise ValueError("upload_id is required")

    if not exists_object(object_id):
        raise FileNotFoundError(object_id)

    path = resolve_object_path(object_id)
    read_kwargs = dict(
        dtype=str,
        keep_default_na=False,
        na_values=[],
        encoding="utf-8",
        encoding_errors="ignore",
    )
    if limit is not None and limit > 0:
        read_kwargs["nrows"] = int(limit)
    return pd.read_csv(path, **read_kwargs)


def _build_manual_mapping_configs(
    mappings: List[Dict[str, Any]],
) -> List[ManualMappingConfig]:
    configs: List[ManualMappingConfig] = []
    for entry in mappings or []:
        if not isinstance(entry, dict):
            continue
        source = str(entry.get("source") or "").strip()
        reference = str(entry.get("reference") or "").strip()
        if not source or not reference:
            continue
        algo = str(entry.get("algo") or "token_set").strip().lower()
        try:
            weight = float(entry.get("weight", 0.0))
        except (TypeError, ValueError):
            weight = 0.0
        if weight > 1:
            weight = weight / 100.0
        weight = min(max(weight, 0.0), 1.0)
        opts = entry.get("opts") or {}
        options = ManualMappingOptions(
            casefold=bool(opts.get("casefold")),
            strip_punctuation=bool(opts.get("strip_punctuation")),
        )
        configs.append(
            ManualMappingConfig(
                source=source,
                reference=reference,
                algo=algo,
                weight=weight,
                opts=options,
            )
        )

    return configs


@dataclass
class ManualRequestContext:
    source_upload_id: str
    reference_upload_id: str
    mapping_configs: List[ManualMappingConfig]
    raw_mappings: List[Dict[str, Any]]
    anchors: Optional[Dict[str, Any]]
    blocks: Dict[str, bool]
    threshold: float
    best_only: bool
    strip_b2c: bool
    limit: Optional[int]
    enable_identity_transforms: bool
    transforms: Dict[str, Any]
    source_id_column: Optional[str]
    reference_id_column: Optional[str]


def _parse_manual_request(
    req: ManualExecuteRequest, *, default_limit: Optional[int] = None
) -> ManualRequestContext:
    if req is None:
        raise HTTPException(400, "Request body required")
    if not req.mappings:
        raise HTTPException(400, "At least one mapping is required")
    source_upload_id = getattr(req.source, "upload_id", None) if req.source else None
    reference_upload_id = (
        getattr(req.reference, "upload_id", None) if req.reference else None
    )
    if not source_upload_id:
        raise HTTPException(400, "source.upload_id is required")
    if not reference_upload_id:
        raise HTTPException(400, "reference.upload_id is required")
    mapping_dicts = [m.model_dump(mode="python") for m in req.mappings]
    mapping_configs = _build_manual_mapping_configs(mapping_dicts)
    if not mapping_configs:
        raise HTTPException(
            400, "At least one mapping must include source and reference columns"
        )
    anchors_payload = req.anchors.model_dump(mode="python") if req.anchors else {}
    anchor_value = str(anchors_payload.get("require_exact_on") or "").strip()
    anchors = (
        {"require_exact_on": anchor_value}
        if anchor_value and anchor_value.lower() != "none"
        else None
    )
    blocks_payload = req.blocks.model_dump(mode="python") if req.blocks else {}
    blocks = {
        "domain": bool(blocks_payload.get("domain")),
        "state": bool(blocks_payload.get("state")),
    }
    threshold = _normalize_manual_threshold(req.threshold)
    best_only = True if req.best_match_only is None else bool(req.best_match_only)
    strip_b2c = True if req.strip_b2c is None else bool(req.strip_b2c)
    limit = req.limit
    if limit is not None:
        try:
            limit = int(limit)
        except (TypeError, ValueError):
            raise HTTPException(400, "limit must be an integer")
        if limit <= 0:
            limit = None
    if limit is None and default_limit is not None:
        limit = int(default_limit)

    transforms_payload = req.transforms or {}
    if not isinstance(transforms_payload, dict):
        transforms_payload = {}

    def _coerce_bool(value, default=False):
        if value is None:
            return default
        if isinstance(value, bool):
            return value
        if isinstance(value, (int, float)):
            return value != 0
        if isinstance(value, str):
            return value.strip().lower() in {"1", "true", "t", "yes", "y", "on"}
        return default

    enable_flag = req.enable_identity_transforms
    if enable_flag is None:
        enable_flag = transforms_payload.get("enable_identity_transforms")
        if enable_flag is None and isinstance(
            transforms_payload.get("enabled"), (bool, int, float, str)
        ):
            enable_flag = transforms_payload.get("enabled")
    transforms_enabled = _coerce_bool(enable_flag, default=False)

    source_flag = transforms_payload.get("enable_source")
    if source_flag is None and isinstance(transforms_payload.get("source"), dict):
        source_flag = transforms_payload["source"].get("enabled")
    reference_flag = transforms_payload.get("enable_reference")
    if reference_flag is None and isinstance(transforms_payload.get("reference"), dict):
        reference_flag = transforms_payload["reference"].get("enabled")

    source_enabled = _coerce_bool(source_flag, default=True)
    reference_enabled = _coerce_bool(reference_flag, default=False)

    if not transforms_enabled:
        source_enabled = False
        reference_enabled = False

    transforms_normalized = {
        "enable_identity_transforms": transforms_enabled,
        "enable_source": source_enabled,
        "enable_reference": reference_enabled,
    }

    profile_value = (
        transforms_payload.get("profile")
        if isinstance(transforms_payload, dict)
        else None
    )
    if isinstance(profile_value, str):
        profile_value = profile_value.strip() or None
    elif profile_value is not None:
        profile_value = str(profile_value)

    dns_override = _coerce_bool(
        transforms_payload.get("enable_dns_validation"), default=None
    )
    if dns_override is None and isinstance(
        transforms_payload.get("company_to_domain"), dict
    ):
        dns_override = _coerce_bool(
            transforms_payload["company_to_domain"].get("use_dns"), default=None
        )

    if profile_value:
        transforms_normalized["profile"] = profile_value
    if dns_override is not None:
        transforms_normalized["enable_dns_validation"] = dns_override

    def _normalize_id_column_name(value: Optional[str]) -> Optional[str]:
        if value is None:
            return None
        text = str(value).strip()
        if not text or text.lower() == "auto":
            return None
        return text

    source_id_column = _normalize_id_column_name(
        getattr(req, "source_id_column", None)
        or (getattr(req.source, "id_column", None) if req.source else None)
    )
    reference_id_column = _normalize_id_column_name(
        getattr(req, "reference_id_column", None)
        or (getattr(req.reference, "id_column", None) if req.reference else None)
    )

    return ManualRequestContext(
        source_upload_id=source_upload_id,
        reference_upload_id=reference_upload_id,
        mapping_configs=mapping_configs,
        raw_mappings=mapping_dicts,
        anchors=anchors,
        blocks=blocks,
        threshold=threshold,
        best_only=best_only,
        strip_b2c=strip_b2c,
        limit=limit,
        enable_identity_transforms=transforms_enabled,
        transforms=transforms_normalized,
        source_id_column=source_id_column,
        reference_id_column=reference_id_column,
    )


def _column_coverage(df: pd.DataFrame, column: Optional[str]) -> Optional[float]:
    if not column or column not in df.columns or df.empty:
        return None
    series = df[column].fillna("").astype(str).str.strip()
    if series.empty:
        return None
    non_empty = series[series != ""]
    total = len(series)
    if total == 0:
        return None
    return float(len(non_empty)) / float(total)


def _format_block_label(label: str, coverage: Optional[float]) -> str:
    if coverage is None:
        return label
    return f"{label} (~{int(round(coverage * 100))}% coverage)"


def _summarize_blocking(
    source_df: pd.DataFrame, ctx: ManualRequestContext
) -> Tuple[Optional[str], Optional[float]]:
    blocks = ctx.blocks or {}
    if not blocks:
        return None, None
    if blocks.get("domain"):
        mapping = next(
            (m for m in ctx.mapping_configs if m.algo in ("domain", "email_domain")),
            None,
        )
        coverage = _column_coverage(source_df, mapping.source if mapping else None)
        return _format_block_label("Domain", coverage), coverage
    if blocks.get("state"):
        state_mapping = None
        for mapping in ctx.mapping_configs:
            src_lower = mapping.source.lower()
            if "state" in src_lower or "province" in src_lower or "region" in src_lower:
                state_mapping = mapping
                break
        coverage = _column_coverage(
            source_df, state_mapping.source if state_mapping else None
        )
        return _format_block_label("State", coverage), coverage
    return None, None


def _has_domain_mapping(ctx: ManualRequestContext) -> bool:
    return any(
        mapping.algo in ("domain", "email_domain") for mapping in ctx.mapping_configs
    )


def _has_name_mapping(ctx: ManualRequestContext) -> bool:
    return any(
        mapping.algo in ("token_set", "jaro", "contains")
        or "name" in mapping.source.lower()
        for mapping in ctx.mapping_configs
    )


def _has_phone_mapping(ctx: ManualRequestContext) -> bool:
    return any("phone" in mapping.source.lower() for mapping in ctx.mapping_configs)


def _has_geo_mapping(ctx: ManualRequestContext, keyword: str) -> bool:
    keyword = keyword.lower()
    return any(keyword in mapping.source.lower() for mapping in ctx.mapping_configs)


def _find_column_by_keywords(
    columns: List[str], keywords: Tuple[str, ...]
) -> Optional[str]:
    ordered = list(columns)
    for keyword in keywords:
        key = keyword.lower()
        for col in ordered:
            if key in col.lower():
                return col
    return None


def _normalize_record_for_json(record: Dict[str, Any]) -> Dict[str, Any]:
    normalized: Dict[str, Any] = {}
    for key, value in record.items():
        if isinstance(value, np.generic):
            normalized[key] = value.item()
        else:
            normalized[key] = value
    return normalized


def _execute_manual_match_or_http_error(**kwargs) -> Tuple[pd.DataFrame, Dict[str, Any]]:
    try:
        return manual_match_execute(**kwargs)
    except ManualMatchValidationError as exc:
        raise HTTPException(400, str(exc)) from exc
    except ManualMatchServiceError as exc:
        raise HTTPException(503, str(exc)) from exc


def _validate_manual_request_or_http_error(ctx: ManualRequestContext) -> None:
    try:
        source_df = _load_manual_dataframe(ctx.source_upload_id, limit=1)
    except FileNotFoundError:
        raise HTTPException(404, f"Source upload {ctx.source_upload_id} not found")
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    try:
        reference_df = _load_manual_dataframe(ctx.reference_upload_id, limit=1)
    except FileNotFoundError:
        raise HTTPException(404, f"Reference upload {ctx.reference_upload_id} not found")
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    try:
        validate_manual_match_configuration(
            source_df=source_df,
            reference_df=reference_df,
            mappings=ctx.mapping_configs,
            blocks=ctx.blocks,
            logger=log,
        )
    except ManualMatchValidationError as exc:
        raise HTTPException(400, str(exc)) from exc


def _suggest_manual_mappings(
    source_df: pd.DataFrame,
    reference_df: pd.DataFrame,
    ctx: ManualRequestContext,
    max_suggestions: int = 5,
) -> List[Dict[str, Any]]:
    if source_df is None or reference_df is None:
        return []

    existing_pairs = {
        (mapping.source.lower(), mapping.reference.lower())
        for mapping in ctx.mapping_configs
    }
    suggestions: List[Dict[str, Any]] = []
    seen_pairs: Set[Tuple[str, str]] = set()
    source_columns = list(source_df.columns)
    reference_columns = list(reference_df.columns)

    blueprints = [
        {
            "label": "Email <-> Domain",
            "source_keywords": ("email", "domain", "website", "url"),
            "reference_keywords": ("domain", "website", "url"),
            "algo": "email_domain",
            "weight": 0.8,
            "skip_if": _has_domain_mapping,
        },
        {
            "label": "Company Name",
            "source_keywords": ("company", "account", "name"),
            "reference_keywords": ("company", "account", "name"),
            "algo": "token_set",
            "weight": 0.6,
            "skip_if": _has_name_mapping,
            "opts": {"casefold": True},
        },
        {
            "label": "Website",
            "source_keywords": ("website", "url"),
            "reference_keywords": ("website", "url", "domain"),
            "algo": "domain",
            "weight": 0.6,
            "skip_if": _has_domain_mapping,
        },
        {
            "label": "Phone",
            "source_keywords": ("phone", "tel", "mobile"),
            "reference_keywords": ("phone", "tel", "mobile"),
            "algo": "exact",
            "weight": 0.2,
            "skip_if": _has_phone_mapping,
        },
        {
            "label": "City",
            "source_keywords": ("city",),
            "reference_keywords": ("city",),
            "algo": "exact",
            "weight": 0.2,
            "skip_if": lambda ctx: _has_geo_mapping(ctx, "city"),
        },
        {
            "label": "State",
            "source_keywords": ("state", "province", "region"),
            "reference_keywords": ("state", "province", "region"),
            "algo": "exact",
            "weight": 0.2,
            "skip_if": lambda ctx: _has_geo_mapping(ctx, "state"),
        },
    ]

    for blueprint in blueprints:
        skip_check = blueprint.get("skip_if")
        if callable(skip_check) and skip_check(ctx):
            continue

        src_col = _find_column_by_keywords(source_columns, blueprint["source_keywords"])
        ref_col = _find_column_by_keywords(
            reference_columns, blueprint["reference_keywords"]
        )
        if not src_col or not ref_col:
            continue

        pair_key = (src_col.lower(), ref_col.lower())
        if pair_key in existing_pairs or pair_key in seen_pairs:
            continue

        suggestion = {
            "source": src_col,
            "reference": ref_col,
            "algo": blueprint["algo"],
            "weight": blueprint["weight"],
            "delta": None,
        }
        if "opts" in blueprint:
            suggestion["opts"] = blueprint["opts"]
        suggestions.append(suggestion)
        seen_pairs.add(pair_key)

        if len(suggestions) >= max_suggestions:
            break

    return suggestions


_HASH_IGNORE_TOP_KEYS = {"hash", "config_hash"}


def _normalize_id_header(name: str | None) -> str:
    if not name:
        return ""
    return re.sub(r"[^a-z0-9]", "", str(name).lower())


def _select_best_id_column(initial: str | None, columns) -> str | None:
    if columns is None:
        return initial
    try:
        is_empty = bool(getattr(columns, "empty", False))
    except Exception:
        is_empty = False
    if is_empty:
        return initial
    column_list = list(columns)
    if not column_list:
        return initial
    if initial and initial in column_list:
        return initial
    normalized = {col: _normalize_id_header(col) for col in column_list}
    if initial:
        norm_initial = _normalize_id_header(initial)
        for col, norm in normalized.items():
            if norm == norm_initial:
                return col
    priority_exact = [
        "18digitaccountid",
        "accountid",
        "recordid",
        "sfid",
        "salesforceid",
        "externalid",
    ]
    for target in priority_exact:
        for col, norm in normalized.items():
            if norm == target:
                return col
    priority_contains = [
        "18digitaccountid",
        "accountid",
        "recordid",
        "sfid",
        "crm",
        "externalid",
    ]
    for target in priority_contains:
        for col, norm in normalized.items():
            if target in norm:
                return col
    # Fall back to first column if nothing matched
    return column_list[0] if column_list else initial


def _ensure_matched_domain_family(
    results_df: pd.DataFrame,
    reference_df: Optional[pd.DataFrame],
    reference_id_col: Optional[str],
) -> pd.DataFrame:
    """Ensure results include a matched_domain_family column using reference data or on-the-fly enrichment."""
    try:
        if results_df is None or results_df.empty:
            return results_df

        family_candidates = [
            "DomainFamily",
            "domain_family",
            "domainfamily",
            "Family",
            "family_id",
            "family",
            "domain_family_id",
            "domain_family_name",
        ]

        added = False

        # Attempt to map directly from reference data if a family column exists
        if reference_df is not None and reference_id_col:
            log.info(
                "Family enrichment: reference columns available: %s",
                list(reference_df.columns),
            )
            reference_columns_lower = {col.lower(): col for col in reference_df.columns}
            ref_id_key = reference_id_col.lower()
            if ref_id_key in reference_columns_lower:
                ref_id_actual = reference_columns_lower[ref_id_key]
                ref_family_actual = next(
                    (
                        reference_columns_lower[name.lower()]
                        for name in family_candidates
                        if name.lower() in reference_columns_lower
                    ),
                    None,
                )
                if ref_family_actual:
                    lookup_df = reference_df[
                        [ref_id_actual, ref_family_actual]
                    ].dropna()
                    log.info(
                        "Family enrichment: using reference family column %s with %d rows",
                        ref_family_actual,
                        len(lookup_df),
                    )
                    if not lookup_df.empty:
                        key_series = lookup_df[ref_id_actual].astype(str).str.strip()
                        family_lookup = dict(
                            zip(key_series, lookup_df[ref_family_actual])
                        )
                        results_columns_lower = {
                            col.lower(): col for col in results_df.columns
                        }
                        id_fields = [
                            "reference_id",
                            "matched_id",
                            "matched_account_id",
                            ref_id_actual,
                        ]
                        for field in id_fields:
                            field_key = field.lower()
                            actual_field = results_columns_lower.get(field_key)
                            if actual_field:
                                log.info(
                                    "Family enrichment: mapping using results field %s",
                                    actual_field,
                                )
                                results_df["matched_domain_family"] = (
                                    results_df[actual_field]
                                    .astype(str)
                                    .str.strip()
                                    .map(family_lookup)
                                )
                                added = True
                                break

        # Fallback: compute domain family from domain-like columns in the results
        if not added or "matched_domain_family" not in results_df.columns:
            domain_candidates = [
                "matched_domain",
                "reference_domain",
                "matched_domain_clean",
                "domain",
                "domain_append",
                "ref_domain",
                "r_domain_append",
                "r_dhc_provider_website",
                "dhc_provider_website",
                "reference_website",
                "matched_website",
                "website",
            ]
            results_columns_lower = {col.lower(): col for col in results_df.columns}
            domain_field_key = next(
                (name for name in domain_candidates if name in results_columns_lower),
                None,
            )
            if domain_field_key:
                domain_field = results_columns_lower[domain_field_key]
                log.info(
                    "Family enrichment: attempting domain lookup using field %s",
                    domain_field,
                )
                temp_df = results_df[[domain_field]].copy()
                temp_df.rename(columns={domain_field: "Domain"}, inplace=True)
                temp_df["Domain"] = temp_df["Domain"].astype(str).str.strip()
                temp_df["Domain"] = temp_df["Domain"].apply(
                    lambda v: extract_domain_from_url(v) or v
                )
                enriched_df, _ = enrich_with_domain_families(temp_df)
                if "DomainFamily" in enriched_df.columns:
                    results_df["matched_domain_family"] = enriched_df["DomainFamily"]
                    added = True
                    log.info(
                        "Family enrichment: domain lookup produced %d families",
                        results_df["matched_domain_family"].notna().sum(),
                    )

        if added and "matched_domain_family" in results_df.columns:
            non_null = results_df["matched_domain_family"].notna().sum()
            results_df["matched_domain_family"] = results_df[
                "matched_domain_family"
            ].fillna("")
            log.info(
                "Family enrichment: final matched_domain_family coverage %d/%d",
                int(non_null),
                len(results_df),
            )
        else:
            log.info("Family enrichment: no domain family data available")

        return results_df
    except Exception as exc:
        log.warning(f"Failed to add matched domain family column: {exc}")
        return results_df


# Use the unified hash function from intelligence.py instead of a local implementation
# This ensures preview and run use the exact same hash algorithm
_intelligence_hash_payload = _stable_intelligence_hash


# Debug endpoint to test request format
@router.post("/run/debug")
async def run_job_debug(request: Request):
    """Debug endpoint that accepts any JSON and logs it"""
    try:
        body = await request.json()
        log.info(f"[RUN DEBUG] Received body: {json.dumps(body, indent=2)}")

        # Try to parse as RunReq
        try:
            from pydantic import ValidationError

            req = RunReq(**body)
            log.info("[RUN DEBUG] Successfully parsed as RunReq")
            return {"status": "valid", "parsed": req.model_dump()}
        except ValidationError as e:
            log.error(f"[RUN DEBUG] Validation failed: {e.errors()}")
            return {"status": "invalid", "errors": e.errors(), "received": body}
    except Exception as e:
        log.error(f"[RUN DEBUG] Failed to parse JSON: {e}")
        return {"status": "error", "message": str(e)}


JUSTCALL_IDEMPOTENCY_KEY = "justcall:job:{account_id}:{idempotency_key}"
JUSTCALL_RESULT_TTL = 86400
HEYREACH_IDEMPOTENCY_KEY = "heyreach:job:{account_id}:{idempotency_key}"
HEYREACH_RESULT_TTL = 86400


async def _run_justcall_job(
    *,
    req: RunReq,
    response: Response,
    idempotency_key: Optional[str],
    db: AsyncSession,
    account_id: UUID,
) -> RunResp:
    if response is not None:
        response.headers["Cache-Control"] = "no-store"
        if idempotency_key:
            response.headers["Idempotency-Key"] = idempotency_key

    if not req.config:
        raise HTTPException(400, "config is required for justcall_push")

    try:
        config_model = JustCallConfig.model_validate(req.config)
    except ValidationError as exc:
        raise HTTPException(status_code=400, detail=exc.errors()) from exc

    rows = req.rows or []
    upload_id = req.upload_id or config_model.upload_id
    if not rows and not upload_id:
        raise HTTPException(400, "upload_id or rows required for justcall_push")
    if rows and len(rows) > 500 and not upload_id:
        raise HTTPException(400, "Inline rows capped at 500; use upload_id instead")

    config_payload = config_model.model_dump()
    if upload_id:
        config_payload["upload_id"] = upload_id

    row_count = len(rows)
    if row_count <= 0:
        row_count = int(config_payload.get("row_count") or 0)
    if row_count <= 0:
        row_count = int((config_payload.get("source") or {}).get("row_count") or 0)

    gateway = UsageGateway(db)
    consume = await gateway.check_and_consume(
        account_id=str(account_id),
        resource="byo_rows",
        amount=max(row_count, 1),
        source="activation.justcall.push",
        idempotency_key=idempotency_key,
    )
    if not consume.allowed:
        raise HTTPException(
            status_code=429,
            detail={
                "error": "BYO_LIMIT_EXCEEDED",
                "message": consume.message or "BYO daily limit reached.",
                "limit": consume.limit,
                "used": consume.used,
                "remaining": consume.remaining,
                "reset_at": consume.reset_at.isoformat() if consume.reset_at else None,
            },
        )

    try:
        redis_conn = await get_redis()
    except Exception:
        redis_conn = None

    idem_key = idempotency_key
    if not idem_key:
        try:
            payload = {
                "config": config_payload,
                "upload_id": upload_id,
                "row_count": row_count,
            }
            idem_key = hashlib.sha256(
                json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
            ).hexdigest()[:32]
        except Exception:
            idem_key = None

    if redis_conn and idem_key:
        hit = await redis_conn.get(
            JUSTCALL_IDEMPOTENCY_KEY.format(
                account_id=account_id, idempotency_key=idem_key
            )
        )
        if hit:
            job_id_existing = hit.decode() if hasattr(hit, "decode") else hit
            response.status_code = 200
            response.headers["Location"] = f"/api/v2/jobs/{job_id_existing}"
            return RunResp(job_id=job_id_existing)

    job_id = str(uuid4())
    job_config = {
        "job_type": "justcall_push",
        "config": config_payload,
        "rows": rows,
        "upload_id": upload_id,
        "row_count": row_count,
    }

    job = Job(
        id=job_id,
        account_id=str(account_id),
        status=JobStatus.QUEUED,
        job_type=JobType.MATCH,
        config=job_config,
    )
    db.add(job)
    await db.commit()

    if redis_conn:
        meta = {
            "account_id": str(account_id),
            "created_at": datetime.utcnow().isoformat(),
            "total_rows": row_count,
            "job_type": "justcall_push",
            "config": config_payload,
        }
        await redis_conn.setex(
            JUSTCALL_META_KEY.format(job_id=job_id), JUSTCALL_RESULT_TTL, json.dumps(meta)
        )
        await redis_conn.setex(
            JUSTCALL_STATE_KEY.format(job_id=job_id), JUSTCALL_RESULT_TTL, "queued"
        )
        await redis_conn.setex(
            JUSTCALL_PROGRESS_KEY.format(job_id=job_id), JUSTCALL_RESULT_TTL, "0"
        )
        await redis_conn.setex(
            JUSTCALL_SUMMARY_KEY.format(job_id=job_id), JUSTCALL_RESULT_TTL, json.dumps({})
        )
        await redis_conn.setex(
            JUSTCALL_ERRORS_KEY.format(job_id=job_id),
            JUSTCALL_RESULT_TTL,
            json.dumps({"errors": [], "errors_truncated": False}),
        )
        await redis_conn.setex(
            f"job:{job_id}",
            JUSTCALL_RESULT_TTL,
            json.dumps({"status": "queued", "progress": 0, "processed_rows": 0}),
        )
        if idem_key:
            await redis_conn.setex(
                JUSTCALL_IDEMPOTENCY_KEY.format(
                    account_id=account_id, idempotency_key=idem_key
                ),
                JUSTCALL_RESULT_TTL,
                job_id,
            )

    fm_inline_value = os.getenv("FM_PROCESS_IN_APP")
    if fm_inline_value == "1":
        job.status = JobStatus.PROCESSING
        await db.commit()
        worker = JustCallWorker(
            account_id=str(account_id),
            job_id=job_id,
            config=config_payload,
            rows=rows,
            redis=redis_conn,
        )
        asyncio.create_task(worker.run())
    else:
        try:
            from ...worker import conn as _rq_conn
            from rq import Queue
            from anyio.to_thread import run_sync

            async def _enqueue():
                def _do():
                    Queue("default", connection=_rq_conn).enqueue(
                        "fmatch.saas.worker.run_matching_job",
                        job_id,
                        job_timeout=_settings.JOB_TIMEOUT_DEFAULT,
                    )

                await run_sync(_do)

            asyncio.create_task(_enqueue())
        except Exception as exc:
            log.error("Failed to enqueue JustCall job %s: %s", job_id, exc)
            raise HTTPException(500, "Failed to enqueue JustCall job")

    response.headers["Location"] = f"/api/v2/jobs/{job_id}"
    return RunResp(job_id=job_id)


async def _run_heyreach_job(
    *,
    req: RunReq,
    response: Response,
    idempotency_key: Optional[str],
    db: AsyncSession,
    account_id: UUID,
) -> RunResp:
    if response is not None:
        response.headers["Cache-Control"] = "no-store"
        if idempotency_key:
            response.headers["Idempotency-Key"] = idempotency_key

    if not req.config:
        raise HTTPException(400, "config is required for heyreach_push")

    try:
        config_model = HeyReachConfig.model_validate(req.config)
    except ValidationError as exc:
        raise HTTPException(status_code=400, detail=exc.errors()) from exc

    rows = req.rows or []
    upload_id = req.upload_id or config_model.upload_id
    if not rows and not upload_id:
        raise HTTPException(400, "upload_id or rows required for heyreach_push")
    if rows and len(rows) > 500 and not upload_id:
        raise HTTPException(400, "Inline rows capped at 500; use upload_id instead")

    config_payload = config_model.model_dump()
    if upload_id:
        config_payload["upload_id"] = upload_id

    row_count = len(rows)
    if row_count <= 0:
        row_count = int(config_payload.get("row_count") or 0)
    if row_count <= 0:
        row_count = int((config_payload.get("source") or {}).get("row_count") or 0)

    gateway = UsageGateway(db)
    consume = await gateway.check_and_consume(
        account_id=str(account_id),
        resource="byo_rows",
        amount=max(row_count, 1),
        source="activation.heyreach.push",
        idempotency_key=idempotency_key,
    )
    if not consume.allowed:
        raise HTTPException(
            status_code=429,
            detail={
                "error": "BYO_LIMIT_EXCEEDED",
                "message": consume.message or "BYO daily limit reached.",
                "limit": consume.limit,
                "used": consume.used,
                "remaining": consume.remaining,
                "reset_at": consume.reset_at.isoformat() if consume.reset_at else None,
            },
        )

    try:
        redis_conn = await get_redis()
    except Exception:
        redis_conn = None

    idem_key = idempotency_key
    if not idem_key:
        try:
            payload = {
                "config": config_payload,
                "upload_id": upload_id,
                "row_count": row_count,
            }
            idem_key = hashlib.sha256(
                json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
            ).hexdigest()[:32]
        except Exception:
            idem_key = None

    if redis_conn and idem_key:
        hit = await redis_conn.get(
            HEYREACH_IDEMPOTENCY_KEY.format(
                account_id=account_id, idempotency_key=idem_key
            )
        )
        if hit:
            job_id_existing = hit.decode() if hasattr(hit, "decode") else hit
            response.status_code = 200
            response.headers["Location"] = f"/api/v2/jobs/{job_id_existing}"
            return RunResp(job_id=job_id_existing)

    job_id = str(uuid4())
    job_config = {
        "job_type": "heyreach_push",
        "config": config_payload,
        "rows": rows,
        "upload_id": upload_id,
        "row_count": row_count,
    }

    job = Job(
        id=job_id,
        account_id=str(account_id),
        status=JobStatus.QUEUED,
        job_type=JobType.MATCH,
        config=job_config,
    )
    db.add(job)
    await db.commit()

    if redis_conn:
        meta = {
            "account_id": str(account_id),
            "created_at": datetime.utcnow().isoformat(),
            "total_rows": row_count,
            "job_type": "heyreach_push",
            "config": config_payload,
        }
        await redis_conn.setex(
            HEYREACH_META_KEY.format(job_id=job_id), HEYREACH_RESULT_TTL, json.dumps(meta)
        )
        await redis_conn.setex(
            HEYREACH_STATE_KEY.format(job_id=job_id), HEYREACH_RESULT_TTL, "queued"
        )
        await redis_conn.setex(
            HEYREACH_PROGRESS_KEY.format(job_id=job_id), HEYREACH_RESULT_TTL, "0"
        )
        await redis_conn.setex(
            HEYREACH_SUMMARY_KEY.format(job_id=job_id), HEYREACH_RESULT_TTL, json.dumps({})
        )
        await redis_conn.setex(
            HEYREACH_ERRORS_KEY.format(job_id=job_id),
            HEYREACH_RESULT_TTL,
            json.dumps({"errors": [], "errors_truncated": False}),
        )
        await redis_conn.setex(
            f"job:{job_id}",
            HEYREACH_RESULT_TTL,
            json.dumps({"status": "queued", "progress": 0, "processed_rows": 0}),
        )
        if idem_key:
            await redis_conn.setex(
                HEYREACH_IDEMPOTENCY_KEY.format(
                    account_id=account_id, idempotency_key=idem_key
                ),
                HEYREACH_RESULT_TTL,
                job_id,
            )

    fm_inline_value = os.getenv("FM_PROCESS_IN_APP")
    if fm_inline_value == "1":
        job.status = JobStatus.PROCESSING
        await db.commit()
        worker = HeyReachWorker(
            account_id=str(account_id),
            job_id=job_id,
            config=config_payload,
            rows=rows,
            redis=redis_conn,
        )
        asyncio.create_task(worker.run())
    else:
        try:
            from ...worker import conn as _rq_conn
            from rq import Queue
            from anyio.to_thread import run_sync

            async def _enqueue():
                def _do():
                    Queue("default", connection=_rq_conn).enqueue(
                        "fmatch.saas.worker.run_matching_job",
                        job_id,
                        job_timeout=_settings.JOB_TIMEOUT_DEFAULT,
                    )

                await run_sync(_do)

            asyncio.create_task(_enqueue())
        except Exception as exc:
            log.error("Failed to enqueue HeyReach job %s: %s", job_id, exc)
            raise HTTPException(500, "Failed to enqueue HeyReach job")

    response.headers["Location"] = f"/api/v2/jobs/{job_id}"
    return RunResp(job_id=job_id)


@router.post("/run", response_model=RunResp, status_code=201)
async def run_job(
    req: RunReq,
    response: Response,
    idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"),
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    # Debug logging for Google Sheets issues
    log.info("[DEBUG] /api/v2/jobs/run received request")
    log.info(f"[DEBUG] Request body: {req.model_dump_json()}")
    intel_profile = getattr(req.intelligence, "profile", "MISSING") if req.intelligence else "MISSING"
    intel_threshold = getattr(req.intelligence, "threshold", "MISSING") if req.intelligence else "MISSING"
    log.info(
        f"[DEBUG] Intelligence: profile={intel_profile}, threshold={intel_threshold}"
    )

    job_type_input = (req.job_type or req.mode or "").strip().lower()
    if job_type_input == "justcall_push":
        return await _run_justcall_job(
            req=req,
            response=response,
            idempotency_key=idempotency_key,
            db=db,
            account_id=account_id,
        )
    if job_type_input == "heyreach_push":
        return await _run_heyreach_job(
            req=req,
            response=response,
            idempotency_key=idempotency_key,
            db=db,
            account_id=account_id,
        )

    if not (req.input and (req.input.object_id or req.input.uri)):
        raise HTTPException(400, "input.object_id or input.uri required")
    if not req.intelligence:
        raise HTTPException(400, "intelligence is required")

    intel_dict = req.intelligence.model_dump(mode="python")
    submitted_hash = (
        intel_dict.get("config_hash")
        or intel_dict.get("hash")
        or (intel_dict.get("options") or {}).get("config_hash")
    )
    if not submitted_hash:
        raise HTTPException(
            400,
            "intelligence.config_hash missing. Run preview before /run.",
        )

    calculated_hash = _intelligence_hash_payload(intel_dict)
    redis_conn_hash_cache = None
    try:
        redis_conn_hash_cache = await get_redis()
    except Exception:
        redis_conn_hash_cache = None
    if redis_conn_hash_cache:
        try:
            cache_payload = json.dumps(
                intel_dict, separators=(",", ":"), ensure_ascii=False, default=str
            )
            await redis_conn_hash_cache.set(
                f"intelligence:{calculated_hash}", cache_payload, ex=86400
            )
            await redis_conn_hash_cache.set(
                f"intelligence:{calculated_hash[:16]}", cache_payload, ex=86400
            )
            await redis_conn_hash_cache.set(
                f"intel:index:short:{calculated_hash[:16]}", calculated_hash, ex=86400
            )
            log.info(
                "[RUN] Cached intelligence config full=%s short=%s",
                calculated_hash,
                calculated_hash[:16],
            )
        except Exception as cache_exc:
            log.warning("[RUN] Failed to cache intelligence config: %s", cache_exc)
    try:
        log.info(
            "jobs/run: client hash=%s computed hash=%s", submitted_hash, calculated_hash
        )
    except Exception as log_exc:
        log.warning("jobs/run: could not log client hash: %s", log_exc)
    if calculated_hash != submitted_hash:
        log.warning(
            "intelligence hash mismatch: submitted=%s calculated=%s",
            submitted_hash,
            calculated_hash,
        )
        # TEMP during stabilization: log but proceed
        log.info("Proceeding despite hash mismatch (temporary soft-fail mode)")
        try:
            log.debug(
                "intel_for_hash=%s",
                json.dumps(intel_dict, sort_keys=True, default=str)[:2000],
            )
        except Exception:
            pass

    # Headers
    response.headers["Cache-Control"] = "no-store"
    if idempotency_key:
        response.headers["Idempotency-Key"] = idempotency_key

    # Idempotency short-circuit (tolerate missing Redis)
    try:
        redis_conn = await get_redis()
    except Exception:
        redis_conn = None
    if idempotency_key and redis_conn:
        hit = await redis_conn.get(f"idem_v2:{account_id}:{idempotency_key}")
        if hit:
            job_id_existing = hit.decode() if hasattr(hit, "decode") else hit
            response.status_code = 200
            response.headers["Location"] = f"/api/v2/jobs/{job_id_existing}"
            return RunResp(job_id=job_id_existing, echo_config_hash=submitted_hash)

    # Resolve source path the worker can read
    mode_input = req.mode or req.job_type or "match"
    mode = str(mode_input).strip().lower()
    if mode not in ("match", "dedupe"):
        raise HTTPException(400, "mode must be 'match' or 'dedupe'")

    if req.input.uri:
        source_url = req.input.uri
    else:
        source_url = _resolve_upload_path(req.input.object_id)

    if req.input.reference_uri:
        reference_url = req.input.reference_uri
    else:
        reference_url = _resolve_upload_path(req.input.reference_object_id)

    if mode == "match" and not reference_url:
        raise HTTPException(
            400, "reference_object_id or reference_uri required for match mode"
        )

    cfg: Dict[str, Any] = {
        "mode": mode,
        "source_url": source_url,
        "threshold": float(req.intelligence.threshold),
        "intelligence": intel_dict,
    }
    options = dict(intel_dict.get("options") or {})
    source_id_override = intel_dict.get("source_id_column") or intel_dict.get(
        "sourceIdColumn"
    )
    reference_id_override = intel_dict.get("reference_id_column") or intel_dict.get(
        "referenceIdColumn"
    )
    if not source_id_override:
        source_id_override = options.get("source_id_column") or options.get(
            "sourceIdColumn"
        )
    if not reference_id_override:
        reference_id_override = options.get("reference_id_column") or options.get(
            "referenceIdColumn"
        )

    if source_id_override:
        cfg["source_id_column"] = source_id_override
        options["source_id_column"] = source_id_override
        intel_dict["source_id_column"] = source_id_override
    if reference_id_override:
        cfg["reference_id_column"] = reference_id_override
        options["reference_id_column"] = reference_id_override
        intel_dict["reference_id_column"] = reference_id_override
    if source_id_override or reference_id_override:
        intel_dict["options"] = options

    maps = (
        intel_dict.get("mappings") or options.get("maps") or options.get("column_maps")
    )
    if isinstance(maps, list) and maps:
        cfg["mappings"] = maps  # FIXED: Changed from "maps" to "mappings" to match execute_job expectation
    cfg["apply_blocking"] = bool(
        intel_dict.get("apply_blocking", options.get("apply_blocking", True))
    )
    block_col = options.get("block_col") or options.get("blocking_column")
    if block_col:
        cfg["block_col"] = block_col
    src_block = options.get("source_block_col")
    if src_block:
        cfg["source_block_col"] = src_block
    ref_block = options.get("ref_block_col")
    if ref_block:
        cfg["ref_block_col"] = ref_block
    mapping_warnings = intel_dict.get("mapping_warnings")
    if isinstance(mapping_warnings, list) and mapping_warnings:
        cfg["mapping_warnings"] = mapping_warnings
    if mode == "match":
        cfg["reference_url"] = reference_url
    # For dedupe mode, add required fields with defaults
    elif mode == "dedupe":
        cfg["ruleset_name"] = "default"
        cfg["block_key_length"] = _settings.DEFAULT_BLOCK_KEY_LENGTH
        cfg["dup_block_limit"] = _settings.DEFAULT_DUP_BLOCK_LIMIT
        cfg.setdefault("apply_blocking", True)
        cfg.setdefault("block_col", "Domain")
    if req.input.union:
        cfg["union"] = list(req.input.union)

    cfg["config_hash"] = submitted_hash

    # Persist row count hints from preview/intelligence
    # Check both intel_dict (top level) and options (nested) for row count
    source_row_count_hint = int(
        intel_dict.get("source_row_count") or options.get("source_row_count", 0) or 0
    )
    reference_row_count_hint = int(
        intel_dict.get("reference_row_count") or options.get("reference_row_count", 0) or 0
    )
    if source_row_count_hint > 0:
        cfg["source_row_count"] = source_row_count_hint
    if mode == "match" and reference_row_count_hint >= 0:
        cfg["reference_row_count"] = reference_row_count_hint

    job_id = str(uuid4())

    # Create job
    job = Job(
        id=job_id,
        account_id=str(account_id),
        status=JobStatus.QUEUED,
        job_type=JobType.DEDUPE if mode == "dedupe" else JobType.MATCH,
        config=cfg,
    )
    assert (
        job.config.get("source_row_count", 0) > 0
    ), f"Job {job_id} config missing source_row_count - billing will fail"
    db.add(job)
    await db.commit()

    # Seed Redis status and idempotency (best-effort)
    if redis_conn:
        meta = {"status": "queued", "mode": "match", "progress": 0, "processed_rows": 0}
        await redis_conn.set(f"job:{job_id}", json.dumps(meta), ex=86400)

        key = (
            idempotency_key
            or hashlib.sha256(
                json.dumps(req.model_dump(), sort_keys=True).encode()
            ).hexdigest()[:32]
        )
        await redis_conn.set(f"idem_v2:{account_id}:{key}", job_id, ex=86400)

    # Check if we should process inline with FM_PROCESS_IN_APP
    fm_inline_value = os.getenv("FM_PROCESS_IN_APP")
    log.info(f"[DEBUG] Job {job_id} - FM_PROCESS_IN_APP environment variable value: {fm_inline_value!r}")
    if fm_inline_value == "1":
        log.info(f"FM_PROCESS_IN_APP=1, executing job {job_id} synchronously")

        # Update job status to processing
        job.status = JobStatus.PROCESSING
        await db.commit()

        # Update Redis metadata with input object IDs
        if redis_conn:
            meta = {
                "status": "processing",
                "mode": mode,
                "progress": 5,
                "processed_rows": 0,
                "input": {
                    "object_id": req.input.object_id,
                    "source_object_id": req.input.object_id,
                    "reference_object_id": req.input.reference_object_id,
                },
                "source_object_id": req.input.object_id,
                "reference_object_id": req.input.reference_object_id,
            }
            await redis_conn.set(f"job:{job_id}", json.dumps(meta), ex=86400)

        # Prepare execute request
        execute_request = {
            "intelligence": intel_dict,
            "configuration": cfg,  # FIXED: Pass full config with mappings, not just hash
        }

        try:
            # Execute the job inline (execute_job is defined later in this file)
            result = await execute_job(
                job_id=job_id,
                request_body=execute_request,
                background_tasks=None,
                session=db,
                account_id=account_id,
            )
            log.info(f"Job {job_id} executed inline successfully")
        except Exception as e:
            log.error(f"Failed to execute job {job_id} inline: {e}")
            # Update job status to failed
            job.status = JobStatus.FAILED
            await db.commit()
            if redis_conn:
                meta = {"status": "failed", "error": str(e)}
                await redis_conn.set(f"job:{job_id}", json.dumps(meta), ex=86400)
            # Still return the job_id but with error status
    else:
        # Enqueue RQ job for background processing
        try:
            from ...worker import conn as _rq_conn
            from rq import Queue
            from anyio.to_thread import run_sync

            async def _enqueue():
                def _do():
                    Queue("default", connection=_rq_conn).enqueue(
                        "fmatch.saas.worker.run_matching_job",
                        job_id,
                        job_timeout=_settings.JOB_TIMEOUT_DEFAULT,
                    )

                return await run_sync(_do)

            await _enqueue()
        except Exception as e:
            if redis_conn:
                try:
                    await redis_conn.delete(f"job:{job_id}")
                except Exception:
                    pass
            # In development, allow returning job_id even if enqueue fails (test friendliness)
            if getattr(_settings, "ENV", "development").lower() in (
                "dev",
                "development",
            ):
                log.warning(f"enqueue_failed in dev; returning job_id anyway: {e}")
            else:
                raise HTTPException(500, f"enqueue_failed: {e}")

    response.headers["Location"] = f"/api/v2/jobs/{job_id}"
    return RunResp(job_id=job_id, echo_config_hash=submitted_hash)


async def _persist_manual_job_failure(job_id: str, error: str) -> None:
    try:
        async with AsyncSessionLocal() as session:
            job = await session.get(Job, job_id)
            if job:
                job.status = JobStatus.FAILED
                job.error = error
                await session.commit()
    except Exception:
        log.exception("[MANUAL JOB] Failed to persist failure for %s", job_id)


async def _persist_manual_job_completion(job_id: str) -> None:
    meta: Dict[str, Any] = {}
    try:
        redis_conn = await get_redis()
    except Exception:
        redis_conn = None
    else:
        if redis_conn:
            try:
                meta_raw = await redis_conn.get(f"job:{job_id}")
            except Exception:
                meta_raw = None
            if meta_raw:
                try:
                    meta = json.loads(meta_raw)
                except Exception:
                    meta = {}
    try:
        async with AsyncSessionLocal() as session:
            job = await session.get(Job, job_id)
            if not job:
                return
            matches_found = int(
                meta.get("matches_found") or meta.get("total_matches") or 0
            )
            job.matches_found = matches_found
            result_object_id = meta.get("result_object_id")
            if result_object_id:
                job.results_path = result_object_id
            job.status = JobStatus.COMPLETED
            job.error = None
            job.config = job.config or {}
            await session.commit()

            # Track usage quota based on source rows processed
            if job.account_id:
                try:
                    from ...middleware.quota_checker import record_quota_usage

                    config_dict = job.config or {}
                    rows_processed_raw = (
                        config_dict.get("source_row_count")
                        or config_dict.get("source_rows")
                        or config_dict.get("rows_processed")
                    )
                    try:
                        rows_processed = int(rows_processed_raw or 0)
                    except (TypeError, ValueError):
                        rows_processed = 0

                    if rows_processed == 0:
                        meta_rows = meta.get("processed_rows") or meta.get(
                            "rows_processed"
                        )
                        try:
                            rows_processed = int(meta_rows or 0)
                        except (TypeError, ValueError):
                            rows_processed = 0

                    used_fallback_matches = False
                    if rows_processed == 0 and matches_found > 0:
                        log.warning(
                            f"[QUOTA] Job {job_id} missing source_row_count in config, "
                            f"falling back to matches_found={matches_found}"
                        )
                        rows_processed = matches_found
                        used_fallback_matches = True

                    if rows_processed > 0:
                        await record_quota_usage(
                            account_id=str(job.account_id),
                            db=session,
                            rows_returned=rows_processed,
                            credits_consumed=0,  # No enrichment in match jobs
                            used_grace=False,
                        )
                        log.info(
                            f"[QUOTA] Billed {rows_processed} source rows "
                            f"(matches_found={matches_found}, fallback={used_fallback_matches}) "
                            f"for job {job_id}, account {job.account_id}"
                        )
                except Exception as quota_err:
                    log.warning(
                        f"[QUOTA] Failed to track usage for job {job_id}: {quota_err}"
                    )
    except Exception:
        log.exception("[MANUAL JOB] Failed to persist completion for %s", job_id)


async def _run_manual_job_async(job_id: str) -> None:
    try:
        process_job_v2 = _get_process_job_v2()
        await asyncio.to_thread(process_job_v2, job_id)
    except Exception as exc:
        message = str(exc)
        try:
            redis_conn = await get_redis()
        except Exception:
            redis_conn = None
        else:
            if redis_conn:
                try:
                    meta_raw = await redis_conn.get(f"job:{job_id}")
                except Exception:
                    meta_raw = None
                if meta_raw:
                    try:
                        meta = json.loads(meta_raw)
                    except Exception:
                        meta = {}
                else:
                    meta = {}
                meta.update({"status": "failed", "state": "failed", "error": message})
                try:
                    await redis_conn.set(f"job:{job_id}", json.dumps(meta), ex=86400)
                except Exception:
                    log.debug(
                        "[MANUAL JOB] Unable to update failure metadata for %s", job_id
                    )
        await _persist_manual_job_failure(job_id, message)
        log.error("Manual match job %s failed", job_id, exc_info=True)
    else:
        # Ensure database is updated before continuing
        await _persist_manual_job_completion(job_id)
        log.info(f"[MANUAL JOB] Job {job_id} completed and persisted to database")


@manual_router.post("/preview")
async def manual_preview(
    req: ManualExecuteRequest,
    account_id: UUID = Depends(get_current_account),
):
    ctx = _parse_manual_request(req, default_limit=50)
    limit = ctx.limit or 50

    try:
        source_df = _load_manual_dataframe(ctx.source_upload_id, limit=limit)
    except FileNotFoundError:
        raise HTTPException(404, f"Source upload {ctx.source_upload_id} not found")
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    try:
        reference_df = _load_manual_dataframe(ctx.reference_upload_id)
    except FileNotFoundError:
        raise HTTPException(
            404, f"Reference upload {ctx.reference_upload_id} not found"
        )
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    domain_registry = _load_domain_registry_safe()

    result_df, metrics = _execute_manual_match_or_http_error(
        source_df=source_df,
        reference_df=reference_df,
        mappings=ctx.mapping_configs,
        anchors=ctx.anchors,
        blocks=ctx.blocks,
        threshold=ctx.threshold,
        best_only=ctx.best_only,
        strip_b2c=ctx.strip_b2c,
        limit=limit,
        logger=log,
        source_id_column=ctx.source_id_column,
        reference_id_column=ctx.reference_id_column,
        domain_registry=domain_registry,
    )

    if limit:
        result_df = result_df.head(limit)

    return {
        "preview": True,
        "rows": result_df.to_dict(orient="records"),
        "metrics": metrics,
        "manual_mode": True,
        "mappings_count": len(ctx.mapping_configs),
        "limit": limit,
        "rowCounts": {
            "source": int(len(source_df)),
            "reference": int(len(reference_df)),
        },
        "estimatedCredits": int(len(source_df)),
    }


@manual_router.post("/impact")
async def manual_impact(
    req: ManualExecuteRequest,
    account_id: UUID = Depends(get_current_account),
):
    ctx = _parse_manual_request(req, default_limit=300)
    sample_limit = ctx.limit or 300

    try:
        full_source_df = _load_manual_dataframe(ctx.source_upload_id)
    except FileNotFoundError:
        raise HTTPException(404, f"Source upload {ctx.source_upload_id} not found")
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    total_rows = len(full_source_df)
    source_sample = (
        full_source_df.head(sample_limit)
        if sample_limit and sample_limit < total_rows
        else full_source_df
    )

    try:
        reference_df = _load_manual_dataframe(ctx.reference_upload_id)
    except FileNotFoundError:
        raise HTTPException(
            404, f"Reference upload {ctx.reference_upload_id} not found"
        )
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    domain_registry = _load_domain_registry_safe()

    result_df, metrics = _execute_manual_match_or_http_error(
        source_df=source_sample,
        reference_df=reference_df,
        mappings=ctx.mapping_configs,
        anchors=ctx.anchors,
        blocks=ctx.blocks,
        threshold=ctx.threshold,
        best_only=ctx.best_only,
        strip_b2c=ctx.strip_b2c,
        limit=sample_limit,
        logger=log,
        source_id_column=ctx.source_id_column,
        reference_id_column=ctx.reference_id_column,
        domain_registry=domain_registry,
    )

    sample_rows = int(metrics.get("rows_evaluated") or len(source_sample))
    reference_rows = int(len(reference_df))
    estimated_pairs_before = int(total_rows * reference_rows)
    sample_pairs_before = int(sample_rows * reference_rows)
    account_match_ids = set(
        result_df.loc[result_df["match_status"] == "Account Match", "source_id"].astype(
            str
        )
    )
    possible_match_ids = set(
        result_df.loc[
            result_df["match_status"] == "Possible Match", "source_id"
        ].astype(str)
    )
    high_conf_ids: Set[str] = set()
    if "match_score" in result_df.columns:
        high_conf_ids = set(
            result_df.loc[result_df["match_score"] >= 90, "source_id"].astype(str)
        )

    match_rate = (len(account_match_ids) / sample_rows) if sample_rows else 0.0
    est_matches = (
        int(round(match_rate * total_rows)) if total_rows else len(account_match_ids)
    )
    est_high_conf = (
        int(round((len(high_conf_ids) / sample_rows) * total_rows))
        if sample_rows and total_rows
        else len(high_conf_ids)
    )

    estimated_pairs_after = metrics.get("engine_projected_pairs")
    strategy_details = metrics.get("engine_block_strategy_details")
    if estimated_pairs_after is None and isinstance(strategy_details, dict):
        estimated_pairs_after = strategy_details.get("estimated_pairs_after")
        if estimated_pairs_after is None and isinstance(
            strategy_details.get("fallback_strategy"), dict
        ):
            estimated_pairs_after = strategy_details["fallback_strategy"].get(
                "estimated_pairs_after"
            )
    try:
        estimated_pairs_after = (
            int(round(float(estimated_pairs_after)))
            if estimated_pairs_after is not None
            else None
        )
    except (TypeError, ValueError):
        estimated_pairs_after = None

    scaled_pairs_after = estimated_pairs_after
    if (
        estimated_pairs_after is not None
        and sample_rows
        and total_rows
        and sample_rows != total_rows
    ):
        scaled_pairs_after = int(round(estimated_pairs_after * (total_rows / sample_rows)))

    block_label, block_coverage = _summarize_blocking(full_source_df, ctx)
    suggestions = _suggest_manual_mappings(full_source_df, reference_df, ctx)

    metrics.update(
        {
            "sample_rows": sample_rows,
            "sample_account_matches": len(account_match_ids),
            "sample_possible_matches": len(possible_match_ids),
            "sample_high_confidence": len(high_conf_ids),
            "total_rows": total_rows,
        }
    )

    return {
        "impact": True,
        "manual_mode": True,
        "sample_rows": sample_rows,
        "total_rows": total_rows,
        "sample_account_matches": len(account_match_ids),
        "sample_possible_matches": len(possible_match_ids),
        "sample_high_confidence": len(high_conf_ids),
        "estMatches": est_matches,
        "estRate": match_rate,
        "estHighConf": est_high_conf,
        "referenceRows": reference_rows,
        "estimatedPairsBefore": estimated_pairs_before,
        "samplePairsBefore": sample_pairs_before,
        "estimatedPairsAfter": scaled_pairs_after,
        "sampleEstimatedPairsAfter": estimated_pairs_after,
        "blockingStrategy": metrics.get("engine_block_strategy"),
        "guardrailTriggered": bool(metrics.get("engine_guardrail_triggered")),
        "engineResidualLimit": metrics.get("engine_residual_limit"),
        "blockKey": block_label,
        "blockCoverage": block_coverage,
        "suggestions": suggestions,
        "metrics": metrics,
    }


@manual_router.post("/examples")
async def manual_examples(
    req: ManualExecuteRequest,
    account_id: UUID = Depends(get_current_account),
):
    ctx = _parse_manual_request(req, default_limit=5)
    limit = ctx.limit or 5
    sample_size = max(limit * 3, limit)

    try:
        source_df = _load_manual_dataframe(ctx.source_upload_id, limit=sample_size)
    except FileNotFoundError:
        raise HTTPException(404, f"Source upload {ctx.source_upload_id} not found")
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    try:
        reference_df = _load_manual_dataframe(ctx.reference_upload_id)
    except FileNotFoundError:
        raise HTTPException(
            404, f"Reference upload {ctx.reference_upload_id} not found"
        )
    except ValueError as exc:
        raise HTTPException(400, str(exc))

    domain_registry = _load_domain_registry_safe()

    result_df, metrics = _execute_manual_match_or_http_error(
        source_df=source_df,
        reference_df=reference_df,
        mappings=ctx.mapping_configs,
        anchors=ctx.anchors,
        blocks=ctx.blocks,
        threshold=ctx.threshold,
        best_only=ctx.best_only,
        strip_b2c=ctx.strip_b2c,
        limit=sample_size,
        logger=log,
        source_id_column=ctx.source_id_column,
        reference_id_column=ctx.reference_id_column,
        domain_registry=domain_registry,
    )

    if result_df.empty:
        return {
            "manual_mode": True,
            "rows": [],
            "limit": limit,
            "metrics": metrics,
        }

    working_df = result_df.copy()
    if "match_score" in working_df.columns:
        working_df = working_df.sort_values(by="match_score", ascending=False)

    rows: List[Dict[str, Any]] = []
    seen_sources: Set[str] = set()

    for status in ("Account Match", "Possible Match", "No Match"):
        subset = working_df[working_df["match_status"] == status]
        if subset.empty:
            continue
        for _, row in subset.iterrows():
            src_id = str(row.get("source_id", ""))
            if src_id in seen_sources:
                continue
            rows.append(_normalize_record_for_json(row.to_dict()))
            seen_sources.add(src_id)
            break

    if len(rows) < limit:
        for _, row in working_df.iterrows():
            if len(rows) >= limit:
                break
            src_id = str(row.get("source_id", ""))
            if src_id in seen_sources:
                continue
            rows.append(_normalize_record_for_json(row.to_dict()))
            seen_sources.add(src_id)

    if len(rows) > limit:
        rows = rows[:limit]

    return {
        "manual_mode": True,
        "rows": rows,
        "limit": limit,
        "metrics": metrics,
    }


@manual_router.post("/execute")
async def manual_execute_route(
    req: ManualExecuteRequest,
    response: Response,
    request: Request,
    background_tasks: BackgroundTasks = None,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    # Safety rail: Validate mode and reject intelligence_correlation_id
    if req.mode and req.mode != "custom":
        raise HTTPException(
            400, f"Invalid mode '{req.mode}' for manual match. Expected 'custom'."
        )

    payload_snapshot = req.model_dump(mode="python")
    if payload_snapshot.get("intelligence_correlation_id"):
        raise HTTPException(
            400, "intelligence_correlation_id not allowed for manual match"
        )

    ctx = _parse_manual_request(req)
    _validate_manual_request_or_http_error(ctx)
    request_id = (
        getattr(getattr(request, "state", None), "request_id", None)
        or request.headers.get("X-Request-ID")
        or request.headers.get("X-Request-Id")
    )
    config: Dict[str, Any] = {
        "mode": "manual",
        "manual_mode": True,
        "mappings": ctx.raw_mappings,
        "mappings_count": len(ctx.mapping_configs),
        "anchors": ctx.anchors,
        "blocks": ctx.blocks,
        "threshold": ctx.threshold,
        "threshold_pct": int(round(ctx.threshold * 100)),
        "best_match_only": ctx.best_only,
        "strip_b2c": ctx.strip_b2c,
        "limit": ctx.limit,
        "source_upload_id": ctx.source_upload_id,
        "reference_upload_id": ctx.reference_upload_id,
        "enable_identity_transforms": ctx.enable_identity_transforms,
        "transforms": ctx.transforms,
    }
    if request_id:
        config["request_id"] = request_id
    if ctx.source_id_column:
        config["source_id_column"] = ctx.source_id_column
    if ctx.reference_id_column:
        config["reference_id_column"] = ctx.reference_id_column
    if payload_snapshot.get("sourceRange"):
        config["source_range"] = payload_snapshot["sourceRange"]
    if payload_snapshot.get("refRange"):
        config["reference_range"] = payload_snapshot["refRange"]

    job_id = str(uuid4())
    job = Job(
        id=job_id,
        account_id=str(account_id),
        status=JobStatus.QUEUED,
        job_type=JobType.MATCH,
        config=config,
    )
    db.add(job)
    await db.commit()

    redis_conn = await get_redis()
    if not redis_conn:
        job.status = JobStatus.FAILED
        job.error = "Cache service unavailable"
        await db.commit()
        raise HTTPException(503, "Cache service unavailable")

    job_meta: Dict[str, Any] = {
        "job_id": job_id,
        "status": "queued",
        "state": "queued",
        "mode": "manual",
        "manual_mode": True,
        "progress": 0,
        "processed_rows": 0,
        "mappings_count": len(ctx.mapping_configs),
        "config": config,
        "enable_identity_transforms": ctx.enable_identity_transforms,
        "transforms": ctx.transforms,
        "input": {
            "source_object_id": ctx.source_upload_id,
            "reference_object_id": ctx.reference_upload_id,
        },
        "source_object_id": ctx.source_upload_id,
        "reference_object_id": ctx.reference_upload_id,
        "best_match_only": ctx.best_only,
        "submitted_at": time.time(),
    }
    if request_id:
        job_meta["request_id"] = request_id
    await redis_conn.set(f"job:{job_id}", json.dumps(job_meta), ex=86400)

    job.status = JobStatus.PROCESSING
    await db.commit()

    job_meta.update(
        {
            "status": "processing",
            "state": "processing",
            "progress": 0.05,
            "message": "Running manual match",
        }
    )
    await redis_conn.set(f"job:{job_id}", json.dumps(job_meta), ex=86400)

    asyncio.create_task(_run_manual_job_async(job_id))

    response.headers["Location"] = f"/api/v2/jobs/{job_id}"

    return {
        "job_id": job_id,
        "status": "processing",
        "mode": "manual",
        "manual_mode": True,
        "mappings_count": len(ctx.mapping_configs),
        "request_id": request_id,
    }


class JobOverview(BaseModel):
    job_id: str
    state: str
    processed_rows: int = 0
    error: Optional[str] = None


@router.get("/{job_id}", response_model=JobOverview)
async def get_job_overview(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
    response: Response = None,
):
    # Ownership check
    rec = await db.get(Job, job_id)
    if not rec or (rec.account_id and str(rec.account_id) != str(account_id)):
        # Hide existence on mismatch
        raise HTTPException(404, "job not found")

    redis_conn = await get_redis()
    meta_raw = await redis_conn.get(f"job:{job_id}")
    state = "unknown"
    processed = 0
    error = None
    if meta_raw:
        try:
            m = json.loads(meta_raw)
            state = m.get("status") or m.get("state") or state
            processed = int(m.get("processed_rows") or 0)
            error = m.get("error")
        except Exception:
            pass

    if state == "unknown":
        try:
            state = str(
                rec.status.value if hasattr(rec.status, "value") else rec.status
            )
            processed = rec.matches_found or processed
            error = rec.error or error
        except Exception:
            pass

    if response is not None:
        response.headers["Cache-Control"] = "no-store"
    return {
        "job_id": job_id,
        "state": state,
        "processed_rows": processed,
        "error": error,
    }


@router.get("/__debug")
async def debug_info():
    """Check what code is actually running"""
    import fmatch.saas.api.v2.jobs as this_module

    redis_conn = await get_redis()

    # Check a recent job
    test_job_id = "db20fdb7-7d5c-4b9d-8a92-c20c8eb5ff7a"
    redis_data = await redis_conn.get(f"job:{test_job_id}") if redis_conn else None

    return {
        "module_file": this_module.__file__,
        "module_modified": os.path.getmtime(this_module.__file__),
        "redis_connected": redis_conn is not None,
        "test_job_exists": redis_data is not None,
        "current_time": datetime.utcnow().isoformat(),
    }


# Note: get_redis() is imported from progress.py at line 74
# Do not redefine it here to avoid overriding the correct implementation


# Helper functions for job management
async def get_job_metadata(job_id: str) -> Dict[str, Any]:
    """Get job metadata from Redis with SIMPLE KEY"""
    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    job_key = f"job:{job_id}"
    metadata_raw = await redis_conn.get(job_key)
    if not metadata_raw:
        raise HTTPException(404, f"Job {job_id} not found at key {job_key}")

    return json.loads(metadata_raw)


async def load_source_dataframe(job_id: str) -> pd.DataFrame:
    """Load source DataFrame from cached data - check parquet first, then Redis"""
    # Try to load from parquet file first (used by Salesforce integration)
    output_dir = get_temp_dir(job_id)
    parquet_path = output_dir / "source.parquet"

    if parquet_path.exists():
        log.info(f"Loading source from parquet: {parquet_path}")
        return pd.read_parquet(parquet_path)

    # Fall back to Redis (CSV data)
    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    source_data = await redis_conn.get(f"job:{job_id}:source_data")
    if not source_data:
        raise HTTPException(404, f"Source data for job {job_id} not found")

    return read_csv_with_encoding(source_data)


async def load_reference_dataframe(job_id: str) -> Optional[pd.DataFrame]:
    """Load reference DataFrame from cached data - check parquet first, then Redis"""
    # Try to load from parquet file first (used by Salesforce integration)
    output_dir = get_temp_dir(job_id)
    parquet_path = output_dir / "reference.parquet"

    if parquet_path.exists():
        log.info(f"Loading reference from parquet: {parquet_path}")
        return pd.read_parquet(parquet_path)

    # Fall back to Redis (CSV data)
    redis_conn = await get_redis()
    if not redis_conn:
        return None

    ref_data = await redis_conn.get(f"job:{job_id}:ref_data")
    if not ref_data:
        return None

    return read_csv_with_encoding(ref_data)


async def store_job_results(job_id: str, results: Dict[str, Any]) -> None:
    """Store job results in Redis"""
    redis_conn = await get_redis()
    if redis_conn:
        await redis_conn.set(f"job:{job_id}:results", json.dumps(results), ex=3600)


async def update_job_status(
    job_id: str,
    status: str,
    progress: int = None,
    error: str = None,
    db: AsyncSession = None,
) -> None:
    """Update job status in both Redis and database"""
    # Update Redis
    redis_conn = await get_redis()
    if redis_conn:
        job_key = f"job:{job_id}"
        metadata_raw = await redis_conn.get(job_key)
        if metadata_raw:
            metadata = json.loads(metadata_raw)
        else:
            metadata = {}

        metadata["status"] = status
        metadata["state"] = status  # Also set 'state' for compatibility
        if progress is not None:
            metadata["progress"] = progress
        if error:
            metadata["error"] = error
        if status == "completed":
            metadata["completed_at"] = time.time()
            metadata["matches_found"] = metadata.get("matches_found", 0)

        await redis_conn.set(job_key, json.dumps(metadata), ex=86400)  # 24 hours
        log.info(
            f"[UPDATE STATUS] Updated Redis for job {job_id}: status={status}, progress={progress}"
        )

    # Also try to update database if we can get a session
    if db is None:
        try:
            from ...db import get_session

            async for session in get_session():
                db = session
                break
        except Exception as e:
            log.warning(f"[UPDATE STATUS] Could not get DB session: {e}")

    if db:
        try:
            from ...models import JobStatus

            rec = await db.get(Job, job_id)
            if rec:
                # Map status string to JobStatus enum
                status_map = {
                    "completed": JobStatus.COMPLETED,
                    "failed": JobStatus.FAILED,
                    "queued": JobStatus.QUEUED,
                    "processing": JobStatus.PROCESSING,
                    "created": JobStatus.QUEUED,  # Map 'created' to QUEUED
                    "pending": JobStatus.QUEUED,  # Map 'pending' to QUEUED
                    "running": JobStatus.PROCESSING,  # Map 'running' to PROCESSING
                    "error": JobStatus.FAILED,  # Map 'error' to FAILED
                }
                if status in status_map:
                    rec.status = status_map[status]
                else:
                    # Default to PROCESSING for unknown statuses
                    log.warning(
                        f"[UPDATE STATUS] Unknown status '{status}', defaulting to PROCESSING"
                    )
                    rec.status = JobStatus.PROCESSING

                if status == "completed" and "matches_found" in metadata:
                    rec.matches_found = metadata["matches_found"]

                rec.updated_at = datetime.utcnow()
                await db.commit()
                log.info(
                    f"[UPDATE STATUS] Updated database for job {job_id}: status={status}"
                )
        except Exception as e:
            log.error(f"[UPDATE STATUS] Failed to update database: {e}")


def read_csv_with_encoding(content: bytes) -> pd.DataFrame:
    """
    Try to read CSV with proper encoding detection.
    Handles various encodings common in Windows/Excel files.
    """
    # Try to detect encoding if chardet is available
    if CHARDET_AVAILABLE:
        detected = chardet.detect(content[:10000] if len(content) > 10000 else content)
        encoding = detected.get("encoding", "utf-8")
        confidence = detected.get("confidence", 0)
        log.info(f"Detected encoding: {encoding} (confidence: {confidence:.2f})")

        # Start with detected encoding
        encodings_to_try = [encoding] if encoding else []
    else:
        encodings_to_try = []

    # Add common encoding fallbacks
    encodings_to_try.extend(
        ["utf-8", "latin-1", "windows-1252", "iso-8859-1", "cp1252"]
    )
    # Remove duplicates while preserving order
    encodings_to_try = list(dict.fromkeys(encodings_to_try))

    last_error = None
    for enc in encodings_to_try:
        if enc is None:
            continue
        try:
            # Try to decode and read CSV
            log.debug(f"Trying to read CSV with encoding: {enc}")
            df = pd.read_csv(io.BytesIO(content), encoding=enc)
            log.info(f"Successfully read CSV with encoding: {enc}")
            return df
        except (UnicodeDecodeError, UnicodeError) as e:
            log.debug(f"Failed with {enc}: {e}")
            last_error = e
            continue
        except Exception as e:
            if (
                "utf-8" in str(e).lower()
                or "unicode" in str(e).lower()
                or "decode" in str(e).lower()
            ):
                log.debug(f"Encoding error with {enc}: {e}")
                last_error = e
                continue
            # Non-encoding error, re-raise
            raise

    # Last resort - ignore errors
    log.warning(
        f"All encodings failed, using utf-8 with errors='ignore'. Last error: {last_error}"
    )
    return pd.read_csv(io.BytesIO(content), encoding="utf-8", errors="ignore")


@router.post("")  # This creates POST /api/v2/jobs
async def create_job(
    request: Request,
    source_file: Optional[UploadFile] = File(None),
    reference_file: Optional[UploadFile] = File(None),
    mode: str = Form("match"),
    source_provider: str = Form("csv"),
    source_options: Optional[str] = Form(None),  # JSON string
    reference_provider: str = Form("csv"),
    reference_options: Optional[str] = Form(None),  # JSON string
    idempotency_key: Optional[str] = Header(default=None),  # "Idempotency-Key" header
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """
    Create a new job with file uploads (v2 flow) or JSON payload
    Handles various file encodings automatically.
    Supports idempotency via Idempotency-Key header or content fingerprinting.
    """
    # Check if this is a JSON request (Simple Match wizard)
    content_type = request.headers.get("content-type", "")
    if "application/json" in content_type:
        # Handle JSON payload
        try:
            body = await request.json()
            job_request = CreateJobRequest(**body)
            return await create_job_json(job_request, db, account_id)
        except Exception as e:
            log.error(f"Failed to parse JSON request: {e}")
            raise HTTPException(400, f"Invalid JSON request: {str(e)}")

    # Otherwise, handle as form data with file uploads
    # Get async Redis connection
    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    log.info("=== CREATE JOB DEBUG ===")
    log.info(
        f"Source file: {source_file.filename}, size: {source_file.size if hasattr(source_file, 'size') else 'unknown'}"
    )
    log.info(f"Reference file: {reference_file.filename if reference_file else 'None'}")
    log.info(f"Mode: {mode}")
    log.info(f"Idempotency key: {idempotency_key}")

    try:
        # Initialize variables
        source_hash = None
        ref_hash = None
        source_content = None
        ref_content = None

        # For now, skip idempotency check for Salesforce sources
        # (we'll compute hashes differently for SF vs CSV)
        if idempotency_key:
            idem_key = f"idem:{account_id}:{idempotency_key}"
            existing_job_id = await redis_conn.get(idem_key)
            if existing_job_id:
                job_id = (
                    existing_job_id.decode()
                    if isinstance(existing_job_id, bytes)
                    else existing_job_id
                )
                stored = await redis_conn.get(f"job:{job_id}")
                if stored:  # still valid
                    log.info(f"Idempotent hit â†’ returning existing job_id={job_id}")
                    metadata = json.loads(stored)
                    return {
                        "job_id": job_id,
                        "status": metadata.get("status", "QUEUED"),
                        "mode": metadata.get("mode", mode),
                        "source_file": metadata.get(
                            "source_file",
                            source_file.filename if source_file else "Salesforce",
                        ),
                        "reference_file": metadata.get("ref_file"),
                        "_links": {
                            "self": f"/api/v2/jobs/{job_id}",
                            "configure": f"/api/v2/jobs/{job_id}/configuration/suggest",
                            "execute": f"/api/v2/jobs/{job_id}/execute",
                        },
                    }

        # Generate new job ID early - KEEP HYPHENATED
        job_id = str(uuid4())
        log.info(f"Generated job_id: {job_id}")

        # Parse provider options
        source_opts = json.loads(source_options) if source_options else {}
        ref_opts = json.loads(reference_options) if reference_options else {}

        # Initialize configuration dict for ID columns
        configuration = {}

        # Handle source data based on provider
        if source_provider == "salesforce":
            # Extract options
            obj = source_opts.get("object", "Lead")
            requested_fields = source_opts.get(
                "fields", ["Id", "Name", "Email", "Website", "Company"]
            )
            where = source_opts.get("where")

            # Validate fields against object describe
            desc = await sf.describe_object(obj)
            valid_field_names = {f["name"] for f in desc.get("fields", [])}

            # Smart field validation - handle Domain->Website conversion
            fields = ["Id"]
            for field in requested_fields:
                if field == "Id":
                    continue
                elif field == "Domain" and field not in valid_field_names:
                    # Domain requested but not available - use Website if it exists
                    if "Website" in valid_field_names and "Website" not in fields:
                        fields.append("Website")
                        log.info(f"Using Website field to derive Domain for {obj}")
                    else:
                        log.warning(
                            f"Neither Domain nor Website field available for {obj}"
                        )
                elif field in valid_field_names:
                    fields.append(field)
                else:
                    log.debug(f"Field '{field}' not found in {obj}")

            # Build SOQL
            base_soql = f"SELECT {', '.join(fields)} FROM {obj}"
            soql = base_soql + (f" WHERE {where}" if where else "")

            # Decide bulk vs REST
            use_bulk = source_opts.get("bulk", False)
            if not use_bulk:
                try:
                    where_clause = f" WHERE {where}" if where else ""
                    count_soql = f"SELECT count() FROM {obj}{where_clause}"
                    result = await sf.soql(count_soql)
                    count = result.get("totalSize", 0)
                    use_bulk = count > 50000
                    log.info(f"Salesforce {obj} count: {count}, use_bulk: {use_bulk}")
                except Exception as e:
                    log.warning(f"Count fallback to REST: {e}")
                    use_bulk = False

            # Log safely (no WHERE clause in logs)
            where_hash = hashlib.md5((where or "").encode()).hexdigest()[:8]
            log.info(
                f"Fetching from Salesforce: object={obj}, fields={len(fields)}, bulk={use_bulk}, where_hash={where_hash}"
            )

            # Materialize to DataFrame
            source_df = await materialize_salesforce(sf, soql, obj, use_bulk, job_id)

            # Smart domain extraction - add Domain if missing
            if "Domain" not in source_df.columns:
                if "Website" in source_df.columns:
                    log.info("Domain field not found, deriving from Website field")
                    source_df["Domain"] = source_df["Website"].apply(
                        extract_domain_from_url
                    )
                    domain_coverage = (
                        source_df["Domain"].notna().sum() / len(source_df) * 100
                    )
                    log.info(
                        f"Derived domain for {domain_coverage:.1f}% of records from Website"
                    )
                elif "Email" in source_df.columns:
                    # Email domain extraction as fallback
                    log.info("No Website field, extracting domain from Email")
                    source_df["Domain"] = (
                        source_df["Email"]
                        .astype(str)
                        .str.extract(r"@([A-Za-z0-9.-]+\.[A-Za-z]{2,})", expand=False)
                    )
                    domain_coverage = (
                        source_df["Domain"].notna().sum() / len(source_df) * 100
                    )
                    log.info(
                        f"Extracted domain for {domain_coverage:.1f}% of records from Email"
                    )

            # Set ID column configuration
            configuration["source_id_column"] = "Id"
            source_filename = f"Salesforce_{obj}"
            source_hash = hashlib.md5(soql.encode()).hexdigest()

        else:  # CSV provider
            if not source_file:
                raise HTTPException(400, "Source file required for CSV provider")
            source_filename = source_file.filename
            log.info(f"Processing source file: {source_filename}")
            source_content = await source_file.read()
            source_hash = hashlib.md5(source_content).hexdigest()
            source_df = read_csv_with_encoding(source_content)
            # Don't set source_id_column for CSV - let auto-detection handle it

        source_row_count = int(len(source_df))
        log.info(
            f"Source DataFrame: {source_row_count} rows, {len(source_df.columns)} columns"
        )

        # Handle reference data
        ref_df = None
        ref_filename = None
        if mode == "match":
            if reference_provider == "salesforce":
                # Using injected SalesforceGateway (sf)

                obj = ref_opts.get("object", "Account")
                requested_fields = ref_opts.get(
                    "fields", ["Id", "Name", "Website", "BillingCity", "BillingState"]
                )
                where = ref_opts.get("where")

                # Validate fields against object describe
                desc = await sf.describe(obj)
                valid_field_names = {f["name"] for f in desc.get("fields", [])}

                # Smart field validation - handle Domain->Website conversion
                fields = ["Id"]
                for field in requested_fields:
                    if field == "Id":
                        continue
                    elif field == "Domain" and field not in valid_field_names:
                        # Domain requested but not available - use Website if it exists
                        if "Website" in valid_field_names and "Website" not in fields:
                            fields.append("Website")
                            log.info(f"Using Website field to derive Domain for {obj}")
                        else:
                            log.warning(
                                f"Neither Domain nor Website field available for {obj}"
                            )
                    elif field in valid_field_names:
                        fields.append(field)
                    else:
                        log.debug(f"Field '{field}' not found in {obj}")

                # Build SOQL
                base_soql = f"SELECT {', '.join(fields)} FROM {obj}"
                soql = base_soql + (f" WHERE {where}" if where else "")

                # Decide bulk vs REST
                use_bulk = ref_opts.get("bulk", False)
                if not use_bulk:
                    try:
                        where_clause = f" WHERE {where}" if where else ""
                        count_soql = f"SELECT count() FROM {obj}{where_clause}"
                        result = await sf.soql(count_soql)
                        count = result.get("totalSize", 0)
                        use_bulk = count > 50000
                        log.info(
                            f"Salesforce {obj} count: {count}, use_bulk: {use_bulk}"
                        )
                    except Exception as e:
                        log.warning(f"Count fallback to REST: {e}")
                        use_bulk = False

                where_hash = hashlib.md5((where or "").encode()).hexdigest()[:8]
                log.info(
                    f"Fetching from Salesforce: object={obj}, fields={len(fields)}, bulk={use_bulk}, where_hash={where_hash}"
                )

                ref_df = await materialize_salesforce(sf, soql, obj, use_bulk, job_id)

                # Smart domain extraction for reference data
                if "Domain" not in ref_df.columns:
                    if "Website" in ref_df.columns:
                        log.info(
                            "Domain field not found in reference, deriving from Website field"
                        )
                        ref_df["Domain"] = ref_df["Website"].apply(
                            extract_domain_from_url
                        )
                        domain_coverage = (
                            ref_df["Domain"].notna().sum() / len(ref_df) * 100
                        )
                        log.info(
                            f"Derived domain for {domain_coverage:.1f}% of reference records from Website"
                        )
                    elif "Email" in ref_df.columns:
                        # Email domain extraction as fallback
                        log.info(
                            "No Website field in reference, extracting domain from Email"
                        )
                        ref_df["Domain"] = (
                            ref_df["Email"]
                            .astype(str)
                            .str.extract(
                                r"@([A-Za-z0-9.-]+\.[A-Za-z]{2,})", expand=False
                            )
                        )
                        domain_coverage = (
                            ref_df["Domain"].notna().sum() / len(ref_df) * 100
                        )
                        log.info(
                            f"Extracted domain for {domain_coverage:.1f}% of reference records from Email"
                        )

                configuration["reference_id_column"] = "Id"
                ref_filename = f"Salesforce_{obj}"
                ref_hash = (
                    hashlib.md5(soql.encode()).hexdigest() if not ref_hash else ref_hash
                )

                # Convert DataFrame to CSV for Redis storage
                ref_content = ref_df.to_csv(index=False).encode("utf-8")
                log.info(
                    f"Converted Salesforce DataFrame to CSV for Redis storage: {len(ref_content)} bytes"
                )

            else:  # CSV provider
                if reference_file:
                    ref_filename = reference_file.filename
                    log.info(f"Processing reference file: {ref_filename}")
                    ref_content = await reference_file.read()
                    ref_hash = hashlib.md5(ref_content).hexdigest()
                    ref_df = read_csv_with_encoding(ref_content)
                    log.info(
                        f"Reference DataFrame: {len(ref_df)} rows, {len(ref_df.columns)} columns"
                    )

        # First ensure Domain column exists for CSV files
        for df in [source_df, ref_df] if ref_df is not None else [source_df]:
            if df is not None and "Domain" not in df.columns:
                # Try to derive from Website or Email columns
                if "Website" in df.columns:
                    log.info("Creating Domain column from Website for CSV data")
                    df["Domain"] = df["Website"].apply(extract_domain_from_url)
                elif "Email" in df.columns:
                    log.info("Creating Domain column from Email for CSV data")
                    df["Domain"] = (
                        df["Email"]
                        .astype(str)
                        .str.extract(r"@([A-Za-z0-9.-]+\.[A-Za-z]{2,})", expand=False)
                    )

        # Check feature flag for domain family enrichment
        enable_domain_families = (
            os.getenv("FEATURE_DOMAIN_FAMILY", "false").lower() == "true"
        )

        # Enrich with domain families if enabled
        if enable_domain_families:
            log.info("Domain family enrichment enabled via feature flag")
            for df in [source_df, ref_df] if ref_df is not None else [source_df]:
                if df is not None and "Domain" in df.columns:
                    # Normalize domain casing
                    df["Domain"] = df["Domain"].astype(str).str.strip().str.lower()

                    # Use batch enrichment for better performance
                    updated_df, family_coverage = enrich_with_domain_families(df)

                    # Update the original df
                    if "DomainFamily" in updated_df.columns:
                        df["DomainFamily"] = updated_df["DomainFamily"]

                    log.info(f"Domain family coverage: {family_coverage:.1f}%")
        else:
            log.debug(
                "Domain family enrichment disabled (set FEATURE_DOMAIN_FAMILY=true to enable)"
            )

        # After domain normalization and empty checks
        if source_df.empty or (mode == "match" and (ref_df is None or ref_df.empty)):
            log.warning(
                f"Empty dataset(s) for job {job_id} "
                f"(source_rows={len(source_df)}, ref_rows={0 if ref_df is None else len(ref_df)})"
            )

        # Still stash what you have (even if empty)
        await stash_dataframe(job_id, source_df, db, account_id, kind="source")
        if ref_df is not None:
            await stash_dataframe(job_id, ref_df, db, account_id, kind="reference")

        reference_row_count = int(len(ref_df)) if ref_df is not None else 0
        log.info(
            f"[JOB {job_id}] Source rows: {source_row_count}, Reference rows: {reference_row_count}"
        )

        # Prepare metadata
        configuration["source_row_count"] = source_row_count
        if mode == "match":
            configuration["reference_row_count"] = reference_row_count

        metadata = {
            "job_id": job_id,
            "mode": mode,
            "source_provider": source_provider,
            "source_file": source_filename,
            "source_rows": source_row_count,
            "source_row_count": source_row_count,
            "source_cols": list(source_df.columns),
            "source_hash": source_hash,
            "reference_provider": reference_provider if mode == "match" else None,
            "ref_file": ref_filename,
            "ref_rows": reference_row_count,
            "reference_row_count": reference_row_count,
            "ref_cols": list(ref_df.columns) if ref_df is not None else [],
            "reference_hash": ref_hash if ref_hash else None,
            "configuration": configuration,  # Store initial config
            "status": "QUEUED",
            "created_at": datetime.utcnow().isoformat(),
        }

        # STEP 1: Write to SQL Database (source of truth for job existence)
        try:
            job_record = Job(
                id=job_id,  # Keep as hyphenated string
                account_id=str(account_id) if account_id is not None else None,
                project_id=None,  # Explicitly set nullable fields
                status="QUEUED",  # Changed from "created" to match enum
                job_type=mode,  # 'match' or 'dedupe'
                config=metadata,
                created_at=datetime.utcnow(),
                updated_at=datetime.utcnow(),
            )
            db.add(job_record)
            assert (
                job_record.config.get("source_row_count", 0) > 0
            ), f"Job {job_id} config missing source_row_count - billing will fail"
            await db.commit()  # Commit the transaction
            await db.refresh(job_record)  # Refresh to get DB-generated values
            log.info(f"[SQL] Created job row id={job_record.id}")

            # Double-verify with raw SQL to bypass ORM
            verify_result = await db.execute(
                text("SELECT id, status FROM jobs WHERE id = :id"), {"id": job_id}
            )
            verify_row = verify_result.first()
            if verify_row:
                log.info(
                    f"[SQL] Verified job exists: id={verify_row[0]}, status={verify_row[1]}"
                )
            else:
                log.error(f"[SQL] âœ— Could not verify job {job_id} after insert!")

        except Exception as e:
            log.error(f"[SQL] âœ— Write FAILED for job {job_id}: {str(e)}")
            log.error(f"[SQL] Exception type: {type(e).__name__}")
            await db.rollback()
            # Continue anyway - Redis has the data
            log.info("[SQL] Continuing with Redis storage despite SQL failure")

        # STEP 2: Write to Redis (for fast access and data storage)
        if redis_conn:
            log.info("Storing job data in Redis with 1-hour expiry...")
            # Store file data in Redis for configuration step
            await redis_conn.set(f"job:{job_id}:source_data", source_content, ex=3600)
            await redis_conn.set(f"job:{job_id}:source_hash", source_hash, ex=3600)
            log.info(f"Stored source_data and source_hash for job:{job_id}")

            if ref_content:
                await redis_conn.set(f"job:{job_id}:ref_data", ref_content, ex=3600)
                await redis_conn.set(f"job:{job_id}:ref_hash", ref_hash, ex=3600)
                log.info(f"Stored ref_data and ref_hash for job:{job_id}")

            # Store metadata with SIMPLE KEY - no :metadata suffix!
            job_key = f"job:{job_id}"
            await redis_conn.set(job_key, json.dumps(metadata), ex=3600)

            # Remember for 24h for idempotency
            if idempotency_key:
                idem_key = f"idem:{account_id}:{idempotency_key}"
                await redis_conn.set(idem_key, job_id, ex=86400)
            else:
                # For content-based idempotency
                content_fp = f"{account_id}:{mode}:{source_hash}:{ref_hash or ''}"
                idem_key = f"idem:{content_fp}"
                await redis_conn.set(idem_key, job_id, ex=86400)

            log.info(f"Stored metadata with SIMPLE key: {job_key}")

            # Debug log to PROVE it's stored
            exists = await redis_conn.exists(job_key)
            log.info(f"DEBUG: redis.exists('{job_key}') = {exists}")

            # Verify storage
            verify_keys = await redis_conn.keys(f"job:{job_id}:*")
            log.info(
                f"Verification - Keys stored: {[k.decode() if isinstance(k, bytes) else k for k in verify_keys]}"
            )
        else:
            log.error("Redis connection is None - cannot store job data!")

        response = {
            "job_id": job_id,
            "status": "QUEUED",
            "mode": mode,
            "source_file": source_filename,  # Use the variable we set, not source_file.filename
            "reference_file": ref_filename,  # Use the variable we set
            "_links": {
                "self": f"/api/v2/jobs/{job_id}",
                "configure": f"/api/v2/jobs/{job_id}/configuration/suggest",  # Fixed URL
                "execute": f"/api/v2/jobs/{job_id}/execute",  # Fixed URL
            },
        }
        log.info(f"Returning response: {response}")
        return response

    except Exception as e:
        log.error(f"Error creating job: {e}", exc_info=True)
        await db.rollback()  # Rollback SQL on error
        raise HTTPException(500, f"Failed to create job: {str(e)}")


@router.post("/json")  # JSON endpoint for Simple Match wizard
async def create_job_json(
    request: CreateJobRequest,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    """
    Create a job from JSON payload (used by Simple Match wizard)
    Accepts object_ids for pre-uploaded files
    """
    from ...storage import exists_object, resolve_object_path
    from ...models import JobStatus

    log.info(f"[CREATE JOB JSON] Received request from account {account_id}")

    # Validate that uploaded files exist
    missing = []
    src_key = request.input.object_id or request.input.uri
    ref_key = request.input.reference_object_id or request.input.reference_uri

    if not src_key:
        raise HTTPException(400, {"error": "input.object_id or input.uri required"})

    # Validate source file exists
    if src_key and not exists_object(src_key):
        missing.append(
            {
                "which": "source",
                "object_id": src_key,
                "path_tried": resolve_object_path(src_key),
            }
        )

    # Validate reference file exists (if provided)
    if ref_key and not exists_object(ref_key):
        missing.append(
            {
                "which": "reference",
                "object_id": ref_key,
                "path_tried": resolve_object_path(ref_key),
            }
        )

    if missing:
        log.error(f"[CREATE JOB JSON] Missing uploads: {missing}")
        raise HTTPException(
            400, {"error": "Missing uploaded files", "missing": missing}
        )

    # Generate job ID
    job_id = str(uuid4())
    mode = "match" if ref_key else "dedupe"

    log.info(f"[CREATE JOB JSON] Created job {job_id}, mode={mode}")

    # Derive row counts for billing/telemetry
    source_row_count: int = 0
    reference_row_count: int = 0

    try:
        source_path = resolve_object_path(src_key)
        with open(source_path, "rb") as f:
            source_bytes = f.read()
        source_df = read_csv_with_encoding(source_bytes)
        source_row_count = int(len(source_df))
        log.info(
            f"[CREATE JOB JSON] Source dataset rows={source_row_count}, cols={len(source_df.columns)}"
        )
    except Exception as exc:
        log.error(f"[CREATE JOB JSON] Failed to load source data for {job_id}: {exc}")
        raise HTTPException(500, {"error": "source_read_failed", "detail": str(exc)})

    reference_df = None
    if ref_key:
        try:
            ref_path = resolve_object_path(ref_key)
            with open(ref_path, "rb") as f:
                ref_bytes = f.read()
            reference_df = read_csv_with_encoding(ref_bytes)
            reference_row_count = int(len(reference_df))
            log.info(
                f"[CREATE JOB JSON] Reference dataset rows={reference_row_count}, cols={len(reference_df.columns)}"
            )
        except Exception as exc:
            log.error(
                f"[CREATE JOB JSON] Failed to load reference data for {job_id}: {exc}"
            )
            raise HTTPException(
                500, {"error": "reference_read_failed", "detail": str(exc)}
            )

    # Store metadata in Redis (compatible with status checks and suggest)
    redis_conn = await get_redis()
    if redis_conn:
        metadata = {
            "status": "created",
            "mode": mode,
            "source_object_id": src_key,
            "reference_object_id": ref_key,
            "object_id": src_key,  # For compatibility with suggest
            "input": {  # Store in format that suggest expects
                "object_id": src_key,
                "reference_object_id": ref_key,
                "has_header": True,
                "reference_has_header": True,
            },
            "intelligence": request.intelligence.model_dump(),
            "created_at": datetime.utcnow().isoformat(),
            "source_row_count": source_row_count,
            "reference_row_count": reference_row_count,
        }
        await redis_conn.set(
            f"job:{job_id}", json.dumps(metadata), ex=86400
        )  # 24 hours

    # Create job record in database
    job_record = Job(
        id=job_id,
        account_id=str(account_id),
        status=JobStatus.QUEUED,  # Use valid enum value instead of "created"
        job_type=mode,
        config={
            "source_object_id": src_key,
            "reference_object_id": ref_key,
            "intelligence": request.intelligence.model_dump(),
            "source_row_count": source_row_count,
            "reference_row_count": reference_row_count,
        },
        created_at=datetime.utcnow(),
        updated_at=datetime.utcnow(),
    )
    db.add(job_record)
    assert (
        job_record.config.get("source_row_count", 0) > 0
    ), f"Job {job_id} config missing source_row_count - billing will fail"
    await db.commit()

    return {
        "job_id": job_id,
        "status": "created",
        "mode": mode,
        "_links": {
            "self": f"/api/v2/jobs/{job_id}",
            "configure": f"/api/v2/jobs/{job_id}/configuration/suggest",
            "execute": f"/api/v2/jobs/{job_id}/execute",
        },
    }


@router.delete("/{job_id}", status_code=202)
async def cancel_job(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    """
    Signal cancellation for a running job.
    Note: Early stop requires the core to periodically check a cancel flag in RunStats.
    """
    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    # Set a cancel flag and update status
    await redis_conn.set(f"job:{job_id}:cancel", "1", ex=3600)
    await update_job_status(job_id, "cancel_requested", progress=None)

    return {"job_id": job_id, "status": "cancel_requested"}


@router.post("/{job_id}/execute")  # Don't force 202, let it be dynamic
async def execute_job(
    job_id: str,
    request_body: Dict[str, Any] = Body(...),
    background_tasks: BackgroundTasks = None,
    session: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    """
    Execute a job with the provided configuration using the engine bridge.
    Accepts either { configuration: {...} } or { intelligence: { config_hash: "..." } }
    """
    # Get async Redis connection
    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    log.info("=== EXECUTE JOB DEBUG ===")
    log.info(f"Received job_id: {job_id}")
    log.info(f"Request body keys: {list(request_body.keys())}")
    log.info(f"Request body: {request_body}")

    # Handle both formats: { configuration: {...} } and { intelligence: { config_hash: "..." } }
    warnings: List[str] = []
    configuration: Dict[str, Any] = {}
    submitted_hash: Optional[str] = None
    submitted_hash_raw: Optional[Any] = None

    if "configuration" in request_body:
        configuration = request_body["configuration"]
        if isinstance(configuration, dict):
            submitted_hash_raw = (
                configuration.get("hash")
                or configuration.get("config_hash")
                or configuration.get("hash_prefix")
                or configuration.get("intelligence_hash")
            )
    elif "intelligence" in request_body:
        # Simple Match wizard sends { intelligence: { config_hash: "..." } }
        intel = request_body["intelligence"]
        if isinstance(intel, dict):
            submitted_hash_raw = (
                intel.get("hash")
                or intel.get("config_hash")
                or intel.get("hash_prefix")
                or intel.get("intelligence_hash")
            )
            # FIXED: Only set configuration from hash if it's empty (not already set from "configuration" key)
            if submitted_hash_raw and not configuration:
                configuration = {"hash": submitted_hash_raw}
            log.info(
                f"[EXECUTE] Simple Match mode with config hash payload: {submitted_hash_raw}"
            )

    # FIXED: Extract blocking_strategy from intelligence block if it's missing from configuration
    if "intelligence" in request_body and isinstance(request_body["intelligence"], dict):
        intel = request_body["intelligence"]
        if "blocking_strategy" in intel and "blocking_strategy" not in configuration:
            configuration["blocking_strategy"] = intel["blocking_strategy"]
            log.info("[EXECUTE] Extracted blocking_strategy from intelligence block")

    if hasattr(configuration, "model_dump"):
        configuration = configuration.model_dump()

    if not isinstance(configuration, dict):
        configuration = dict(configuration or {})

    def _clean_hash(value: Optional[Any]) -> Optional[str]:
        if value is None:
            return None
        if isinstance(value, (bytes, bytearray)):
            value = value.decode("utf-8", "ignore")
        value = str(value).strip()
        return value.lower() if value else None

    if not isinstance(configuration, dict):
        configuration = dict(configuration or {})

    hash_inputs: List[Any] = []

    def _extend_hash_inputs(source: Optional[Dict[str, Any]]) -> None:
        if not isinstance(source, dict):
            return
        hash_inputs.extend(
            [
                source.get("hash"),
                source.get("config_hash"),
                source.get("hash_prefix"),
                source.get("intelligence_hash"),
            ]
        )
        aliases = source.get("hash_aliases")
        if isinstance(aliases, dict):
            hash_inputs.extend(list(aliases.values()))

    _extend_hash_inputs(configuration)

    has_mappings = bool(configuration.get("mappings") or configuration.get("maps"))

    if "hash" in request_body:
        hash_inputs.append(request_body.get("hash"))

    intel_block = request_body.get("intelligence")
    if isinstance(intel_block, dict):
        _extend_hash_inputs(intel_block)
        if not has_mappings:
            has_mappings = bool(
                intel_block.get("mappings")
                or (
                    isinstance(intel_block.get("options"), dict)
                    and intel_block["options"].get("maps")
                )
            )

    if submitted_hash_raw is not None:
        hash_inputs.insert(0, submitted_hash_raw)

    primary_raw = next((h for h in hash_inputs if h), None)
    if primary_raw is None:
        primary_raw = submitted_hash_raw

    submitted_hash = _clean_hash(primary_raw)
    if not submitted_hash:
        for value in hash_inputs:
            candidate_clean = _clean_hash(value)
            if candidate_clean:
                submitted_hash = candidate_clean
                break

    log.info(
        f"[EXECUTE] Hash primary={primary_raw} submitted={submitted_hash} has_mappings={has_mappings}"
    )

    hash_candidates: List[str] = []
    seen_candidates: Set[str] = set()

    def _add_candidate(value: Optional[Any]) -> None:
        normalized = _clean_hash(value)
        if not normalized:
            return
        if normalized not in seen_candidates:
            hash_candidates.append(normalized)
            seen_candidates.add(normalized)
        if len(normalized) > 16:
            short = normalized[:16]
            if short not in seen_candidates:
                hash_candidates.append(short)
                seen_candidates.add(short)

    job_suggested: Optional[Dict[str, Any]] = None

    for value in hash_inputs:
        _add_candidate(value)

    index_hits: List[str] = []
    for candidate in list(hash_candidates):
        if len(candidate) == 16:
            index_key = f"intel:index:short:{candidate}"
            indexed_full = await redis_conn.get(index_key)
            if indexed_full:
                mapped_value = (
                    indexed_full.decode("utf-8")
                    if isinstance(indexed_full, bytes)
                    else str(indexed_full)
                )
                _add_candidate(mapped_value)
                index_hits.append(index_key)

    computed_stable_from_config: Optional[str] = None
    if has_mappings:
        try:
            computed_stable_from_config = _clean_hash(
                _stable_intelligence_hash(configuration)
            )
        except Exception as err:
            log.debug(
                f"[EXECUTE] Unable to compute stable hash from inline config: {err}"
            )
    if computed_stable_from_config:
        _add_candidate(computed_stable_from_config)

    log.info(f"[EXECUTE] Hash candidates for job {job_id}: {hash_candidates}")
    if index_hits:
        log.info(f"[EXECUTE] Index expansions used: {index_hits}")

    cache_hit_key: Optional[str] = None
    global_config: Optional[Dict[str, Any]] = None
    for candidate in hash_candidates:
        cache_key = f"intelligence:{candidate}"
        cached_payload = await redis_conn.get(cache_key)
        if not cached_payload:
            continue
        if isinstance(cached_payload, bytes):
            cached_payload = cached_payload.decode("utf-8")
        try:
            global_config = json.loads(cached_payload)
            cache_hit_key = cache_key
            log.info(f"[EXECUTE] Cache hit at {cache_key}")
            break
        except Exception as decode_err:
            log.warning(f"Failed to decode cached config at {cache_key}: {decode_err}")
            global_config = None

    target_hashes: Set[str] = set(hash_candidates)

    def _merge_configs(
        base: Dict[str, Any], override: Dict[str, Any]
    ) -> Dict[str, Any]:
        merged = base.copy()
        for key, value in override.items():
            if value is not None:
                merged[key] = value
        return merged

    def _hash_matches(candidate: Optional[Dict[str, Any]], targets: Set[str]) -> bool:
        if not candidate:
            return False

        candidate_hashes: Set[str] = set()
        keys_to_check = (
            "intelligence_hash",
            "config_hash",
            "hash",
            "hash_prefix",
            "hash_short",
            "legacy_hash",
        )

        for key in keys_to_check:
            candidate_value = _clean_hash(candidate.get(key))
            if candidate_value:
                candidate_hashes.add(candidate_value)
                if len(candidate_value) > 16:
                    candidate_hashes.add(candidate_value[:16])

        aliases = candidate.get("hash_aliases")
        if isinstance(aliases, dict):
            for alias_value in aliases.values():
                alias_clean = _clean_hash(alias_value)
                if alias_clean:
                    candidate_hashes.add(alias_clean)
                    if len(alias_clean) > 16:
                        candidate_hashes.add(alias_clean[:16])

        computed_hash: Optional[str] = None
        try:
            computed_hash = _clean_hash(_stable_intelligence_hash(candidate))
        except Exception as hash_err:
            log.debug(
                f"[EXECUTE] Unable to compute stable hash for candidate config: {hash_err}"
            )

        if computed_hash:
            candidate_hashes.add(computed_hash)
            if len(computed_hash) > 16:
                candidate_hashes.add(computed_hash[:16])
            candidate.setdefault("intelligence_hash", computed_hash)
            aliases = (
                candidate.setdefault("hash_aliases", {})
                if isinstance(candidate.get("hash_aliases"), dict)
                else {}
            )
            if isinstance(aliases, dict):
                aliases.setdefault("full", computed_hash)
                aliases.setdefault("short", computed_hash[:16])
                candidate["hash_aliases"] = aliases

        normalized_targets: Set[str] = set()
        for target in targets:
            cleaned = _clean_hash(target)
            if cleaned:
                normalized_targets.add(cleaned)
                if len(cleaned) > 16:
                    normalized_targets.add(cleaned[:16])

        if not normalized_targets:
            return bool(candidate_hashes)

        return bool(candidate_hashes & normalized_targets)

    rehydration_source: Optional[str] = None
    if hash_candidates and not has_mappings:
        base_config: Optional[Dict[str, Any]] = None
        if job_suggested and _hash_matches(job_suggested, target_hashes):
            base_config = job_suggested
            rehydration_source = "job"
        elif global_config and _hash_matches(global_config, target_hashes):
            base_config = global_config
            rehydration_source = cache_hit_key or "global"

        if base_config:
            configuration = _merge_configs(base_config, configuration)
            warnings.append("CONFIG_REHYDRATED_FROM_HASH")
            has_mappings = bool(
                configuration.get("mappings") or configuration.get("maps")
            )
            log.info(
                f"[EXECUTE] Rehydrated configuration for job {job_id} from {rehydration_source}"
            )
        else:
            warnings.append("CONFIG_MISSING_INTELLIGENCE")
            log.error(
                "No suggested configuration available for hash-only execute payload"
            )
            raise HTTPException(
                status_code=400,
                detail={
                    "code": "INTEL_CONFIG_REQUIRED",
                    "message": "Hash provided but no suggested configuration found. Call /configuration/suggest first or include mappings in request.",
                    "warnings": warnings,
                    "candidates": hash_candidates,
                },
            )
    elif job_suggested and _hash_matches(job_suggested, target_hashes):
        configuration = _merge_configs(job_suggested, configuration)
        has_mappings = bool(configuration.get("mappings") or configuration.get("maps"))
        log.info("Found suggested configuration, merging with request config")

    legacy_effective = _clean_hash(
        configuration.get("hash") or configuration.get("config_hash")
    )
    intelligence_effective = _clean_hash(configuration.get("intelligence_hash"))
    if legacy_effective:
        configuration["hash"] = legacy_effective
        configuration["config_hash"] = legacy_effective
        configuration.setdefault("hash_prefix", legacy_effective[:16])
    if intelligence_effective:
        configuration["intelligence_hash"] = intelligence_effective
        aliases = (
            configuration.get("hash_aliases")
            if isinstance(configuration.get("hash_aliases"), dict)
            else {}
        )
        if isinstance(aliases, dict):
            if "short" not in aliases:
                aliases["short"] = legacy_effective or intelligence_effective[:16]
            aliases.setdefault("full", intelligence_effective)
            configuration["hash_aliases"] = aliases

    if has_mappings:
        cache_full_hash = intelligence_effective
        if not cache_full_hash:
            try:
                cache_full_hash = _clean_hash(_stable_intelligence_hash(configuration))
            except Exception as cache_err:
                log.debug(
                    f"[EXECUTE] Unable to compute stable hash for provided configuration: {cache_err}"
                )
        if cache_full_hash:
            cache_short_hash = cache_full_hash[:16]
            cache_payload = json.dumps(
                configuration, separators=(",", ":"), ensure_ascii=False, default=str
            )
            cache_ttl = 86400
            await redis_conn.set(
                f"intelligence:{cache_full_hash}", cache_payload, ex=cache_ttl
            )
            await redis_conn.set(
                f"intelligence:{cache_short_hash}", cache_payload, ex=cache_ttl
            )
            await redis_conn.set(
                f"intel:index:short:{cache_short_hash}", cache_full_hash, ex=cache_ttl
            )
            log.info(
                f"[EXECUTE] Seeded config cache full={cache_full_hash} short={cache_short_hash}"
            )

    # correlation_id is OPTIONAL - generate one if missing for metrics/tracing
    correlation_id = configuration.get("correlation_id")
    if not correlation_id:
        correlation_id = f"auto:{job_id}:{uuid4().hex[:8]}"
        configuration["correlation_id"] = correlation_id
        log.warning(
            f"No correlation_id provided; generated '{correlation_id}' for job {job_id}"
        )
    else:
        # If correlation_id is provided, validate it (but don't fail if invalid, just log)
        correlation_data = await redis_conn.get(f"correlation:{correlation_id}")
        if correlation_data:
            correlation_info = json.loads(correlation_data)
            log.info(
                f"Valid correlation_id from account {correlation_info.get('account_id')}"
            )

            # Verify account matches (warn but don't fail)
            if correlation_info.get("account_id") != str(account_id):
                log.warning(
                    f"Account mismatch for correlation_id: {correlation_id} - proceeding anyway"
                )
        else:
            log.warning(
                f"Correlation_id {correlation_id} not found in cache - proceeding anyway"
            )

    try:
        log.info(f"Redis connection established: {redis_conn}")

        # Debug: Check what keys exist for this job
        all_job_keys = []
        async for key in redis_conn.scan_iter(f"job:{job_id}:*"):
            all_job_keys.append(key.decode() if isinstance(key, bytes) else key)
        log.info(f"Found Redis keys for job {job_id}: {all_job_keys}")

        # Also check for any temp job keys
        if job_id.startswith("temp-"):
            temp_keys = []
            async for key in redis_conn.scan_iter("latest_*"):
                temp_keys.append(key.decode() if isinstance(key, bytes) else key)
            log.info(f"Found temp/latest keys: {temp_keys}")

        # Get job metadata with SIMPLE KEY!
        job_key = f"job:{job_id}"
        log.info(f"Looking for job with key: {job_key}")
        metadata_raw = await redis_conn.get(job_key)
        if not metadata_raw:
            log.error(f"No metadata found for job {job_id} at key {job_key}")
            # List all available job keys for debugging
            all_keys = []
            async for key in redis_conn.scan_iter("job:*"):
                all_keys.append(key.decode() if isinstance(key, bytes) else key)
            log.info(f"Available job keys in Redis: {all_keys[:20]}")  # Show first 20
            raise HTTPException(
                404, f"Job {job_id} not found - no metadata in Redis at key {job_key}"
            )

        metadata = json.loads(metadata_raw)

        existing_warnings = metadata.get("warnings")
        combined_warnings: List[str] = []
        if isinstance(existing_warnings, list):
            combined_warnings.extend(existing_warnings)
        elif isinstance(existing_warnings, str) and existing_warnings:
            combined_warnings.append(existing_warnings)
        for warning in warnings:
            if warning not in combined_warnings:
                combined_warnings.append(warning)
        if combined_warnings:
            metadata["warnings"] = combined_warnings

        metadata["config_hash_candidates_tried"] = hash_candidates
        if cache_hit_key:
            metadata["config_hash_lookup_hit"] = cache_hit_key
        if submitted_hash:
            metadata["config_hash_submitted"] = submitted_hash
        if legacy_effective:
            metadata["config_hash_effective"] = legacy_effective
        if intelligence_effective:
            metadata["intelligence_hash_effective"] = intelligence_effective

        if configuration:
            metadata["config"] = configuration

        mode = metadata.get("mode", "match")
        log.info(f"Job {job_id} metadata: mode={mode}, status={metadata.get('status')}")

        # Check if this is a Simple Match job (has object_ids instead of data in Redis)
        is_simple_match = False
        src_oid = metadata.get("source_object_id") or metadata.get("object_id")
        ref_oid = metadata.get("reference_object_id")

        # For JSON-created jobs, check the input field
        if not src_oid and "input" in metadata:
            inp = metadata["input"]
            src_oid = inp.get("object_id") or inp.get("source_object_id")
            ref_oid = inp.get("reference_object_id")

        if src_oid:
            is_simple_match = True
            log.info(
                f"[EXECUTE] Simple Match job detected with object_ids: src={src_oid}, ref={ref_oid}"
            )

        # Load source data - from Redis for regular jobs, from files for Simple Match
        source_data = None
        if is_simple_match:
            from ...storage import resolve_object_path, exists_object

            if src_oid and exists_object(src_oid):
                src_path = resolve_object_path(src_oid)
                with open(src_path, "rb") as f:
                    source_data = f.read()
                log.info(f"[EXECUTE] Loaded source data from file: {src_path}")
        else:
            source_data = await redis_conn.get(f"job:{job_id}:source_data")

        if not source_data:
            log.error(f"No source_data found for job:{job_id}")

            # Try fallback to file hash
            source_hash = metadata.get("source_hash") or await redis_conn.get(
                f"job:{job_id}:source_hash"
            )
            if source_hash:
                source_hash = (
                    source_hash.decode()
                    if isinstance(source_hash, bytes)
                    else source_hash
                )
                log.info(f"Trying fallback with source hash: {source_hash}")

                # Try file_profile cache
                file_profile_key = f"file_profile:{source_hash}"
                file_profile = await redis_conn.get(file_profile_key)
                if file_profile:
                    log.info(
                        f"Found file profile for hash {source_hash}, but need actual CSV data"
                    )

                # Try file_cache
                file_cache_key = f"file_cache:{source_hash}"
                source_data = await redis_conn.get(file_cache_key)
                if source_data:
                    log.info(f"Found source data in file_cache:{source_hash}")

            if not source_data:
                raise HTTPException(404, f"Source data not found for job {job_id}")

        log.info(
            f"Loading source data for job {job_id}, size: {len(source_data)} bytes"
        )
        source_df = read_csv_with_encoding(source_data)
        log.info(
            f"Source DataFrame loaded: {len(source_df)} rows, {len(source_df.columns)} columns"
        )

        # Enrich source with domain families if enabled and Domain column exists
        if (
            os.getenv("FEATURE_DOMAIN_FAMILY", "false").lower() == "true"
            and "Domain" in source_df.columns
        ):
            source_df, family_coverage = enrich_with_domain_families(source_df)
            log.info(f"Source domain family coverage: {family_coverage:.1f}%")

        # Load reference data if match mode
        ref_df = None
        if mode == "match":
            ref_data = None
            if is_simple_match:
                if ref_oid and exists_object(ref_oid):
                    ref_path = resolve_object_path(ref_oid)
                    with open(ref_path, "rb") as f:
                        ref_data = f.read()
                    log.info(f"[EXECUTE] Loaded reference data from file: {ref_path}")
            else:
                ref_data = await redis_conn.get(f"job:{job_id}:ref_data")

            if not ref_data:
                # Try fallback
                ref_hash = metadata.get("reference_hash") or await redis_conn.get(
                    f"job:{job_id}:ref_hash"
                )
                if ref_hash:
                    ref_hash = (
                        ref_hash.decode() if isinstance(ref_hash, bytes) else ref_hash
                    )
                    log.info(f"Trying fallback with reference hash: {ref_hash}")
                    ref_data = await redis_conn.get(f"file_cache:{ref_hash}")

            if ref_data:
                log.info(f"Loading reference data for job {job_id}")
                ref_df = read_csv_with_encoding(ref_data)
                log.info(
                    f"Reference DataFrame loaded: {len(ref_df)} rows, {len(ref_df.columns)} columns"
                )

                # Enrich reference with domain families if enabled and Domain column exists
                if (
                    os.getenv("FEATURE_DOMAIN_FAMILY", "false").lower() == "true"
                    and "Domain" in ref_df.columns
                ):
                    ref_df, family_coverage = enrich_with_domain_families(ref_df)
                    log.info(
                        f"Reference domain family coverage: {family_coverage:.1f}%"
                    )
            else:
                log.warning(
                    f"No reference data found for match mode job {job_id} - checking parquet fallback"
                )
                # Try loading from parquet file
                temp_dir = get_temp_dir(job_id)
                ref_parquet = temp_dir / "reference.parquet"
                if ref_parquet.exists():
                    log.info(f"Loading reference from parquet: {ref_parquet}")
                    ref_df = pd.read_parquet(ref_parquet)
                    log.info(
                        f"Reference DataFrame loaded from parquet: {len(ref_df)} rows"
                    )

                    # Enrich parquet-loaded reference with domain families if enabled
                    if (
                        os.getenv("FEATURE_DOMAIN_FAMILY", "false").lower() == "true"
                        and "Domain" in ref_df.columns
                    ):
                        ref_df, family_coverage = enrich_with_domain_families(ref_df)
                        log.info(
                            f"Reference (parquet) domain family coverage: {family_coverage:.1f}%"
                        )
                else:
                    log.error(
                        f"No reference data found in Redis or parquet for job {job_id}"
                    )
                    # Don't fail - let it continue to handle the error properly

        # Store configuration
        if redis_conn:
            await redis_conn.set(
                f"job:{job_id}:configuration", json.dumps(configuration), ex=3600
            )

            # Update status to processing with SIMPLE KEY
            metadata["status"] = "processing"
            metadata["configuration"] = configuration
            metadata["config"] = configuration
            metadata["progress"] = 10

            if warnings:
                existing = metadata.get("warnings")
                if isinstance(existing, list):
                    for warning in warnings:
                        if warning not in existing:
                            existing.append(warning)
                    metadata["warnings"] = existing
                elif existing:
                    combined = [existing]
                    combined.extend([w for w in warnings if w not in combined])
                    metadata["warnings"] = combined
                else:
                    metadata["warnings"] = warnings

            await redis_conn.set(f"job:{job_id}", json.dumps(metadata), ex=3600)

        # Process through the engine bridge
        log.info(f"Processing job {job_id} in {mode} mode")
        log.info(
            f"Source: {len(source_df)} rows, Reference: {len(ref_df) if ref_df is not None else 0} rows"
        )

        # Create output directory
        output_dir = get_temp_dir(job_id)
        output_dir.mkdir(parents=True, exist_ok=True)

        # Create progress callback for SSE/WebSocket updates
        async def progress_callback(event: ProgressEvent):
            """Handle progress events from the core engine"""
            try:
                log.debug(f"Progress: {event.phase} - {event.message}")

                # Get phase as string
                phase_str = (
                    event.phase if isinstance(event.phase, str) else event.phase.value
                )

                redis_data = {
                    "phase": phase_str,
                    "message": event.message,
                    "timestamp": datetime.utcnow().isoformat(),
                    "total_items": event.total_items,
                    "items_delta": getattr(event, "items_delta", 0),
                    "comparisons_delta": getattr(event, "comparisons_delta", 0),
                }

                # Calculate progress
                phase_progress = {
                    "INIT": 5,
                    "PREPROCESS": 10,
                    "BLOCKING": 20,
                    "COMPUTING": 50,
                    "DEDUPLICATING": 70,
                    "FINALIZING": 90,
                    "FINAL": 100,
                    "RUN": 50,
                }

                progress = phase_progress.get(phase_str, 50)
                redis_data["percent"] = progress

                # Store progress in Redis using async operations
                redis_conn = await get_redis()
                if redis_conn:
                    await redis_conn.set(
                        f"job:{job_id}:progress", json.dumps(redis_data), ex=300
                    )

                await update_job_status(job_id, "processing", progress=progress)

            except Exception as e:
                log.error(f"Error in progress_callback: {e}")

        # Defensive column checking for blocking strategy
        blocking_strategy = configuration.get("blocking_strategy")
        if blocking_strategy:
            # Check for different key variations
            source_block_col = (
                blocking_strategy.get("source_block_col")
                or blocking_strategy.get("source_column")
                or blocking_strategy.get("source_col")
            )
            ref_block_col = (
                blocking_strategy.get("ref_block_col")
                or blocking_strategy.get("reference_column")
                or blocking_strategy.get("ref_column")
                or blocking_strategy.get("reference_col")
                or source_block_col
            )  # Fallback to source column

            # Validate columns exist
            source_block_col_exists = (
                source_block_col and source_block_col in source_df.columns
            )
            ref_block_col_exists = ref_df is None or (
                ref_block_col and ref_block_col in ref_df.columns
            )

            if not source_block_col_exists:
                log.warning(
                    f"Source blocking column '{source_block_col}' not found in columns: {source_df.columns.tolist()}"
                )
                blocking_strategy = None  # Disable blocking
            elif not ref_block_col_exists:
                log.warning(
                    f"Reference blocking column '{ref_block_col}' not found in columns: {ref_df.columns.tolist() if ref_df is not None else []}"
                )
                blocking_strategy = None  # Disable blocking
            else:
                # Update with validated column names
                blocking_strategy["source_block_col"] = source_block_col
                blocking_strategy["ref_block_col"] = ref_block_col

            # CRITICAL: When blocking is disabled, ensure we don't use scale mode
            if blocking_strategy is None:
                log.warning(
                    "Blocking disabled - forcing simple mode to avoid empty results"
                )
                configuration["apply_blocking"] = False
                configuration["blocking_strategy"] = None

                # Check if we're in scale mode and switch to simple
                current_mode = configuration.get("mode", "simple")
                if current_mode == "scale":
                    log.info(
                        "Switching from 'scale' to 'simple' mode due to missing blocking columns"
                    )
                    configuration["mode"] = "simple"

        # Add default mappings if none provided
        if not configuration.get("mappings") or len(configuration["mappings"]) == 0:
            # Create default mappings based on column names
            log.warning("No mappings provided, creating default mappings")

            source_cols = list(source_df.columns)
            ref_cols = list(ref_df.columns) if ref_df is not None else []

            # Create basic mappings
            default_mappings = []
            if mode == "match" and ref_cols:
                # For match mode, try to map columns with similar names
                for s_col in source_cols:
                    for r_col in ref_cols:
                        # Simple name matching (customize as needed)
                        if (
                            s_col.lower() in r_col.lower()
                            or r_col.lower() in s_col.lower()
                        ):
                            default_mappings.append(
                                {
                                    "source": s_col,
                                    "reference": r_col,
                                    "similarity_measure": "token_sort_ratio",
                                }
                            )
                            break

                # If no automatic mappings found, create at least one
                if not default_mappings and source_cols and ref_cols:
                    default_mappings = [
                        {
                            "source": source_cols[0],
                            "reference": ref_cols[0],
                            "similarity_measure": "token_sort_ratio",
                        }
                    ]
            elif mode == "dedupe" and source_cols:
                # For dedupe mode, use first non-id column
                for col in source_cols:
                    if "id" not in col.lower():
                        default_mappings = [
                            {"source": col, "similarity_measure": "token_sort_ratio"}
                        ]
                        break
                if not default_mappings:
                    default_mappings = [
                        {
                            "source": source_cols[0],
                            "similarity_measure": "token_sort_ratio",
                        }
                    ]

            configuration["mappings"] = default_mappings
            log.info(
                f"Created {len(default_mappings)} default mappings: {default_mappings}"
            )

        if mode == "match" and ref_df is not None:
            # Use detected or configured ID columns - don't hardcode!
            # First try configuration, then metadata, then auto-detect
            source_id = configuration.get("source_id_column")
            if not source_id:
                # Try to get from metadata's first column (often the ID)
                source_cols = metadata.get("source_cols", [])
                if source_cols:
                    # Try to find a column that looks like an ID
                    from fmatch.core.engine import auto_select_id_column

                    detected_id = auto_select_id_column(source_df)
                    source_id = detected_id if detected_id else source_cols[0]
                else:
                    source_id = (
                        source_cols[0] if source_cols else "id"
                    )  # Use first column as fallback

            reference_id = configuration.get("reference_id_column")
            if not reference_id:
                ref_cols = metadata.get("ref_cols", [])
                if ref_cols:
                    from fmatch.core.engine import auto_select_id_column

                    detected_id = auto_select_id_column(ref_df)
                    reference_id = detected_id if detected_id else ref_cols[0]
                else:
                    reference_id = (
                        ref_cols[0] if ref_cols else "id"
                    )  # Use first column as fallback

            source_id = _select_best_id_column(source_id, source_df.columns)
            reference_id = _select_best_id_column(reference_id, ref_df.columns)

            # CRITICAL: Log and verify ID columns exist
            log.info("\n=== ID COLUMN DETECTION ===")
            log.info(f"Source ID column selected: '{source_id}'")
            log.info(f"Reference ID column selected: '{reference_id}'")
            log.info(f"Source columns available: {list(source_df.columns)}")
            log.info(f"Reference columns available: {list(ref_df.columns)}")

            # Verify columns exist
            if source_id not in source_df.columns:
                log.warning(
                    f"Source ID column '{source_id}' not found! Using first column: {source_df.columns[0]}"
                )
                source_id = source_df.columns[0]

            if reference_id not in ref_df.columns:
                log.warning(
                    f"Reference ID column '{reference_id}' not found! Using first column: {ref_df.columns[0]}"
                )
                reference_id = ref_df.columns[0]

            # Store in configuration for the engine to use
            configuration["source_id_column"] = source_id
            configuration["reference_id_column"] = reference_id
            log.info(
                f"Final ID columns: source='{source_id}', reference='{reference_id}'"
            )
            log.info("========================\n")

            # Prepare mappings with ensemble support
            enable_multi_algo = configuration.get("enable_multi_algo", True)
            mappings = []

            for m in configuration.get("mappings", []):
                mapping = {
                    "source": m.get("source"),
                    "ref": m.get("ref")
                    or m.get("reference"),  # Use 'ref' as required by engine
                    "weight": m.get("weight", 1.0),
                    "algorithm": m.get(
                        "algorithm", configuration.get("algorithm", "WRatio")
                    ),
                }

                # Add ensemble algorithms if multi-algo is enabled
                if enable_multi_algo:
                    # Check if field names suggest specific ensemble
                    field_name = (
                        str(m.get("source", ""))
                        + "|"
                        + str(m.get("ref", m.get("reference", "")))
                    ).lower()

                    # Set ensemble algorithms based on field type
                    if any(
                        term in field_name
                        for term in ["domain", "website", "url", "web"]
                    ):
                        mapping["algorithms"] = [
                            {"name": "TokenSetRatio", "weight": 0.5},
                            {"name": "WRatio", "weight": 0.3},
                            {"name": "JaroWinkler", "weight": 0.2},
                        ]
                    elif any(
                        term in field_name
                        for term in ["name", "company", "account", "org"]
                    ):
                        mapping["algorithms"] = [
                            {"name": "WRatio", "weight": 0.5},
                            {"name": "TokenSetRatio", "weight": 0.3},
                            {"name": "PartialRatio", "weight": 0.2},
                        ]
                    elif any(
                        term in field_name for term in ["address", "street", "location"]
                    ):
                        mapping["algorithms"] = [
                            {"name": "TokenSetRatio", "weight": 0.4},
                            {"name": "WRatio", "weight": 0.4},
                            {"name": "PartialRatio", "weight": 0.2},
                        ]
                    elif any(term in field_name for term in ["email", "mail"]):
                        mapping["algorithms"] = [
                            {"name": "Exact", "weight": 0.5},
                            {"name": "JaroWinkler", "weight": 0.3},
                            {"name": "WRatio", "weight": 0.2},
                        ]
                    elif any(term in field_name for term in ["id", "code", "number"]):
                        mapping["algorithms"] = [
                            {"name": "Exact", "weight": 0.7},
                            {"name": "JaroWinkler", "weight": 0.3},
                        ]
                    else:
                        # Default ensemble for general text fields
                        mapping["algorithms"] = [
                            {"name": "WRatio", "weight": 0.6},
                            {"name": "TokenSetRatio", "weight": 0.4},
                        ]

                    # Also set preferred_algo for backward compatibility
                    mapping["preferred_algo"] = mapping["algorithms"][0]["name"]

                mappings.append(mapping)

            # Debug what we're passing to the engine
            log.info(f"[ENGINE DEBUG] Source shape: {source_df.shape}")
            log.info(f"[ENGINE DEBUG] Source columns: {list(source_df.columns)[:10]}")
            log.info(
                f"[ENGINE DEBUG] Reference shape: {ref_df.shape if ref_df is not None else 'None'}"
            )
            log.info(
                f"[ENGINE DEBUG] Reference columns: {list(ref_df.columns)[:10] if ref_df is not None else 'None'}"
            )
            log.info(f"[ENGINE DEBUG] Mappings count: {len(mappings)}")
            log.info(
                f"[ENGINE DEBUG] First mapping: {mappings[0] if mappings else 'No mappings'}"
            )
            log.info(f"[ENGINE DEBUG] Source ID col: {source_id}")
            log.info(f"[ENGINE DEBUG] Reference ID col: {reference_id}")

            # Create MatchConfig
            config = MatchConfig(
                job_id=job_id,
                src_path=source_df,  # Pass DataFrame directly
                ref_path=ref_df,
                src_id_col=source_id,
                ref_id_col=reference_id,
                algorithm=configuration.get("algorithm", "WRatio"),
                threshold=configuration.get("threshold", 85),
                maps=mappings,
                apply_blocking=bool(blocking_strategy),
                source_block_col=blocking_strategy.get("source_block_col")
                if blocking_strategy
                else None,
                ref_block_col=blocking_strategy.get("ref_block_col")
                if blocking_strategy
                else None,
                block_key_length=blocking_strategy.get("block_key_len", 3)
                if blocking_strategy
                else 3,
                output_dir=output_dir,
            )

            # Pass ensemble flags to engine for advanced processing
            # Default to enable_multi_algo=True for better accuracy
            config.enable_multi_algo = enable_multi_algo
            for k in ["coverage_gate", "processing_mode", "max_workers"]:
                if k in configuration:
                    setattr(config, k, configuration[k])

            # Use the bridge!
            results_df, run_stats = await match(config, progress_callback)

            # Apply adaptive widening if enabled and yield is too low
            apply_adaptive = bool(configuration.get("adaptive_blocking", True))
            min_expected = max(
                50, int(0.002 * len(source_df))
            )  # ~0.2% of sources or at least 50

            if apply_adaptive and len(results_df) < min_expected:
                log.warning(
                    f"Low-yield run ({len(results_df)} < {min_expected}). Re-running with widened blocking."
                )

                # Widen blocking: reduce block key length and enable soft widening
                bs = configuration.get("blocking_strategy", {}) or {}
                bs["block_key_len"] = max(2, bs.get("block_key_len", 3) - 1)
                bs["soft_widen"] = True
                configuration["blocking_strategy"] = bs

                # Update config with widened blocking (rebuild MatchConfig)
                config_wide = MatchConfig(
                    job_id=job_id,
                    src_path=source_df,
                    ref_path=ref_df,
                    src_id_col=source_id,
                    ref_id_col=reference_id,
                    algorithm=configuration.get("algorithm", "WRatio"),
                    threshold=configuration.get("threshold", 85),
                    maps=mappings,
                    apply_blocking=bool(bs),
                    source_block_col=bs.get("source_block_col") if bs else None,
                    ref_block_col=bs.get("ref_block_col") if bs else None,
                    block_key_length=bs.get("block_key_len", 3) if bs else 3,
                    output_dir=output_dir,
                )
                # Preserve optional tuning flags
                config_wide.enable_multi_algo = enable_multi_algo
                for k in ["coverage_gate", "processing_mode", "max_workers"]:
                    if k in configuration:
                        setattr(config_wide, k, configuration[k])

                # Re-run scoring with wider blocks
                results_df_wide, _ = await match(config_wide, progress_callback)

                # Combine results and de-duplicate
                combined = pd.concat([results_df, results_df_wide], ignore_index=True)

                # Detect ID columns for deduplication
                src_col = None
                ref_col = None
                for c in [
                    "source_id",
                    configuration.get("source_id_column"),
                    "Lead ID",
                    "left_id",
                    "src_id",
                ]:
                    if c and c in combined.columns:
                        src_col = c
                        break
                for c in [
                    "reference_id",
                    configuration.get("reference_id_column"),
                    "18-Digit Account ID",
                    "ref_id",
                ]:
                    if c and c in combined.columns:
                        ref_col = c
                        break

                if src_col and ref_col:
                    combined = combined.drop_duplicates(
                        subset=[src_col, ref_col], keep="first"
                    )

                # Apply best_match_only if requested
                if configuration.get("best_match_only", True):
                    score_col = (
                        "score" if "score" in combined.columns else "confidence_score"
                    )
                    if src_col and score_col in combined.columns:
                        idx = combined.groupby(src_col, dropna=False)[
                            score_col
                        ].idxmax()
                        combined = combined.loc[idx].copy()

                log.info(
                    f"Adaptive widen kept {len(combined)} vs {len(results_df)} on first pass"
                )
                results_df = combined

        else:  # dedupe mode
            # Use detected or configured ID column - don't hardcode!
            source_id = configuration.get("source_id_column")
            if not source_id:
                source_cols = metadata.get("source_cols", [])
                if source_cols:
                    from fmatch.core.engine import auto_select_id_column

                    detected_id = auto_select_id_column(source_df)
                    source_id = detected_id if detected_id else source_cols[0]
                else:
                    source_id = (
                        source_df.columns[0] if len(source_df.columns) > 0 else "id"
                    )  # Use first column

            # CRITICAL: Log and verify ID column exists
            log.info("\n=== DEDUPE ID COLUMN DETECTION ===")
            log.info(f"Source ID column selected: '{source_id}'")
            log.info(f"Source columns available: {list(source_df.columns)}")

            # Verify column exists
            if source_id not in source_df.columns:
                log.warning(
                    f"Source ID column '{source_id}' not found! Using first column: {source_df.columns[0]}"
                )
                source_id = source_df.columns[0]

            # Store in configuration
            configuration["source_id_column"] = source_id
            log.info(f"Final ID column: '{source_id}'")
            log.info("========================\n")

            # Prepare mappings with ensemble support for dedupe
            enable_multi_algo = configuration.get("enable_multi_algo", True)
            dedupe_mappings = []

            for m in configuration.get("mappings", []):
                mapping = {
                    "column": m.get("source"),
                    "weight": m.get("weight", 1.0),
                    "algorithm": m.get(
                        "algorithm", configuration.get("algorithm", "WRatio")
                    ),
                }

                # Add ensemble algorithms if multi-algo is enabled
                if enable_multi_algo:
                    # Check if field names suggest specific ensemble
                    field_name = str(m.get("source", "")).lower()

                    # Set ensemble algorithms based on field type
                    if any(
                        term in field_name
                        for term in ["domain", "website", "url", "web"]
                    ):
                        mapping["algorithms"] = [
                            {"name": "TokenSetRatio", "weight": 0.5},
                            {"name": "WRatio", "weight": 0.3},
                            {"name": "JaroWinkler", "weight": 0.2},
                        ]
                    elif any(
                        term in field_name
                        for term in ["name", "company", "account", "org"]
                    ):
                        mapping["algorithms"] = [
                            {"name": "WRatio", "weight": 0.5},
                            {"name": "TokenSetRatio", "weight": 0.3},
                            {"name": "PartialRatio", "weight": 0.2},
                        ]
                    elif any(
                        term in field_name for term in ["address", "street", "location"]
                    ):
                        mapping["algorithms"] = [
                            {"name": "TokenSetRatio", "weight": 0.4},
                            {"name": "WRatio", "weight": 0.4},
                            {"name": "PartialRatio", "weight": 0.2},
                        ]
                    elif any(term in field_name for term in ["email", "mail"]):
                        mapping["algorithms"] = [
                            {"name": "Exact", "weight": 0.5},
                            {"name": "JaroWinkler", "weight": 0.3},
                            {"name": "WRatio", "weight": 0.2},
                        ]
                    elif any(term in field_name for term in ["id", "code", "number"]):
                        mapping["algorithms"] = [
                            {"name": "Exact", "weight": 0.7},
                            {"name": "JaroWinkler", "weight": 0.3},
                        ]
                    else:
                        # Default ensemble for general text fields
                        mapping["algorithms"] = [
                            {"name": "WRatio", "weight": 0.6},
                            {"name": "TokenSetRatio", "weight": 0.4},
                        ]

                    # Also set preferred_algo for backward compatibility
                    mapping["preferred_algo"] = mapping["algorithms"][0]["name"]

                dedupe_mappings.append(mapping)

            config = DedupeConfig(
                job_id=job_id,
                input_path=source_df,  # Pass DataFrame directly
                rec_id_col=source_id,
                algorithm=configuration.get("algorithm", "WRatio"),
                threshold=configuration.get("threshold", 85),
                maps=dedupe_mappings,
                apply_blocking=bool(blocking_strategy),
                block_col=blocking_strategy.get("source_block_col")
                if blocking_strategy
                else None,
                source_block_col=blocking_strategy.get("source_block_col")
                if blocking_strategy
                else None,
                block_key_length=blocking_strategy.get("block_key_len", 3)
                if blocking_strategy
                else 3,
                ruleset_name=configuration.get("ruleset_name", "default"),
                dup_block_limit=configuration.get(
                    "dup_block_limit", _settings.DEFAULT_DUP_BLOCK_LIMIT
                ),
                output_dir=output_dir,
            )

            # Pass ensemble flags to engine for advanced processing
            # Default to enable_multi_algo=True for better accuracy
            config.enable_multi_algo = enable_multi_algo
            for k in ["coverage_gate", "processing_mode", "max_workers"]:
                if k in configuration:
                    setattr(config, k, configuration[k])

            # Use the bridge!
            results_df, run_stats = await dedupe(config, progress_callback)

        # Store results
        if results_df is not None and not results_df.empty:
            # CRITICAL DEBUG: Log what the engine returned
            log.info("\n=== ENGINE RESULTS ANALYSIS ===")
            log.info(f"Results shape: {results_df.shape}")
            log.info(f"Results columns: {results_df.columns.tolist()}")
            log.info(f"Results dtypes: {results_df.dtypes.to_dict()}")
            if not results_df.empty:
                log.info("First 3 results:")
                for i in range(min(3, len(results_df))):
                    log.info(f"  Row {i}: {results_df.iloc[i].to_dict()}")
            log.info("================================\n")

            best_match_stats: Dict[str, Any] = {}
            candidates_total = len(results_df)
            confidence_bands = None

            best_only = bool(configuration.get("best_match_only", True))
            max_matches_config = configuration.get("max_matches_per_source")
            max_matches_per_source: Optional[int] = None
            if max_matches_config is not None:
                try:
                    max_matches_per_source = max(1, int(max_matches_config))
                except Exception:
                    max_matches_per_source = None

            if mode == "match" and not results_df.empty:
                target_per_source = max_matches_per_source or 1
                if (
                    best_only
                    or max_matches_per_source is not None
                    or target_per_source == 1
                ):
                    reduced_df, stats = apply_best_match_per_source(
                        results_df,
                        configuration=configuration,
                        max_per_source=target_per_source,
                        logger=log,
                    )
                    if stats.get("best_match_applied"):
                        best_match_stats = stats
                        results_df = reduced_df
                        log.info(
                            "best_match_only applied: kept %d of %d rows (max_per_source=%d)",
                            stats.get("kept_rows", len(results_df)),
                            stats.get("candidates_total", len(results_df)),
                            stats.get("max_per_source", target_per_source),
                        )
                    else:
                        log.warning(
                            "best-match reduction requested but source/score columns were not detected; continuing with full candidate set."
                        )
            candidates_total = best_match_stats.get(
                "candidates_total", candidates_total
            )

            # Ensure consistent column names for frontend
            # The frontend expects: source_id, reference_id (or ref_id), score, confidence
            log.info(
                f"Standardizing column names. Current columns: {results_df.columns.tolist()}"
            )

            # Rename columns to match frontend expectations
            column_mapping = {}

            # Map source ID columns
            for col in results_df.columns:
                if col in [
                    "src_id",
                    "source",
                    "source_idx",
                    "s_id",
                    "source_record_id",
                    "s_record_id",
                    "source_row_id",
                ]:
                    column_mapping[col] = "source_id"
                elif col in [
                    "ref_id",
                    "reference",
                    "reference_idx",
                    "target_id",
                    "ref_idx",
                    "r_id",
                ]:
                    column_mapping[col] = "reference_id"
                elif col in ["confidence_score", "match_score"]:
                    column_mapping[col] = "score"
                elif col == "confidence" and "score" not in results_df.columns:
                    column_mapping[col] = "score"

            if column_mapping:
                results_df = results_df.rename(columns=column_mapping)
                log.info(f"Renamed columns: {column_mapping}")

            # CRITICAL: Add aliases for actual ID columns to standard names
            # This ensures frontend always gets source_id and reference_id
            src_id_col = configuration.get("source_id_column", "Lead ID")
            ref_id_col = configuration.get("reference_id_column", "18-Digit Account ID")

            # Create source_id alias if the actual ID column exists
            if (
                src_id_col in results_df.columns
                and "source_id" not in results_df.columns
            ):
                results_df["source_id"] = results_df[src_id_col]
                log.info(f"Created source_id alias from '{src_id_col}'")

            # Create reference_id alias if the actual ID column exists
            if (
                ref_id_col in results_df.columns
                and "reference_id" not in results_df.columns
            ):
                results_df["reference_id"] = results_df[ref_id_col]
                log.info(f"Created reference_id alias from '{ref_id_col}'")

            # Ensure score column exists (as percentage 0-100)
            if "score" in results_df.columns:
                # Convert to percentage if it's in 0-1 range
                if results_df["score"].max() <= 1.0:
                    results_df["score"] = results_df["score"] * 100
            elif "confidence" in results_df.columns:
                results_df["score"] = results_df["confidence"] * 100

            if mode == "dedupe" and "score" in results_df.columns:
                try:
                    results_df["confidence_band"] = results_df["score"].apply(
                        band_for_score
                    )
                    confidence_bands = count_confidence_bands(
                        results_df["score"].tolist()
                    )
                except Exception as exc:
                    log.warning("Confidence band assignment failed: %s", exc)

            # field_score_details normalization removed - column is now dropped

            # CRITICAL: Ensure field_score_details is valid JSON for frontend
            def _strip_fka_from_df(df: pd.DataFrame) -> tuple[pd.DataFrame, int]:
                """Remove `(FKA â€¦)` style annotations from all string columns and count changes."""
                if df is None or df.empty:
                    return df, 0
                object_columns = df.select_dtypes(include=["object"]).columns
                if not len(object_columns):
                    return df, 0
                changes = 0
                for col in object_columns:
                    original = df[col]
                    sanitized = original.apply(strip_formerly_known_as)
                    diff_mask = sanitized != original
                    diff_mask = diff_mask.fillna(False)
                    changes += int(diff_mask.sum())
                    df[col] = sanitized
                return df, changes

            def _jsonify_field_details(v):
                import json

                if isinstance(v, (list, dict)):
                    return json.dumps(v, ensure_ascii=False)
                # leave empty if null-like
                if v in (
                    None,
                    "",
                    "nan",
                    "NaN",
                    pd.isna(v) if hasattr(pd, "isna") else False,
                ):
                    return "[]"
                # tolerate python-ish strings -> try to coerce
                s = str(v)
                s2 = (
                    s.replace("'", '"')
                    .replace(": None", ": null")
                    .replace(": True", ": true")
                    .replace(": False", ": false")
                )
                try:
                    json.loads(s2)  # validate
                    return s2
                except Exception:
                    return "[]"

            # Drop field_score_details column as it's always empty
            if "field_score_details" in results_df.columns:
                results_df = results_df.drop(columns=["field_score_details"])
                log.info("Dropped field_score_details column (always empty)")

            # Ensure reasons column is present
            if "reasons" not in results_df.columns:
                log.warning(
                    "reasons column missing - this should have been added by worker"
                )
                try:
                    from ...utils.reason_codes import (
                        add_reason_codes_to_results,
                        update_match_status_with_reasons,
                    )

                    reason_source_col = configuration.get("source_id_column")
                    reason_ref_col = configuration.get("reference_id_column")
                    mapping_candidates = (
                        configuration.get("mappings") or configuration.get("maps") or []
                    )
                    results_df = add_reason_codes_to_results(
                        results_df,
                        source_df,
                        ref_df,
                        mapping_candidates,
                        source_id_col=reason_source_col,
                        ref_id_col=reason_ref_col,
                    )
                    results_df = update_match_status_with_reasons(results_df)
                    if "reasons" in results_df.columns:
                        results_df["reasons"] = results_df["reasons"].apply(
                            normalize_reason_tokens
                        )
                    log.info("Regenerated reasons column inline after worker failure")
                except Exception as reason_err:
                    log.warning("Inline reason regeneration failed: %s", reason_err)
                    results_df["reasons"] = ""
            elif results_df["reasons"].isna().all():
                try:
                    from ...utils.reason_codes import update_match_status_with_reasons

                    results_df["reasons"] = results_df["reasons"].fillna("")
                    results_df = update_match_status_with_reasons(results_df)
                    results_df["reasons"] = results_df["reasons"].apply(
                        normalize_reason_tokens
                    )
                except Exception:
                    results_df["reasons"] = results_df["reasons"].fillna("")
                    results_df["reasons"] = results_df["reasons"].apply(
                        normalize_reason_tokens
                    )

            if "reasons" in results_df.columns:
                empty_reasons = int(results_df["reasons"].fillna("").eq("").sum())
                log.info(
                    f"[EXECUTE] Reasons empty rows: {empty_reasons}/{len(results_df)}"
                )

            if "match_reason" not in results_df.columns:

                def _summarize_reason(row):
                    parts = []
                    name = (
                        row.get("matched_account_name")
                        or row.get("reference_name")
                        or row.get("Account Name")
                    )
                    if name:
                        parts.append(f"Match: {name}")
                    reasons = row.get("reasons")
                    if reasons:
                        parts.append(f"Reasons {reasons}")
                    score_val = row.get("match_score") or row.get("score")
                    try:
                        if score_val not in (None, ""):
                            score_num = float(score_val)
                            if score_num:
                                parts.append(f"Score {int(round(score_num))}")
                    except Exception:
                        pass
                    return " | ".join(parts) if parts else ""

                results_df["match_reason"] = results_df.apply(_summarize_reason, axis=1)

            results_df = _ensure_matched_domain_family(
                results_df,
                ref_df if mode == "match" else None,
                configuration.get("reference_id_column"),
            )

            log.info(f"Final columns: {results_df.columns.tolist()}")
            if not results_df.empty:
                log.info(
                    f"Sample result after renaming: {results_df.iloc[0].to_dict()}"
                )

            # Clean NaN values before storage
            log.info(
                f"Cleaning NaN values from {len(results_df)} results before storage"
            )

            # Strip `(FKA â€¦)` annotations from displayable string columns
            results_df, sanitizer_hits = _strip_fka_from_df(results_df)
            if sanitizer_hits:
                metrics_bucket = metadata.setdefault("metrics", {})
                metrics_bucket["sanitized_values"] = (
                    metrics_bucket.get("sanitized_values", 0) + sanitizer_hits
                )

            # Replace NaN with None for JSON compatibility
            clean_results_df = results_df.replace({np.nan: None})

            # For score columns, replace None with 0 (better than null for sorting)
            score_columns = [
                col
                for col in clean_results_df.columns
                if ("score" in col.lower() or "confidence" in col.lower())
                and not col.lower().endswith("_band")
            ]
            for col in score_columns:
                if col in clean_results_df.columns:
                    clean_results_df[col] = clean_results_df[col].fillna(0)
                    log.debug(f"Filled NaN values in {col} with 0")

            # Handle any infinite values
            clean_results_df = clean_results_df.replace([np.inf, -np.inf], None)

            # Save cleaned data to temp file first
            temp_results_path = output_dir / "results.csv"
            clean_results_df.to_csv(temp_results_path, index=False)
            log.info(
                f"Saved {len(clean_results_df)} clean results to {temp_results_path}"
            )

            # Copy to canonical uploads/results location
            from ...storage import resolve_uploads_base
            import shutil

            base = resolve_uploads_base()
            results_dir = os.path.join(base, "results")
            os.makedirs(results_dir, exist_ok=True)
            canonical_path = os.path.join(results_dir, f"{job_id}.csv")
            shutil.copy2(temp_results_path, canonical_path)
            log.info(f"Copied results to canonical location: {canonical_path}")

            # Set the result object ID for storage references
            result_object_id = f"uploads/results/{job_id}.csv"

            # Store in Redis for retrieval with enhanced stats
            enhanced_stats = {
                "source_rows": int(len(source_df)),
                "reference_rows": int(len(ref_df)) if ref_df is not None else 0,
                "matches_found": int(len(clean_results_df)),
                "threshold_used": int(configuration.get("threshold", 85)),
                "blocking_applied": bool(configuration.get("blocking_strategy")),
                "algorithm_used": configuration.get("algorithm", "WRatio"),
                "engine_used": "bridge",
                "duration": getattr(
                    run_stats, "total_duration", getattr(run_stats, "duration", 0)
                )
                if run_stats
                else 0,
                # Legacy naming for backward compatibility
                "total_records_source": int(len(source_df)),
                "total_records_reference": int(len(ref_df))
                if ref_df is not None
                else 0,
                "candidates_total": int(
                    best_match_stats.get("candidates_total", candidates_total)
                ),
            }

            # Detect source ID column and add unique sources count
            src_col = None
            for c in [
                "source_id",
                configuration.get("source_id_column"),
                "Lead ID",
                "left_id",
                "src_id",
                "s_id",
                "source_record_id",
                "s_record_id",
                "source_row_id",
            ]:
                if c and c in clean_results_df.columns:
                    src_col = c
                    break
            if src_col:
                enhanced_stats["unique_sources_w_candidates"] = int(
                    clean_results_df[src_col].nunique()
                )

            # Add best_match_only stats if they exist
            if best_match_stats:
                enhanced_stats.update(best_match_stats)
            if confidence_bands:
                enhanced_stats["confidence_bands"] = [
                    band.model_dump() for band in confidence_bands
                ]

            results = {
                "matches": clean_results_df.to_dict("records"),
                "stats": enhanced_stats,
                "result_object_id": result_object_id,  # Add pointer to results file
                "result_rows": len(clean_results_df),
            }

            # Fixed: Using Windows-compatible check mark
            message = f"{CHECK_MARK} Found {len(clean_results_df)} {'matches' if mode == 'match' else 'duplicates'}!"
            duration = (
                getattr(run_stats, "total_duration", getattr(run_stats, "duration", 0))
                if run_stats
                else 0
            )
            log.info(f"{message} Duration: {duration:.2f}s" if duration else message)
        else:
            results = {
                "matches": [],
                "stats": {
                    "total_records_source": len(source_df),
                    "total_records_reference": len(ref_df) if ref_df is not None else 0,
                    "matches_found": 0,
                    "engine_used": "bridge",
                    "duration": getattr(
                        run_stats, "total_duration", getattr(run_stats, "duration", 0)
                    )
                    if "run_stats" in locals() and run_stats
                    else 0,
                },
            }
            message = "No matches found"

        # Store results using helper function
        await store_job_results(job_id, results)

        # Update Redis job document with result pointers
        redis_conn = await get_redis()
        if redis_conn:
            job_key = f"job:{job_id}"
            saved = await redis_conn.get(job_key)
            job_meta = json.loads(saved) if saved else {}
            job_meta["result_object_id"] = results.get(
                "result_object_id", f"uploads/results/{job_id}.csv"
            )
            job_meta["result_rows"] = results.get("result_rows", 0)
            job_meta["matches_found"] = results.get("result_rows", 0)
            # Preserve metrics from metadata (including sanitized_values)
            if "metrics" in metadata:
                job_meta["metrics"] = metadata["metrics"]
            await redis_conn.setex(job_key, 86400, json.dumps(job_meta))
            log.info(
                f"Updated Redis with result pointers: result_object_id={job_meta['result_object_id']}, result_rows={job_meta['result_rows']}"
            )

        # Update database with results_path
        rec = await session.get(Job, job_id)
        if rec:
            rec.results_path = results.get(
                "result_object_id", f"uploads/results/{job_id}.csv"
            )
            rec.matches_found = results.get("result_rows", 0)
            await session.commit()
            log.info(
                f"Updated database with results_path={rec.results_path}, matches_found={rec.matches_found}"
            )

        # Update status to completed using helper function
        await update_job_status(job_id, "completed", progress=100, db=session)

        matches_count = len(results.get("matches", []))

        return {
            "job_id": job_id,
            "status": "completed",
            "message": message,
            "matches_found": matches_count,
            "stats": results.get("stats", {}),
            "_links": {
                "self": f"/api/v2/jobs/{job_id}",
                "status": f"/api/v2/jobs/{job_id}/status",
                "results": f"/api/v2/jobs/{job_id}/results",
            },
        }

    except Exception as e:
        log.error(f"Error processing job {job_id}: {e}")
        import traceback

        log.error(traceback.format_exc())

        # Update status to failed
        await update_job_status(job_id, "failed", error=str(e), db=session)

        raise HTTPException(500, f"Failed to execute job: {str(e)}")


@router.get("/{job_id}/status")
async def get_job_status(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    """Return current job status with counters for UI polling."""
    log.info(f"[STATUS] Checking status for job {job_id}")

    try:
        redis_conn = await get_redis()
    except Exception:
        redis_conn = None

    rec = await db.get(Job, job_id)
    if not rec or (rec.account_id and str(rec.account_id) != str(account_id)):
        log.error(f"[STATUS] Job {job_id} not found in database")
        raise HTTPException(404, "job not found")

    metadata: Dict[str, Any] = {}
    if redis_conn:
        job_key = f"job:{job_id}"
        try:
            raw = await redis_conn.get(job_key)
        except Exception:
            raw = None
        if raw:
            try:
                if isinstance(raw, bytes):
                    raw = raw.decode("utf-8")
                metadata = json.loads(raw)
            except Exception:
                metadata = {}
        summary_key = f"job:summary:{job_id}"
        try:
            summary_raw = await redis_conn.get(summary_key)
        except Exception:
            summary_raw = None
        if summary_raw:
            try:
                if isinstance(summary_raw, bytes):
                    summary_raw = summary_raw.decode("utf-8")
                summary_payload = json.loads(summary_raw)
            except Exception:
                summary_payload = None
            if isinstance(summary_payload, dict):
                metadata.setdefault("summary", summary_payload)
                metrics_block = metadata.get("metrics")
                if isinstance(metrics_block, dict):
                    metrics_block.setdefault("summary", summary_payload)
                else:
                    metadata["metrics"] = {"summary": summary_payload}

    redis_status_val = str(
        metadata.get("status") or metadata.get("state") or ""
    ).lower()
    status_val = redis_status_val or "unknown"
    if hasattr(rec.status, "value"):
        status_val = rec.status.value
    elif rec.status:
        status_val = str(rec.status)

    # Normalize to lowercase for client compatibility
    status_val = str(status_val).lower()
    # Prefer live Redis state when DB still shows queued/pending.
    if (
        status_val in {"queued", "pending", "created", "unknown"}
        and redis_status_val
        and redis_status_val
        not in {"queued", "pending", "created", "unknown"}
    ):
        status_val = redis_status_val

    matches_found = (
        rec.matches_found
        or metadata.get("matches_found")
        or metadata.get("total_matches")
        or 0
    )
    try:
        matches_found = int(matches_found)
    except Exception:
        matches_found = 0

    config_dict = rec.config if isinstance(rec.config, dict) else {}
    rows_processed = (
        metadata.get("processed_rows") or metadata.get("rows_processed") or 0
    )
    if not rows_processed and status_val in {"completed", "succeeded"}:
        try:
            rows_processed = int(
                config_dict.get("source_rows") or config_dict.get("rows_processed") or 0
            )
        except Exception:
            rows_processed = 0
    try:
        rows_processed = int(rows_processed)
    except Exception:
        rows_processed = 0

    candidates_total = metadata.get("candidates_total")
    if isinstance(candidates_total, str):
        try:
            candidates_total = int(candidates_total)
        except Exception:
            candidates_total = None

    elapsed_s: Optional[float] = None
    if rec.created_at and rec.updated_at:
        try:
            elapsed_s = max(0.0, (rec.updated_at - rec.created_at).total_seconds())
        except Exception:
            elapsed_s = None

    progress_raw = metadata.get("progress")
    if progress_raw is None:
        progress_raw = metadata.get("progress_pct")
    metrics = metadata.get("metrics", {})
    if progress_raw is None and isinstance(metrics, dict):
        progress_raw = metrics.get("progress_pct")
    progress = _coerce_progress_percent(progress_raw)
    if progress is None:
        progress = 0
    if progress == 0:
        total_rows_hint = (
            metadata.get("total_rows")
            or metadata.get("rows_total")
            or config_dict.get("source_rows")
            or 0
        )
        try:
            total_rows_hint = int(total_rows_hint)
        except Exception:
            total_rows_hint = 0
        if total_rows_hint > 0 and rows_processed > 0:
            progress = int(
                max(0, min(100, round((rows_processed / float(total_rows_hint)) * 100)))
            )
    if status_val in {"completed", "succeeded"}:
        progress = 100

    message = metadata.get("message", "")
    if not message:
        if status_val in {"queued", "pending"}:
            message = "Account Hunter job queued"
        elif status_val in {"processing", "running"}:
            message = "Account Hunter is running"
    error_val = (
        metadata.get("error")
        if (status_val in {"failed", "error", "cancelled", "canceled"})
        else None
    )
    if not error_val:
        rec_error = getattr(rec, "error", None)
        if rec_error and status_val in {"failed", "error", "cancelled", "canceled"}:
            error_val = rec_error
    mode = metadata.get("mode") or config_dict.get("mode")
    updated_at = metadata.get("updated_at") or (
        rec.updated_at.isoformat() if rec.updated_at else None
    )

    if status_val == "completed" and rec.results_path and matches_found == 0:
        log.warning("Job %s completed but matches_found=0 during status poll", job_id)

    normalized_results_path = _normalize_results_path(rec.results_path)

    # Get result_object_id from Redis metadata or database
    result_object_id = metadata.get("result_object_id") or normalized_results_path
    result_rows = metadata.get("result_rows", matches_found)

    # Extract Phase-2 metrics if available
    if not isinstance(metrics, dict):
        metrics = {}

    summary = metadata.get("summary")
    if not summary and isinstance(rec.results, dict):
        summary = rec.results.get("summary")

    results_payload = metadata.get("results")
    if not results_payload and isinstance(rec.results, dict):
        results_payload = rec.results

    errors_payload = metadata.get("errors")

    response = {
        "job_id": job_id,
        "status": status_val,
        "state": status_val,  # Include 'state' for compatibility with Simple Match wizard
        "request_id": metadata.get("request_id") or config_dict.get("request_id"),
        "mode": mode,
        "progress": progress,
        "progress_pct": progress,  # Alias used by Sheets add-on pollers
        "message": message,
        "error": error_val,
        "matches_found": matches_found,
        "rows_processed": rows_processed,
        "elapsed_s": elapsed_s,
        "results_path": normalized_results_path,
        "result_object_id": result_object_id,  # Add for Simple Match wizard
        "result_rows": result_rows,  # Add for Simple Match wizard
        "effective_config": metadata.get("effective_config"),
        "warnings": metadata.get("warnings") or metrics.get("warnings"),
        "fallback_rows_pct": metrics.get("fallback_rows_pct"),
        "fallback_rows": metrics.get("fallback_rows"),
        "updated_at": updated_at,
        "candidates_total": candidates_total,
        # Phase-2 B2B intelligence metrics
        "blocked_pairs": metrics.get("blocked_pairs"),
        "family_hits": metrics.get("family_hits"),
        "website_hits": metrics.get("website_hits"),
        "geo_nudges": metrics.get("geo_nudges"),
        "b2c_filtered": metrics.get("b2c_filtered"),
        "no_match_rows": metrics.get("no_match_rows"),
        "engine_ms": metrics.get("engine_ms"),
        "write_ms": metrics.get("write_ms"),
        "domain_expansions": metrics.get("domain_expansions"),
        "sanitized_values": metrics.get("sanitized_values"),
        # Match status counts for QA
        "account_matches": metrics.get("account_matches"),
        "possible_matches": metrics.get("possible_matches"),
        "no_matches": metrics.get("no_matches"),
        # New surgical improvements metrics
        "domain_gated_rows": metrics.get("domain_gated_rows"),
        "domain_gated_hits": metrics.get("domain_gated_hits"),
        "confusable_penalties": metrics.get("confusable_penalties"),
        "account_match_with_domain_pct": metrics.get("account_match_with_domain_pct"),
        "possible_match_name_only_pct": metrics.get("possible_match_name_only_pct"),
        # Return full metrics object for UI access
        "metrics": metrics,
        "summary": summary,
    }

    if results_payload:
        response["results"] = results_payload
    if isinstance(errors_payload, dict):
        response["errors"] = errors_payload.get("errors", [])
        response["errors_truncated"] = bool(errors_payload.get("errors_truncated"))

    log.info(
        f"[STATUS] Returning status for job {job_id}: state={status_val} (normalized to lowercase), progress={progress}, matches={matches_found}, result_object_id={result_object_id}"
    )
    return response


@router.get("/{job_id}/stream")
async def stream_job_status(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    """
    Stream job status updates via Server-Sent Events for real-time progress monitoring
    """
    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    async def generate():
        last_status = None
        while True:
            try:
                # Get job status from Redis
                job_key = f"job:{job_id}"
                metadata_raw = await redis_conn.get(job_key)

                if metadata_raw:
                    metadata = json.loads(metadata_raw)
                    status = metadata.get("status", "unknown")

                    # Send update if status changed or progress updated
                    current_status = {
                        "status": status,
                        "progress": metadata.get("progress", 0),
                        "current_step": metadata.get("current_step", ""),
                        "total_rows": metadata.get("total_rows", 0),
                        "processed_rows": metadata.get("processed_rows", 0),
                        "message": metadata.get("message", ""),
                    }

                    if current_status != last_status:
                        yield {"event": "progress", "data": json.dumps(current_status)}
                        last_status = current_status

                    # Exit if job is complete or failed
                    if status in ["completed", "failed", "error"]:
                        yield {
                            "event": "done",
                            "data": json.dumps(
                                {
                                    "status": status,
                                    "message": metadata.get("message", "Job finished"),
                                }
                            ),
                        }
                        break
                else:
                    # Job not found
                    yield {
                        "event": "error",
                        "data": json.dumps({"error": f"Job {job_id} not found"}),
                    }
                    break

            except Exception as e:
                log.error(f"Error streaming job {job_id}: {e}")
                yield {"event": "error", "data": json.dumps({"error": str(e)})}
                break

            # Wait before next check
            await asyncio.sleep(1)

    return EventSourceResponse(generate())


@router.get("/{job_id}/results-legacy", include_in_schema=False)
async def get_job_results_legacy(
    job_id: str,
    format: str = "json",
    limit: int = 100,
    offset: int = 0,
    sort_by: str = "confidence_score",
    compression: str = "gzip",  # Support compression parameter
    columns: str = None,  # Optional column pruning
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    """
    Get paginated results of a completed job.
    Efficiently reads only the requested slice from results file.
    """
    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    # Check job status
    job_key = f"job:{job_id}"
    metadata_raw = await redis_conn.get(job_key)
    if not metadata_raw:
        raise HTTPException(404, f"Job {job_id} not found at key {job_key}")

    metadata = json.loads(metadata_raw)

    if metadata.get("status") != "completed":
        return {
            "job_id": job_id,
            "status": metadata.get("status", "unknown"),
            "message": "Job not yet completed",
            "progress": metadata.get("progress", 0),
        }

    # Check if job has result_object_id (from storage-based processing)
    result_object_id = metadata.get("result_object_id")

    # Handle CSV format export with optional compression
    if format.lower() == "csv":
        # First check if results are in storage (uploads/results/)
        if result_object_id:
            from ...storage import resolve_object_path, exists_object

            if exists_object(result_object_id):
                results_path = Path(resolve_object_path(result_object_id))
                log.info(f"Serving results from storage: {results_path}")

                # Direct file serving for storage results
                if not columns and compression != "gzip":
                    return FileResponse(
                        path=results_path,
                        media_type="text/csv",
                        filename=f"{job_id}_results.csv",
                    )
                else:
                    # Need to process the file
                    df = pd.read_csv(results_path)

                    # Apply column filtering if requested
                    if columns:
                        selected_cols = columns.split(",")
                        available_cols = [
                            col for col in selected_cols if col in df.columns
                        ]
                        if available_cols:
                            df = df[available_cols]

                    # Return processed CSV
                    output = io.StringIO()
                    df.to_csv(output, index=False)
                    return Response(
                        content=output.getvalue(),
                        media_type="text/csv",
                        headers={
                            "Content-Disposition": f'attachment; filename="{job_id}_results.csv"'
                        },
                    )

        # Fall back to temp directory (legacy path)
        output_dir = get_temp_dir(job_id)
        results_file = output_dir / "results.csv"

        # Fast-path: if no column pruning needed and file exists, stream directly
        if os.path.exists(results_file) and not columns:
            if compression == "gzip":
                # Stream to temp gz without loading into pandas
                with tempfile.NamedTemporaryFile(
                    mode="wb", suffix=".csv.gz", delete=False
                ) as tmp:
                    with open(results_file, "rb") as src, gzip.open(tmp, "wb") as gz:
                        for chunk in iter(lambda: src.read(1024 * 1024), b""):
                            gz.write(chunk)
                    tmp_path = tmp.name

                def iterfile():
                    try:
                        with open(tmp_path, "rb") as f:
                            yield from f
                    finally:
                        os.unlink(tmp_path)

                return StreamingResponse(
                    iterfile(),
                    media_type="application/gzip",
                    headers={
                        "Content-Encoding": "gzip",
                        "Content-Disposition": f'attachment; filename="{job_id}_results.csv.gz"',
                    },
                )
            else:
                # Use FileResponse for direct file serving (simpler and handles Windows paths better)
                return FileResponse(
                    path=results_file,
                    media_type="text/csv",
                    filename=f"{job_id}_results.csv",
                )

        # Column pruning path - requires pandas
        elif os.path.exists(results_file) and columns:
            selected_cols = columns.split(",") if columns else None
            df = pd.read_csv(results_file)

            # Rename columns to match frontend expectations BEFORE filtering
            column_mapping = {
                "src_id": "source_id",
                "ref_id": "reference_id",
                "match_score": "score",
                "confidence_score": "score",
            }
            rename_dict = {
                old: new for old, new in column_mapping.items() if old in df.columns
            }
            if rename_dict:
                df = df.rename(columns=rename_dict)

            # Convert score to 0-100 scale if needed
            if "score" in df.columns:
                if df["score"].max() <= 1:
                    df["score"] = df["score"] * 100

            # Filter to selected columns that exist
            if selected_cols:
                available_cols = [col for col in selected_cols if col in df.columns]
                if available_cols:
                    df = df[available_cols]

            # Downcast numeric types to reduce size
            for col in df.columns:
                if df[col].dtype in ["float64"]:
                    df[col] = pd.to_numeric(df[col], downcast="float")
                elif df[col].dtype in ["int64"]:
                    df[col] = pd.to_numeric(df[col], downcast="integer")

        elif os.path.exists(results_file):
            # Load file without column pruning
            df = pd.read_csv(results_file)

            # Rename columns to match frontend expectations
            column_mapping = {
                "src_id": "source_id",
                "ref_id": "reference_id",
                "match_score": "score",
                "confidence_score": "score",
            }
            rename_dict = {
                old: new for old, new in column_mapping.items() if old in df.columns
            }
            if rename_dict:
                df = df.rename(columns=rename_dict)

            # Convert score to 0-100 scale if needed
            if "score" in df.columns:
                if df["score"].max() <= 1:
                    df["score"] = df["score"] * 100

            # Downcast all numeric types
            for col in df.columns:
                if df[col].dtype in ["float64"]:
                    df[col] = pd.to_numeric(df[col], downcast="float")
                elif df[col].dtype in ["int64"]:
                    df[col] = pd.to_numeric(df[col], downcast="integer")

        # Apply compression if requested (for both column pruning and regular paths)
        if os.path.exists(results_file) and "df" in locals():
            if compression == "gzip":
                # Write compressed CSV to temp file
                with tempfile.NamedTemporaryFile(
                    mode="wb", suffix=".csv.gz", delete=False
                ) as tmp:
                    with gzip.open(tmp, "wt", encoding="utf-8") as gz:
                        df.to_csv(gz, index=False)
                    tmp_path = tmp.name

                # Stream the compressed file
                def iterfile():
                    try:
                        with open(tmp_path, "rb") as f:
                            yield from f
                    finally:
                        os.unlink(tmp_path)  # Clean up temp file

                return StreamingResponse(
                    iterfile(),
                    media_type="text/csv",
                    headers={
                        "Content-Type": "text/csv",
                        "Content-Encoding": "gzip",
                        "Content-Disposition": f'attachment; filename="{job_id}_results.csv.gz"',
                    },
                )
            else:
                # Stream uncompressed CSV
                output = io.StringIO()
                df.to_csv(output, index=False)
                output.seek(0)

                return StreamingResponse(
                    io.StringIO(output.getvalue()),
                    media_type="text/csv",
                    headers={
                        "Content-Disposition": f'attachment; filename="{job_id}_results.csv"'
                    },
                )
        else:
            # Generate CSV from stored results
            import csv

            # Try to get results from Redis
            results_raw = await redis_conn.get(f"job:{job_id}:results")
            if results_raw:
                results = json.loads(results_raw)
                matches = results.get("matches", [])

                if matches:
                    # Generate CSV
                    def generate_csv():
                        output = io.StringIO()
                        writer = csv.DictWriter(output, fieldnames=matches[0].keys())

                        # Write header
                        writer.writeheader()
                        output.seek(0)
                        yield output.read()
                        output.truncate(0)

                        # Write rows
                        for match in matches:
                            writer.writerow(match)
                            output.seek(0)
                            yield output.read()
                            output.truncate(0)

                    return StreamingResponse(
                        generate_csv(),
                        media_type="text/csv",
                        headers={
                            "Content-Disposition": f'attachment; filename="{job_id}_results.csv"'
                        },
                    )

            # No results found
            return StreamingResponse(
                io.StringIO("No results found"),
                media_type="text/csv",
                headers={
                    "Content-Disposition": f'attachment; filename="{job_id}_empty.csv"'
                },
            )

    # Try to load results from file if available (more efficient for large results)
    output_dir = get_temp_dir(job_id)
    results_file = output_dir / "results.csv"

    if os.path.exists(results_file):
        # Use pandas for efficient pagination
        try:
            # First, get total count without loading all data
            total_matches = sum(1 for _ in open(results_file)) - 1  # -1 for header

            if total_matches == 0:
                return {
                    "job_id": job_id,
                    "status": "completed",
                    "results": [],
                    "pagination": {
                        "total": 0,
                        "limit": limit,
                        "offset": offset,
                        "has_more": False,
                    },
                    "message": "No matches found",
                }

            # Read only the requested slice
            if offset >= total_matches:
                # Offset beyond results
                paginated_matches = []
            else:
                # Use pandas with skiprows and nrows for efficiency
                df = pd.read_csv(
                    results_file, skiprows=range(1, offset + 1), nrows=limit
                )

                # Rename columns to match frontend expectations
                column_mapping = {
                    "src_id": "source_id",
                    "ref_id": "reference_id",
                    "match_score": "score",
                    "confidence_score": "score",
                }
                # Apply renaming for any columns that exist
                rename_dict = {
                    old: new for old, new in column_mapping.items() if old in df.columns
                }
                if rename_dict:
                    df = df.rename(columns=rename_dict)

                # Convert score to 0-100 scale if needed
                if "score" in df.columns:
                    if df["score"].max() <= 1:
                        df["score"] = df["score"] * 100

                # Clean NaN values for JSON serialization
                if not df.empty:
                    # Replace NaN with None for JSON compatibility
                    df = df.replace({np.nan: None})

                    # For numeric columns that should have scores, replace None with 0
                    score_columns = [
                        col
                        for col in df.columns
                        if "score" in col.lower() or "confidence" in col.lower()
                    ]
                    for col in score_columns:
                        if col in df.columns:
                            # Use infer_objects() to avoid FutureWarning about downcasting
                            df[col] = df[col].fillna(0).infer_objects(copy=False)

                    # Handle any infinite values too
                    df = df.replace([np.inf, -np.inf], None)

                    # Drop field_score_details column as it's always empty
                    if "field_score_details" in df.columns:
                        df = df.drop(columns=["field_score_details"])

                # Sort if requested
                if sort_by and sort_by in df.columns:
                    df = df.sort_values(by=sort_by, ascending=False, na_position="last")

                paginated_matches = df.to_dict("records")

            # Get stats from Redis if available
            stats_raw = await redis_conn.get(f"job:{job_id}:results")
            stats = {}
            if stats_raw:
                results = json.loads(stats_raw)
                stats = results.get("stats", {})

            return {
                "job_id": job_id,
                "status": "completed",
                "results": paginated_matches,
                "pagination": {
                    "total": total_matches,
                    "limit": limit,
                    "offset": offset,
                    "has_more": (offset + limit) < total_matches,
                },
                "stats": stats,
                "mode": metadata.get("mode"),
                "_links": {
                    "self": f"/api/v2/jobs/{job_id}/results?limit={limit}&offset={offset}",
                    "next": f"/api/v2/jobs/{job_id}/results?limit={limit}&offset={offset + limit}"
                    if (offset + limit) < total_matches
                    else None,
                    "prev": f"/api/v2/jobs/{job_id}/results?limit={limit}&offset={max(0, offset - limit)}"
                    if offset > 0
                    else None,
                },
            }
        except Exception as e:
            log.warning(f"Failed to read results file, falling back to Redis: {e}")

    # Fallback to Redis if file not available
    results_raw = await redis_conn.get(f"job:{job_id}:results")
    if not results_raw:
        return {
            "job_id": job_id,
            "status": "completed",
            "results": [],
            "pagination": {
                "total": 0,
                "limit": limit,
                "offset": offset,
                "has_more": False,
            },
            "message": "No matches found",
        }

    results = json.loads(results_raw)

    # Apply pagination to matches
    matches = results.get("matches", [])
    total_matches = len(matches)

    # Sort if requested
    if sort_by and matches and sort_by in matches[0]:
        matches = sorted(matches, key=lambda x: x.get(sort_by, 0), reverse=True)

    paginated_matches = (
        matches[offset : offset + limit] if offset < total_matches else []
    )

    return {
        "job_id": job_id,
        "status": "completed",
        "results": paginated_matches,
        "pagination": {
            "total": total_matches,
            "limit": limit,
            "offset": offset,
            "has_more": (offset + limit) < total_matches,
        },
        "stats": results.get("stats", {}),
        "mode": results.get("mode", metadata.get("mode")),
        "_links": {
            "self": f"/api/v2/jobs/{job_id}/results?limit={limit}&offset={offset}",
            "next": f"/api/v2/jobs/{job_id}/results?limit={limit}&offset={offset + limit}"
            if (offset + limit) < total_matches
            else None,
            "prev": f"/api/v2/jobs/{job_id}/results?limit={limit}&offset={max(0, offset - limit)}"
            if offset > 0
            else None,
        },
    }


# Normalized results model and handler
class JobResultsPreview(BaseModel):
    columns: List[str] = Field(default_factory=list)
    rows: List[List[Any]] = Field(default_factory=list)
    total_rows: int = 0
    limit: int = 0
    truncated: bool = False


class JobResultsResponse(BaseModel):
    id: str
    status: str
    matches_found: int = 0
    rows_processed: int = 0
    elapsed_s: Optional[float] = None
    results_format: Optional[str] = None
    results_path: Optional[str] = None
    download_url: Optional[str] = None
    csv_url: Optional[str] = None
    streaming_available: bool = False
    candidates_total: Optional[int] = None
    preview: Optional[JobResultsPreview] = None
    metadata: Dict[str, Any] = Field(default_factory=dict)


class JustCallJobResultsResponse(BaseModel):
    job_id: str
    cursor: int
    next_cursor: Optional[int] = None
    total: int
    results: List[Dict[str, Any]]


class HeyReachJobResultsResponse(BaseModel):
    job_id: str
    cursor: int
    next_cursor: Optional[int] = None
    total: int
    results: List[Dict[str, Any]]


def _infer_results_format(path: Optional[str]) -> Optional[str]:
    if not path:
        return None
    suffix = Path(path).suffix.lower()
    if not suffix:
        return None
    return suffix.lstrip(".") or None


def _collect_columns_from_records(records: List[Dict[str, Any]]) -> List[str]:
    seen: List[str] = []
    for record in records:
        if not isinstance(record, dict):
            continue
        for key in record.keys():
            key_str = str(key)
            if key_str not in seen:
                seen.append(key_str)
    return seen


def _normalize_preview_payload(payload: Any, limit: int) -> Optional[JobResultsPreview]:
    if not payload:
        return None
    limit = max(1, limit)

    columns: List[str] = []
    raw_rows: Any = None

    if isinstance(payload, dict):
        nested_preview = payload.get("preview")
        if nested_preview:
            nested = _normalize_preview_payload(nested_preview, limit)
            if nested:
                return nested
        columns = [str(c) for c in payload.get("columns") or []]
        raw_rows = payload.get("rows") or payload.get("results") or []
    elif isinstance(payload, list):
        raw_rows = payload
    else:
        return None

    if not isinstance(raw_rows, list):
        raw_rows = []

    total_rows = len(raw_rows)

    if not columns:
        if raw_rows and isinstance(raw_rows[0], dict):
            columns = _collect_columns_from_records(raw_rows)
        elif raw_rows and isinstance(raw_rows[0], (list, tuple)):
            max_len = max(
                len(r) if isinstance(r, (list, tuple)) else 0 for r in raw_rows[:limit]
            )
            columns = [f"col_{idx + 1}" for idx in range(max_len)]
        elif raw_rows:
            columns = ["value"]

    matrix: List[List[Any]] = []
    for idx, row in enumerate(raw_rows):
        if idx >= limit:
            break
        if isinstance(row, dict):
            matrix.append([row.get(col) for col in columns])
        elif isinstance(row, (list, tuple)):
            fixed = list(row)
            if len(columns) and len(fixed) < len(columns):
                fixed.extend([None] * (len(columns) - len(fixed)))
            elif len(columns) and len(fixed) > len(columns):
                fixed = fixed[: len(columns)]
            matrix.append(fixed)
        else:
            matrix.append([row])

    truncated = total_rows > len(matrix)
    return JobResultsPreview(
        columns=columns,
        rows=matrix,
        total_rows=total_rows,
        limit=limit,
        truncated=truncated,
    )


def _is_justcall_job(rec: Job) -> bool:
    config = rec.config if isinstance(rec.config, dict) else {}
    return str(config.get("job_type") or "").lower() == "justcall_push"


def _is_heyreach_job(rec: Job) -> bool:
    config = rec.config if isinstance(rec.config, dict) else {}
    return str(config.get("job_type") or "").lower() == "heyreach_push"


async def _get_justcall_results(
    job_id: str,
    cursor: int,
    limit: int,
    redis_conn,
) -> JustCallJobResultsResponse:
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    results_key = JUSTCALL_RESULTS_KEY.format(job_id=job_id)
    total = await redis_conn.llen(results_key)
    if total == 0:
        return JustCallJobResultsResponse(
            job_id=job_id,
            cursor=cursor,
            total=0,
            results=[],
        )

    limit = max(1, min(int(limit), 2000))
    end = min(total, cursor + limit) - 1
    raw_results = await redis_conn.lrange(results_key, cursor, end)

    results: List[Dict[str, Any]] = []
    for item in raw_results:
        text = item.decode() if isinstance(item, (bytes, bytearray)) else item
        try:
            results.append(json.loads(text))
        except Exception:
            continue

    next_cursor = cursor + len(results)
    if next_cursor >= total:
        next_cursor = None

    return JustCallJobResultsResponse(
        job_id=job_id,
        cursor=cursor,
        next_cursor=next_cursor,
        total=total,
        results=results,
    )


async def _get_heyreach_results(
    job_id: str,
    cursor: int,
    limit: int,
    redis_conn,
) -> HeyReachJobResultsResponse:
    if not redis_conn:
        raise HTTPException(503, "Cache service unavailable")

    results_key = HEYREACH_RESULTS_KEY.format(job_id=job_id)
    total = await redis_conn.llen(results_key)
    if total == 0:
        return HeyReachJobResultsResponse(
            job_id=job_id,
            cursor=cursor,
            total=0,
            results=[],
        )

    limit = max(1, min(int(limit), 2000))
    end = min(total, cursor + limit) - 1
    raw_results = await redis_conn.lrange(results_key, cursor, end)

    results: List[Dict[str, Any]] = []
    for item in raw_results:
        text = item.decode() if isinstance(item, (bytes, bytearray)) else item
        try:
            results.append(json.loads(text))
        except Exception:
            continue

    next_cursor = cursor + len(results)
    if next_cursor >= total:
        next_cursor = None

    return HeyReachJobResultsResponse(
        job_id=job_id,
        cursor=cursor,
        next_cursor=next_cursor,
        total=total,
        results=results,
    )


async def _preview_from_storage(job: Job, limit: int) -> Optional[JobResultsPreview]:
    path = getattr(job, "results_path", None)
    path = _normalize_results_path(path) if path else None
    if not path:
        return None

    storage = get_storage_backend()
    try:
        content = await storage.read(path)
    except FileNotFoundError:
        return None
    except Exception as exc:
        log.warning("results preview read failed job=%s: %s", job.id, exc)
        return None

    limit = max(1, limit)
    ext = Path(path).suffix.lower()
    try:
        if ext == ".parquet":
            table = pq.read_table(io.BytesIO(content))
            total_rows = table.num_rows
            table = table.slice(0, min(limit, total_rows))
            df = table.to_pandas()
        elif ext == ".csv":
            df = pd.read_csv(io.BytesIO(content), nrows=limit)
            total_rows = job.matches_found or len(df)
        else:
            df = pd.read_parquet(io.BytesIO(content))
            total_rows = df.shape[0]
            df = df.head(limit)
    except Exception as exc:
        log.warning("results preview decode failed job=%s: %s", job.id, exc)
        return None

    df = df.where(pd.notnull(df), None)
    columns = [str(c) for c in df.columns.tolist()]
    rows = df.values.tolist()
    truncated = bool(total_rows and total_rows > len(rows))
    return JobResultsPreview(
        columns=columns,
        rows=rows,
        total_rows=int(total_rows or len(rows)),
        limit=limit,
        truncated=truncated,
    )


async def _load_results_preview(job: Job, limit: int) -> JobResultsPreview:
    limit = max(1, min(limit, 500))

    direct_preview = _normalize_preview_payload(job.results, limit)
    if direct_preview:
        return direct_preview

    try:
        redis_conn = await get_redis()
    except Exception:
        redis_conn = None

    if redis_conn:
        try:
            raw = await redis_conn.get(f"job:{job.id}:results")
        except Exception:
            raw = None
        if raw:
            try:
                if isinstance(raw, bytes):
                    raw = raw.decode("utf-8")
                data = json.loads(raw)
                redis_preview = _normalize_preview_payload(data, limit)
                if redis_preview:
                    return redis_preview
            except Exception:
                pass

    storage_preview = await _preview_from_storage(job, limit)
    if storage_preview:
        return storage_preview

    return JobResultsPreview(
        columns=[], rows=[], total_rows=0, limit=limit, truncated=False
    )


async def _gather_job_metadata(job: Job) -> Dict[str, Any]:
    metadata: Dict[str, Any] = {}
    try:
        redis_conn = await get_redis()
    except Exception:
        redis_conn = None
    if redis_conn:
        try:
            raw = await redis_conn.get(f"job:{job.id}")
            if raw:
                if isinstance(raw, bytes):
                    raw = raw.decode("utf-8")
                metadata = json.loads(raw)
        except Exception:
            metadata = {}
        summary_key = f"job:summary:{job.id}"
        try:
            summary_raw = await redis_conn.get(summary_key)
        except Exception:
            summary_raw = None
        if summary_raw:
            try:
                if isinstance(summary_raw, bytes):
                    summary_raw = summary_raw.decode("utf-8")
                summary_payload = json.loads(summary_raw)
            except Exception:
                summary_payload = None
            if isinstance(summary_payload, dict):
                metadata.setdefault("summary", summary_payload)
                metrics_block = metadata.get("metrics")
                if isinstance(metrics_block, dict):
                    metrics_block.setdefault("summary", summary_payload)
                else:
                    metadata["metrics"] = {"summary": summary_payload}
    return metadata


@router.get("/{job_id}/results.csv")
async def stream_job_results_csv(
    job_id: str,
    limit: int = 50000,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
):
    """Stream job output as CSV (limited to protect Sheets)."""
    rec = await db.get(Job, job_id)
    if not rec or (rec.account_id and str(rec.account_id) != str(account_id)):
        raise HTTPException(404, "job not found")

    metadata = await _gather_job_metadata(rec)
    candidates_total_header = None
    if isinstance(metadata, dict):
        candidates_total_header = metadata.get("candidates_total")
        if isinstance(candidates_total_header, str):
            try:
                candidates_total_header = int(candidates_total_header)
            except Exception:
                candidates_total_header = None

    db_results_path = getattr(rec, "results_path", None)
    redis_result_object = metadata.get("result_object_id")
    log.info(
        "[RESULTS CSV] job=%s raw results_path db=%s meta=%s",
        job_id,
        db_results_path,
        redis_result_object,
    )
    results_path_raw = db_results_path or redis_result_object
    results_path = _normalize_results_path(results_path_raw)
    if (
        results_path
        and results_path.startswith("/")
        and not results_path.startswith(("s3://", "gs://"))
    ):
        results_path = results_path.lstrip("/")
    log.info(
        "[RESULTS CSV] job=%s normalized results_path=%s",
        job_id,
        results_path,
    )
    if not results_path:
        raise HTTPException(404, "results not available")

    try:
        limit_val = int(limit)
    except Exception:
        limit_val = 50000
    if limit_val < 0:
        limit_val = 0
    limit_val = min(limit_val, 500000)

    storage = get_storage_backend()
    try:
        content = await storage.read(results_path)
    except FileNotFoundError:
        raise HTTPException(404, "results not available")
    except Exception as exc:
        log.warning("results.csv read failed job=%s: %s", job_id, exc)
        raise HTTPException(500, "failed to read results artifact")

    ext = Path(results_path).suffix.lower()
    truncated_flag = False
    total_rows = rec.matches_found or 0
    try:
        if ext == ".parquet":
            table = pq.read_table(io.BytesIO(content))
            total_rows = table.num_rows
            if limit_val:
                table = table.slice(0, min(limit_val, total_rows))
            df = table.to_pandas()
            truncated_flag = bool(limit_val and total_rows > len(df))
        else:
            read_limit = limit_val or None
            df = pd.read_csv(io.BytesIO(content), nrows=read_limit)
            if rec.matches_found:
                total_rows = rec.matches_found
                truncated_flag = bool(limit_val and rec.matches_found > len(df))
            else:
                total_rows = len(df)
                truncated_flag = bool(
                    limit_val and read_limit is not None and len(df) == limit_val
                )
    except Exception as exc:
        log.warning("results.csv conversion failed job=%s: %s", job_id, exc)
        raise HTTPException(500, "failed to convert results to CSV")

    df = df.where(pd.notnull(df), None)
    columns = [str(c) for c in df.columns.tolist()]
    buf = io.StringIO()
    df.to_csv(buf, index=False)
    csv_payload = buf.getvalue()

    headers = {
        "Content-Disposition": f'attachment; filename="job-{job_id}.csv"',
        "Cache-Control": "no-store",
        "X-Total-Rows": str(total_rows),
        "X-Row-Count": str(len(df)),
        "X-Preview-Limit": str(limit_val),
        "X-Columns": ",".join(columns),
        "X-Results-Format": "csv",
    }
    if truncated_flag:
        headers["X-Results-Truncated"] = "true"
    if candidates_total_header is not None:
        headers["X-Candidates-Total"] = str(candidates_total_header)
    if rec.matches_found is not None:
        headers["X-Matches-Found"] = str(rec.matches_found)

    return StreamingResponse(
        iter([csv_payload]), media_type="text/csv", headers=headers
    )


@router.get(
    "/{job_id}/results",
    response_model=Union[JobResultsResponse, JustCallJobResultsResponse, HeyReachJobResultsResponse],
)
async def get_job_results(
    job_id: str,
    limit: int = 25,
    include_preview: bool = True,
    cursor: int = Query(0, ge=0),
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
    response: Response = None,
):
    rec = await db.get(Job, job_id)
    if not rec or (rec.account_id and str(rec.account_id) != str(account_id)):
        raise HTTPException(404, "job not found")

    if response is not None:
        response.headers["Cache-Control"] = "no-store"

    if _is_justcall_job(rec):
        redis_conn = await get_redis()
        return await _get_justcall_results(
            job_id=job_id,
            cursor=cursor,
            limit=limit,
            redis_conn=redis_conn,
        )
    if _is_heyreach_job(rec):
        redis_conn = await get_redis()
        return await _get_heyreach_results(
            job_id=job_id,
            cursor=cursor,
            limit=limit,
            redis_conn=redis_conn,
        )

    metadata = await _gather_job_metadata(rec)
    if not isinstance(metadata, dict):
        metadata = {}

    status_val = metadata.get("status") or metadata.get("state") or "unknown"
    if hasattr(rec.status, "value"):
        status_val = rec.status.value
    elif rec.status:
        status_val = str(rec.status)

    matches_found = (
        rec.matches_found
        or metadata.get("matches_found")
        or metadata.get("total_matches")
        or 0
    )
    try:
        matches_found = int(matches_found)
    except Exception:
        matches_found = 0

    rows_processed = (
        metadata.get("processed_rows") or metadata.get("rows_processed") or 0
    )
    if not rows_processed:
        try:
            rows_processed = int(
                rec.config.get("source_rows") or rec.config.get("rows_processed") or 0
            )
        except Exception:
            rows_processed = 0
    try:
        rows_processed = int(rows_processed)
    except Exception:
        rows_processed = 0

    elapsed_s = None
    if rec.created_at and rec.updated_at:
        try:
            elapsed_s = max(0.0, (rec.updated_at - rec.created_at).total_seconds())
        except Exception:
            elapsed_s = None

    raw_results_path = getattr(rec, "results_path", None)
    results_path = _normalize_results_path(raw_results_path)
    if results_path and metadata.get("results_path") != results_path:
        metadata["results_path"] = results_path

    results_format = _infer_results_format(results_path)

    download_url = None
    if results_path:
        try:
            storage = get_storage_backend()
            download_url = await storage.generate_presigned_url(
                results_path, expires_in=3600
            )
        except Exception as exc:
            log.debug("generate_presigned_url failed for job %s: %s", job_id, exc)
            download_url = None

    streaming_available = bool(results_path)
    csv_url = f"/api/v2/jobs/{job_id}/results.csv" if streaming_available else None

    preview_obj = None
    if include_preview:
        try:
            preview_obj = await _load_results_preview(rec, limit)
        except Exception:
            preview_obj = JobResultsPreview(
                columns=[],
                rows=[],
                total_rows=0,
                limit=max(1, min(limit, 500)),
                truncated=False,
            )

    if preview_obj and matches_found == 0 and preview_obj.total_rows:
        matches_found = preview_obj.total_rows
    if preview_obj and rows_processed == 0 and preview_obj.total_rows:
        rows_processed = preview_obj.total_rows

    if status_val == "completed" and results_path and matches_found == 0:
        log.warning(
            "Job %s completed but matches_found=0 with results_path=%s",
            job_id,
            results_path,
        )

    candidates_total = metadata.get("candidates_total")
    if isinstance(candidates_total, str):
        try:
            candidates_total = int(candidates_total)
        except Exception:
            candidates_total = None
    if not candidates_total:
        best_stats = (
            metadata.get("best_match_stats")
            if isinstance(metadata.get("best_match_stats"), dict)
            else None
        )
        if best_stats:
            candidates_total = best_stats.get("candidates_total")
    if isinstance(candidates_total, float) and candidates_total.is_integer():
        candidates_total = int(candidates_total)
    if candidates_total is not None:
        metadata["candidates_total"] = candidates_total

    return JobResultsResponse(
        id=rec.id,
        status=status_val,
        matches_found=matches_found,
        rows_processed=rows_processed,
        elapsed_s=elapsed_s,
        results_format=results_format,
        results_path=results_path,
        download_url=download_url,
        csv_url=csv_url,
        streaming_available=streaming_available,
        candidates_total=candidates_total,
        preview=preview_obj if include_preview else None,
        metadata=metadata,
    )


@router.post("/{job_id}/configuration/suggest")
async def suggest_configuration(
    job_id: str,
    request_body: Dict[str, Any] = Body({"strategy": "auto"}),
    db: AsyncSession = Depends(get_session),  # Keep for signature but don't use
    account_id: UUID = Depends(get_current_account),
):
    """Returns configuration with column lists and auto-detected ID columns"""
    print(f"[SUGGEST] Called with job_id: {job_id}")
    log.info(f"[SUGGEST] Called with job_id: {job_id}")

    redis_conn = await get_redis()
    if not redis_conn:
        raise HTTPException(503, "Redis required")

    job_key = f"job:{job_id}"
    print(f"[SUGGEST] Looking for Redis key: {job_key}")

    meta_raw = await redis_conn.get(job_key)
    if not meta_raw:
        raise HTTPException(404, "Job not found in Redis")

    print("[SUGGEST] Found in Redis!")
    metadata = json.loads(meta_raw)

    # Import storage helpers for reading files
    from ...storage import resolve_object_path, exists_object
    import csv
    import hashlib

    # Helper function to safely read headers from CSV
    def _safe_header_from_csv(object_id: str, has_header: bool = True) -> list:
        """Read only the header row robustly using our unified resolver."""
        if not object_id or not exists_object(object_id):
            log.warning(f"[SUGGEST] Object not found: {object_id}")
            return []

        path = resolve_object_path(object_id)
        log.info(f"[SUGGEST] Reading headers from: {path}")

        # Try pandas first (fast/robust)
        try:
            df0 = pd.read_csv(
                path,
                nrows=0,
                dtype=str,
                keep_default_na=False,
                encoding="utf-8",
                encoding_errors="ignore",
            )
            cols = list(df0.columns)
            if cols and any(c.strip() for c in cols):
                return cols
        except Exception as e:
            log.warning(f"[SUGGEST] Pandas header read failed: {e}")

        # Fallback: read the first line manually
        try:
            with open(path, "rb") as f:
                head = f.read(4096)  # first chunk
            text = head.decode("utf-8", "ignore").lstrip("\ufeff")  # strip BOM
            first_line = text.splitlines()[0] if "\n" in text else text

            # Try csv Sniffer to detect delimiter
            try:
                dialect = csv.Sniffer().sniff(first_line)
                reader = csv.reader([first_line], dialect)
                row = next(reader, [])
            except Exception:
                # conservative fallback: split on comma
                row = [p.strip() for p in first_line.split(",")]

            # If file has no headers, synthesize
            if not has_header or not any(cell.strip() for cell in row):
                row = [f"Column_{i+1}" for i in range(len(row) if row else 1)]
            return row
        except Exception as e:
            log.error(f"[SUGGEST] Failed to read headers: {e}")
            return []

    # Get object_ids from metadata
    src_oid = metadata.get("source_object_id") or metadata.get("object_id")
    ref_oid = metadata.get("reference_object_id")

    # For JSON-created jobs, check the input field
    if not src_oid and "input" in metadata:
        inp = metadata["input"]
        src_oid = inp.get("object_id") or inp.get("source_object_id")
        ref_oid = inp.get("reference_object_id")

    log.info(f"[SUGGEST] Source object_id: {src_oid}")
    log.info(f"[SUGGEST] Reference object_id: {ref_oid}")

    # Read headers from the actual files
    source_cols = _safe_header_from_csv(src_oid, True) if src_oid else []
    ref_cols = _safe_header_from_csv(ref_oid, True) if ref_oid else []

    print(f"[SUGGEST] Source columns: {source_cols}")
    print(f"[SUGGEST] Reference columns: {ref_cols}")

    # Load the actual data from storage files (not Redis)
    source_df = None
    ref_df = None

    if src_oid and exists_object(src_oid):
        try:
            src_path = resolve_object_path(src_oid)
            source_df = pd.read_csv(
                src_path,
                dtype=str,
                keep_default_na=False,
                encoding="utf-8",
                encoding_errors="ignore",
            )
            log.info(f"[SUGGEST] Loaded source data: {len(source_df)} rows")
        except Exception as e:
            log.error(f"[SUGGEST] Failed to load source data: {e}")

    if ref_oid and exists_object(ref_oid):
        try:
            ref_path = resolve_object_path(ref_oid)
            ref_df = pd.read_csv(
                ref_path,
                dtype=str,
                keep_default_na=False,
                encoding="utf-8",
                encoding_errors="ignore",
            )
            log.info(f"[SUGGEST] Loaded reference data: {len(ref_df)} rows")
        except Exception as e:
            log.error(f"[SUGGEST] Failed to load reference data: {e}")

    # Import ID detection and profiling functions
    from fmatch.preprocessing.core.column_detector import ColumnDetector

    # Initialize defaults
    source_id_col = source_cols[0] if source_cols else "id"
    ref_id_col = ref_cols[0] if ref_cols else "id"
    id_candidates = []
    column_health = {"source": {}, "reference": {}}

    # Helper function to profile columns
    def profile_columns(df, dataset_name):
        """Generate column health metrics"""
        health_metrics = {}
        for col in df.columns:
            col_data = df[col]
            mask, cleaned = normalise_series(col_data)
            total = len(col_data)
            non_null_count = int(mask.sum())
            fill_rate = non_null_count / total if total else 0.0

            distinct_count = int(cleaned[mask].nunique(dropna=True))
            distinct_ratio = distinct_count / non_null_count if non_null_count else 0.0

            null_count = total - non_null_count
            null_ratio = null_count / total if total else 0.0

            samples = cleaned[mask].dropna().head(5).tolist() if non_null_count else []

            is_unique = (
                non_null_count > 0
                and distinct_count == non_null_count
                and null_count == 0
            )

            health_metrics[col] = {
                "fill_rate": round(float(fill_rate), 6),
                "distinct_count": distinct_count,
                "distinct_ratio": round(float(distinct_ratio), 6),
                "null_count": null_count,
                "null_ratio": round(float(null_ratio), 6),
                "samples": samples,
                "dtype": str(col_data.dtype),
                "memory_bytes": int(col_data.memory_usage(deep=True)),
                "is_unique": is_unique,
                "is_id_candidate": False,  # Will be set if detected as ID
            }
        return health_metrics

    # Process source data if available
    if source_df is not None and not source_df.empty:
        try:
            # Generate column health metrics
            column_health["source"] = profile_columns(source_df, "source")

            # Use ColumnDetector for ID detection
            source_id_detection = ColumnDetector.detect_id_column(source_df)
            if source_id_detection:
                source_id_col = source_id_detection.column_name
                id_candidates.append(
                    {
                        "dataset": "source",
                        "column": source_id_detection.column_name,
                        "confidence": source_id_detection.score,
                        "system": source_id_detection.detected_system,
                    }
                )
                # Mark as ID candidate in health metrics
                if source_id_col in column_health["source"]:
                    column_health["source"][source_id_col]["is_id_candidate"] = True
                print(
                    f"[SUGGEST] Detected source ID: {source_id_col} (confidence: {source_id_detection.score})"
                )
        except Exception as e:
            print(f"[SUGGEST] Error processing source data: {e}")

    # Process reference data if available
    if ref_df is not None and not ref_df.empty:
        try:
            # Generate column health metrics
            column_health["reference"] = profile_columns(ref_df, "reference")

            # Use ColumnDetector for ID detection
            ref_id_detection = ColumnDetector.detect_id_column(ref_df)
            if ref_id_detection:
                ref_id_col = ref_id_detection.column_name
                id_candidates.append(
                    {
                        "dataset": "reference",
                        "column": ref_id_detection.column_name,
                        "confidence": ref_id_detection.score,
                        "system": ref_id_detection.detected_system,
                    }
                )
                # Mark as ID candidate in health metrics
                if ref_id_col in column_health["reference"]:
                    column_health["reference"][ref_id_col]["is_id_candidate"] = True
                print(
                    f"[SUGGEST] Detected reference ID: {ref_id_col} (confidence: {ref_id_detection.score})"
                )
        except Exception as e:
            print(f"[SUGGEST] Error processing reference data: {e}")

    # Compute blocking strategy and metrics if we have data
    blocking_strategy = None
    blocking_metrics = None

    if source_df is not None and not source_df.empty:
        try:
            from fmatch.core.engine import (
                determine_optimal_blocking_strategy,
                BlockingMode,
            )

            # Determine mode
            mode = BlockingMode.MATCH if ref_df is not None else BlockingMode.DEDUPE

            # Sample for performance
            sample_size = min(10000, len(source_df))
            source_sample = (
                source_df.sample(n=sample_size, random_state=42)
                if len(source_df) > sample_size
                else source_df
            )
            ref_sample = None
            if ref_df is not None:
                ref_sample = (
                    ref_df.sample(n=sample_size, random_state=42)
                    if len(ref_df) > sample_size
                    else ref_df
                )

            # Check if DomainFamily is available for blocking
            prefer_domain_family = False
            if "DomainFamily" in source_sample.columns:
                if ref_sample is not None and "DomainFamily" in ref_sample.columns:
                    # Check if DomainFamily has good distribution for blocking
                    family_cardinality = source_sample["DomainFamily"].nunique()
                    if 10 < family_cardinality < len(source_sample) * 0.5:
                        prefer_domain_family = True
                        log.info(
                            f"DomainFamily suitable for blocking: {family_cardinality} unique families"
                        )

            # Determine optimal blocking strategy
            strategy_result = determine_optimal_blocking_strategy(
                source_sample,
                ref_sample,
                [],  # Empty mappings for now
                mode,
                "default",
                weight_profile="balanced",
            )

            # Override with DomainFamily if suitable
            if prefer_domain_family and strategy_result:
                strategy_result["source_block_col"] = "DomainFamily"
                strategy_result["ref_block_col"] = (
                    "DomainFamily" if ref_sample is not None else None
                )
                strategy_result["apply_blocking"] = True
                log.info("Using DomainFamily for blocking strategy")

            if strategy_result and strategy_result.get("apply_blocking"):
                blocking_strategy = {
                    "source_block_col": strategy_result.get("source_block_col"),
                    "ref_block_col": strategy_result.get("ref_block_col"),
                    "block_key_len": strategy_result.get("block_key_len", 3),
                    "preproc": strategy_result.get("preproc", "alnum"),
                    "quality_score": strategy_result.get("quality_score", 0),
                }

                # Calculate impact metrics
                n_src = len(source_df)
                n_ref = len(ref_df) if ref_df is not None else n_src
                pairs_before = (
                    n_src * n_ref
                    if mode == BlockingMode.MATCH
                    else n_src * (n_src - 1) / 2
                )

                # Estimate pairs after blocking
                reduction_rate = strategy_result.get("raw_metrics", {}).get("RR", 0.9)
                pairs_after = pairs_before * (1 - reduction_rate)

                blocking_metrics = {
                    "pairs_before": int(pairs_before),
                    "pairs_after": int(pairs_after),
                    "reduction_pct": round(reduction_rate * 100, 1),
                    "est_time_saved_ms": int(max(pairs_before - pairs_after, 0) / 5000),
                    "recommended": True,
                }

                print(f"[SUGGEST] Blocking strategy: {blocking_strategy}")
                print(f"[SUGGEST] Blocking metrics: {blocking_metrics}")
        except Exception as e:
            print(f"[SUGGEST] Error computing blocking strategy: {e}")

    # Helper functions for safer mapping detection
    def _is_domainish(colname):
        """Check if column likely contains domain/URL data"""
        c = colname.lower()
        # More specific B2B domain patterns
        return (
            any(x in c for x in ["domain", "website", "url", "site", "web"])
            or c == "domain_append"
        )

    def _is_nameish(colname):
        """Check if column likely contains name/company data"""
        c = colname.lower()
        # Exclude first_name, last_name to avoid bad mappings
        if "first" in c or "last" in c:
            return False
        return any(
            x in c
            for x in [
                "name",
                "company",
                "account",
                "org",
                "hospital",
                "provider",
                "firm",
                "corp",
            ]
        )

    def _is_emailish(colname):
        """Check if column likely contains email data"""
        c = colname.lower()
        # Include specific B2B email patterns
        return any(x in c for x in ["email", "e-mail", "mail"]) or c in [
            "email_address",
            "email address",
        ]

    def _is_phoneish(colname):
        """Check if column likely contains phone data"""
        c = colname.lower()
        return any(x in c for x in ["phone", "tel", "mobile", "cell"])

    # Create intelligent default mappings based on common column patterns
    default_mappings = []
    if metadata.get("mode") == "match" and source_cols and ref_cols:
        # Get string columns only (object dtype in pandas)
        src_strings = source_cols  # We don't have dtype info here, so include all
        ref_strings = ref_cols

        # CRITICAL FOR B2B: Find and prioritize domain mappings
        # Look for specific B2B domain columns first
        src_dom_candidates = [
            "Email Domain",
            "email_domain",
            "Domain",
            "domain",
            "Email_Domain",
            "domain_append",
            "Domain_Append",
        ]
        ref_dom_candidates = [
            "Domain_Append",
            "domain_append",
            "Domain",
            "domain",
            "Website",
            "website",
            "DHC Provider Website",
        ]

        src_dom = None
        for candidate in src_dom_candidates:
            if candidate in src_strings:
                src_dom = candidate
                break
        # Fallback to generic domain detection
        if not src_dom:
            src_dom = next((c for c in src_strings if _is_domainish(c)), None)

        ref_dom = None
        for candidate in ref_dom_candidates:
            if candidate in ref_strings:
                ref_dom = candidate
                break
        # Fallback to generic domain detection
        if not ref_dom:
            ref_dom = next((c for c in ref_strings if _is_domainish(c)), None)

        # PHASE 1 FIX: If no domain column but email exists, we'll extract domain
        src_email_col = None
        if not src_dom:
            # Find email column to extract domain from
            email_candidates = [
                "Email Address",
                "email_address",
                "Email",
                "email",
                "Email_Address",
            ]
            for candidate in email_candidates:
                if candidate in src_strings:
                    src_email_col = candidate
                    # Create synthetic domain column name
                    src_dom = "Email Domain"  # This will be extracted in worker
                    log.info(
                        f"[SUGGEST] Will extract domain from {src_email_col} column"
                    )
                    break

        # Always add domain mapping as highest priority for B2B
        if src_dom and ref_dom:
            default_mappings.append(
                {
                    "source": src_dom,
                    "reference": ref_dom,  # Consistent key name
                    "ref": ref_dom,  # Keep for EngineClient compatibility
                    "similarity_measure": "Domain",  # Use Domain algo for exact matching
                    "preferred_algo": "Domain",  # EngineClient expects this
                    "weight": 0.8,  # High weight for B2B domain identity
                    "apply_blocking": True,  # Use for blocking
                    "extract_from": src_email_col
                    if src_email_col
                    else None,  # Signal to extract
                }
            )
            log.info(
                f"[SUGGEST] Added primary domain mapping: {src_dom} <-> {ref_dom} (weight=0.8)"
            )

        # Add DomainFamily mapping if present (let intelligence determine optimal weight)
        if "DomainFamily" in src_strings and "DomainFamily" in ref_strings:
            default_mappings.append(
                {
                    "source": "DomainFamily",
                    "ref": "DomainFamily",  # Use "ref" for EngineClient compatibility
                    "similarity_measure": "exact",
                    "preferred_algo": "exact",
                    # No hardcoded weight - let the matching engine optimize it
                }
            )
            log.info(
                "[SUGGEST] Added DomainFamily mapping for improved domain matching"
            )

        # Add company name mapping as secondary signal for B2B
        # Look for specific B2B company columns first
        src_company_candidates = [
            "Company Name",
            "company_name",
            "Company",
            "company",
            "Account Name",
            "account_name",
            "Organization",
            "Hospital System",
        ]
        ref_company_candidates = [
            "Account Name",
            "account_name",
            "Company Name",
            "company_name",
            "Name",
            "name",
            "Account",
            "Hospital System",
        ]

        src_company = None
        for candidate in src_company_candidates:
            if candidate in src_strings:
                src_company = candidate
                break
        # Fallback to generic name detection
        if not src_company:
            src_company = next((c for c in src_strings if _is_nameish(c)), None)

        ref_company = None
        for candidate in ref_company_candidates:
            if candidate in ref_strings:
                ref_company = candidate
                break
        # Fallback to generic name detection
        if not ref_company:
            ref_company = next((c for c in ref_strings if _is_nameish(c)), None)

        if (
            src_company
            and ref_company
            and not any(m["source"] == src_company for m in default_mappings)
        ):
            default_mappings.append(
                {
                    "source": src_company,
                    "reference": ref_company,  # Consistent key name
                    "ref": ref_company,  # Keep for EngineClient compatibility
                    "similarity_measure": "TokenSetRatio",  # Good for company variations
                    "preferred_algo": "TokenSetRatio",
                    "weight": 0.3,  # Secondary signal after domain
                }
            )
            log.info(
                f"[SUGGEST] Added company mapping: {src_company} <-> {ref_company} (weight=0.3)"
            )

        # Add email mapping if present
        src_email = next((c for c in src_strings if _is_emailish(c)), None)
        ref_email = next((c for c in ref_strings if _is_emailish(c)), None)
        if (
            src_email
            and ref_email
            and not any(m["source"] == src_email for m in default_mappings)
        ):
            default_mappings.append(
                {
                    "source": src_email,
                    "reference": ref_email,
                    "similarity_measure": "jaro_winkler",
                    "weight": 0.3,
                }
            )

        # Add phone mapping if present
        src_phone = next((c for c in src_strings if _is_phoneish(c)), None)
        ref_phone = next((c for c in ref_strings if _is_phoneish(c)), None)
        if (
            src_phone
            and ref_phone
            and not any(m["source"] == src_phone for m in default_mappings)
        ):
            default_mappings.append(
                {
                    "source": src_phone,
                    "reference": ref_phone,
                    "similarity_measure": "exact",
                    "weight": 0.2,
                }
            )

        # Log the mappings for debugging
        print(f"[SUGGEST] Generated {len(default_mappings)} safe mappings")
        for m in default_mappings:
            log.info(
                f"[SUGGEST] Mapping: {m['source']} -> {m['reference']} ({m['similarity_measure']}, weight={m['weight']})"
            )

    # Generate a correlation_id for this suggestion
    correlation_id = f"suggest:{job_id}:{uuid4().hex[:8]}"

    # Store the suggested configuration in Redis for execute to use
    suggested_config = {
        "mode": metadata.get("mode", "match"),
        "source_id_column": source_id_col,
        "reference_id_column": ref_id_col,
        "id_candidates": id_candidates,
        "maps": default_mappings,  # Use "maps" for EngineClient compatibility
        "mappings": default_mappings,  # Keep for backward compatibility
        "blocking_strategy": blocking_strategy,
        "blocking_metrics": blocking_metrics,
        "algorithm": "WRatio",
        "threshold": 75,  # Better default for B2B Account Match (0-100 scale)
        "source_columns": source_cols,
        "reference_columns": ref_cols,
        "apply_blocking": True,  # Enable blocking for B2B
        # Phase-2 B2B intelligence options
        "options": {
            "use_domain_family": True,  # Enable domain family expansion
            "use_website": True,  # Enable website anchor
            "use_geo_hint": True,  # Enable geo nudging
            "b2c_filter": True,  # Enable B2C filtering
        },
    }
    # Generate hashes for the suggested configuration
    config_to_hash = {
        "mappings": default_mappings,
        "threshold": suggested_config.get("threshold", 90),
        "source_columns": source_cols,
        "reference_columns": ref_cols,
        "blocking_strategy": blocking_strategy,
    }

    cfg_bytes = json.dumps(config_to_hash, sort_keys=True, ensure_ascii=False).encode(
        "utf-8", "ignore"
    )
    legacy_hash = hashlib.md5(cfg_bytes).hexdigest()[:16]

    intelligence_hash = _stable_intelligence_hash(suggested_config)
    hash_aliases = {"short": legacy_hash, "full": intelligence_hash}

    log.info(
        f"[SUGGEST] Generated hashes legacy={legacy_hash} intelligence={intelligence_hash}"
    )

    # Update the suggested config with hash values (legacy + stable)
    suggested_config["hash"] = legacy_hash
    suggested_config["config_hash"] = (
        legacy_hash  # Legacy alias used by existing clients
    )
    suggested_config["hash_prefix"] = legacy_hash
    suggested_config["intelligence_hash"] = intelligence_hash
    suggested_config["hash_aliases"] = hash_aliases

    payload = json.dumps(suggested_config, separators=(",", ":"), ensure_ascii=False)

    # Store per-job suggestion snapshot
    await redis_conn.set(f"job:{job_id}:suggested_config", payload, ex=3600)

    # Cache by both stable and legacy hashes
    cache_ttl = 86400
    await redis_conn.set(f"intelligence:{intelligence_hash}", payload, ex=cache_ttl)
    await redis_conn.set(f"intelligence:{legacy_hash}", payload, ex=cache_ttl)
    await redis_conn.set(
        f"intel:index:short:{legacy_hash}", intelligence_hash, ex=cache_ttl
    )

    log.info(
        f"[SUGGEST] Cached configuration keys: "
        f"full=intelligence:{intelligence_hash} short=intelligence:{legacy_hash}"
    )

    # Also update the main job metadata with the config and hash
    metadata.setdefault("config", {})
    metadata["config"].update(suggested_config)
    metadata["config"]["hash"] = legacy_hash
    metadata["config"]["config_hash"] = legacy_hash
    metadata["config"]["hash_prefix"] = legacy_hash
    metadata["config"]["intelligence_hash"] = intelligence_hash
    metadata["config"]["hash_aliases"] = hash_aliases
    await redis_conn.setex(f"job:{job_id}", 86400, json.dumps(metadata))

    log.info(f"Stored suggested configuration with hash for job {job_id}")

    return {
        "job_id": job_id,
        "correlation_id": correlation_id,  # Add correlation_id at top level
        "status": "ready",
        "hash": legacy_hash,  # Legacy 16-char hash (unchanged contract)
        "config_hash": legacy_hash,  # Alias for compatibility
        "intelligence_hash": intelligence_hash,
        "hash_aliases": hash_aliases,
        "hash_prefix": legacy_hash,
        "columns": {"source": source_cols, "reference": ref_cols},
        "column_health": column_health,  # Add comprehensive column health metrics
        "configuration": {
            "mode": metadata.get("mode", "match"),
            "source_id_column": source_id_col,
            "reference_id_column": ref_id_col,
            "id_candidates": id_candidates,  # Add detected ID candidates
            "mappings": default_mappings,  # Use intelligent default mappings
            "blocking_strategy": blocking_strategy,
            "blocking_metrics": blocking_metrics,  # Add blocking impact metrics
            "algorithm": "WRatio",
            "threshold": 90,  # Increased threshold for cleaner results
            # Add columns here too for the UI
            "source_columns": source_cols,
            "reference_columns": ref_cols,
            "hash": legacy_hash,  # Include legacy hash in configuration
            "config_hash": legacy_hash,
            "intelligence_hash": intelligence_hash,
            "hash_aliases": hash_aliases,
            "hash_prefix": legacy_hash,
        },
        # Keep these at top level too for backwards compatibility
        "source_columns": source_cols,
        "reference_columns": ref_cols,
        "id_candidates": id_candidates,  # Also include at top level
        "blocking_metrics": blocking_metrics,  # Also include at top level for easy access
        "message": f"Configuration ready with {len(source_cols)} source and {len(ref_cols)} reference columns",
    }
