"""
Redis Queue (RQ) worker for FoundryMatch SaaS
---------------------------------------------

Executed by an external RQ process – or, in tests, by SimpleWorker – to run
long‑running duplicate / match jobs decoupled from the FastAPI thread.
"""

from __future__ import annotations

import asyncio
import json
import logging
import os
import sys
import time
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
from uuid import uuid4, UUID
import requests
from dataclasses import dataclass, asdict
from enum import Enum

import pandas as pd
import redis
import numpy as np
from rq import Queue

from fmatch.core.engine import DedupeConfig, MatchConfig
from .utils import apply_best_match_per_source
from .api.v2.id_hunter import run_id_hunter_job
from .storage import resolve_object_path


def _ensure_posix_path(value: Union[str, Path]) -> str:
    """Return a POSIX-style string for storage keys."""
    if isinstance(value, Path):
        value = str(value)
    return str(value).replace("\\", "/")


from urllib.parse import urlparse, unquote
from datetime import datetime

# Register SQLite UUID adapter for worker-side raw connections (dev convenience)
try:
    import sqlite3
    import uuid

    sqlite3.register_adapter(uuid.UUID, lambda u: str(u))
    sqlite3.register_converter("UUID", lambda b: uuid.UUID(b.decode("ascii")))
except Exception:
    pass
from sqlmodel import Session, create_engine
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from .models import Job, JobStatus, JobType


# ───────────────────────────  Progress Tracking  ───────────────────────────
class ProgressPhase(str, Enum):
    """Progress phases for job execution"""

    INITIALIZING = "initializing"
    LOADING_DATA = "loading_data"
    PREPROCESSING = "preprocessing"
    MATCHING = "matching"
    CLUSTERING = "clustering"
    SAVING_RESULTS = "saving_results"
    COMPLETED = "completed"
    FAILED = "failed"


@dataclass
class ProgressEvent:
    """Structured progress event matching the blueprint"""

    job_id: str
    phase: ProgressPhase
    percent: float
    message: str
    metadata: dict = None

    def to_redis_stream(self, redis_conn, stream_key: str):
        """Publish to Redis Stream for SSE consumption"""
        try:
            redis_conn.xadd(
                stream_key,
                {"event": json.dumps(asdict(self))},
                maxlen=1000,  # Keep last 1000 events
            )
        except Exception as e:
            log.debug(f"Failed to publish to Redis stream {stream_key}: {e}")


# ───────────────────────────  Logging  ────────────────────────────────────
from .logging_config import configure_logging
from .observability.context import set_job_id, set_account_id, clear_context

configure_logging()
log = logging.getLogger(__name__)

# ─────────────────────  Redis connection & queue  ─────────────────────────
# Import from centralized settings
from .settings import REDIS_URL, RQ_QUEUE_NAME

listen = [RQ_QUEUE_NAME]


# Create Redis connection with proper keepalive settings to prevent timeout
def create_redis_connection():
    """Create Redis connection with keepalive settings to prevent disconnections."""
    from urllib.parse import urlparse

    parsed = urlparse(REDIS_URL)

    return redis.Redis(
        host=parsed.hostname or "127.0.0.1",
        port=parsed.port or 6379,
        db=int(parsed.path.lstrip("/")) if parsed.path else 0,
        socket_connect_timeout=10,
        socket_timeout=10,
        socket_keepalive=True,
        socket_keepalive_options={
            1: 60,  # TCP_KEEPIDLE: start keepalive after 60s
            2: 10,  # TCP_KEEPINTVL: interval between keepalive probes
            3: 6,  # TCP_KEEPCNT: number of keepalive probes
        },
        retry_on_timeout=True,
        health_check_interval=30,
    )


conn = create_redis_connection()

log.info("[RQ_WORKER] Boot: redis=%s queues=%s", REDIS_URL, listen)

SHEET_WEBHOOK_DB_RETRY_MAX_ATTEMPTS = max(
    0, int(os.getenv("SHEET_WEBHOOK_DB_RETRY_MAX_ATTEMPTS", "6"))
)
SHEET_WEBHOOK_DB_RETRY_BASE_SECONDS = max(
    1, int(os.getenv("SHEET_WEBHOOK_DB_RETRY_BASE_SECONDS", "2"))
)
SHEET_WEBHOOK_DB_RETRY_MAX_SECONDS = max(
    SHEET_WEBHOOK_DB_RETRY_BASE_SECONDS,
    int(os.getenv("SHEET_WEBHOOK_DB_RETRY_MAX_SECONDS", "60")),
)
_sheet_webhook_loop: Optional[asyncio.AbstractEventLoop] = None

# Database connection for worker
# Import from shared settings to maintain consistency
from fmatch.shared.settings import get_settings

settings = get_settings()
DATABASE_URL = settings.DATABASE_URL

if DATABASE_URL.startswith("postgresql+asyncpg://"):
    SYNC_DATABASE_URL = DATABASE_URL.replace(
        "postgresql+asyncpg://", "postgresql://", 1
    )
elif DATABASE_URL.startswith("sqlite+aiosqlite://"):
    SYNC_DATABASE_URL = DATABASE_URL.replace("sqlite+aiosqlite://", "sqlite://", 1)
else:
    SYNC_DATABASE_URL = DATABASE_URL

SYNC_DB_ENGINE = create_engine(
    SYNC_DATABASE_URL, echo=False, future=True, pool_pre_ping=True
)


# ----------------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------------
def _path_from_file_uri(uri_or_path: str) -> str:
    """Convert file:// URI to OS path; pass-through normal paths.
    Handles Windows URIs like file:///C:/path properly.
    """
    try:
        if not isinstance(uri_or_path, str):
            return uri_or_path
        if not uri_or_path.lower().startswith("file://"):
            return uri_or_path
        parsed = urlparse(uri_or_path)
        path = unquote(parsed.path)
        if os.name == "nt":
            # Strip leading slash for drive letters like /C:/...
            if len(path) >= 4 and path[0] == "/" and path[2] == ":" and path[3] == "/":
                path = path[1:]
            path = path.replace("/", "\\")
        return path
    except Exception:
        # Fallback to simple replace
        return uri_or_path.replace("file://", "")


def emit_progress(
    redis_conn: redis.Redis,
    job_id: str,
    phase: ProgressPhase,
    percent: float,
    message: str,
    **metadata,
):
    """Emit structured progress event to Redis Stream"""
    event = ProgressEvent(
        job_id=job_id,
        phase=phase,
        percent=round(percent, 1),
        message=message,
        metadata=metadata if metadata else {},
    )

    # Publish to Redis Stream for SSE consumption
    event.to_redis_stream(redis_conn, f"progress:{job_id}")

    # Also publish to pub/sub for backwards compatibility
    try:
        channel = f"job:{job_id}:progress"
        redis_conn.publish(channel, json.dumps(asdict(event)))
    except Exception as exc:
        log.debug("Progress publish failed for %s: %s", job_id, exc)


def publish_progress(
    redis_conn: redis.Redis, job_id: str, progress: float, status: str
) -> None:
    """
    Legacy progress publisher for backwards compatibility.
    Fire‑and‑forget progress event. Swallows connection errors so tests pass
    when no Redis server is running.
    """
    # Map old format to new structured format
    phase = ProgressPhase.MATCHING  # Default phase
    if "initializing" in status.lower():
        phase = ProgressPhase.INITIALIZING
    elif "loading" in status.lower():
        phase = ProgressPhase.LOADING_DATA
    elif "preprocessing" in status.lower():
        phase = ProgressPhase.PREPROCESSING
    elif "saving" in status.lower():
        phase = ProgressPhase.SAVING_RESULTS
    elif "completed" in status.lower():
        phase = ProgressPhase.COMPLETED
    elif "failed" in status.lower():
        phase = ProgressPhase.FAILED

    emit_progress(redis_conn, job_id, phase, progress, status)


def create_progress_callback(
    redis_conn: redis.Redis,
    job_id: str,
    base_percent: float = 0,
    percent_range: float = 100,
):
    """
    Create a progress callback function for engine operations.
    This allows fine-grained progress tracking within engine operations.
    """

    def callback(current: int, total: int, message: str = ""):
        if total > 0:
            percent = base_percent + (current / total) * percent_range
            # Determine phase based on message content
            phase = ProgressPhase.MATCHING
            if "preprocess" in message.lower():
                phase = ProgressPhase.PREPROCESSING
            elif "cluster" in message.lower():
                phase = ProgressPhase.CLUSTERING

            emit_progress(
                redis_conn,
                job_id,
                phase,
                percent,
                message,
                current=current,
                total=total,
            )

    return callback


def _set_job_meta(redis_conn: redis.Redis, job_id: str, **fields) -> None:
    """Update simple JSON status blob used by v2 polling (job:{job_id})."""
    try:
        base = {
            "status": fields.get("status", "processing"),
            "progress": fields.get("progress", 0),
            "processed_rows": fields.get("processed_rows", 0),
            "message": fields.get("message", ""),
            "updated_at": datetime.utcnow().isoformat(),
        }
        if "matches_found" in fields:
            try:
                base["matches_found"] = int(fields["matches_found"])
            except Exception:
                base["matches_found"] = fields["matches_found"]
        base.update(fields)
        redis_conn.set(f"job:{job_id}", json.dumps(base), ex=86400)
    except Exception:
        # best-effort only
        pass


def _safe_unlink(path: str) -> None:
    try:
        if path and os.path.exists(path):
            os.unlink(path)
    except Exception:
        pass


async def check_cancellation(redis_conn: redis.Redis, job_id: str) -> bool:
    """Check if job has been cancelled"""
    cancel_key = f"job:cancel:{job_id}"
    return redis_conn.exists(cancel_key)


def check_cancellation_sync(redis_conn: redis.Redis, job_id: str) -> bool:
    """Synchronous version of cancellation check for use in sync functions"""
    cancel_key = f"job:cancel:{job_id}"
    return redis_conn.exists(cancel_key)


def clear_cancellation_flag(redis_conn: redis.Redis, job_id: str) -> None:
    """Clear the cancellation flag for a job"""
    cancel_key = f"job:cancel:{job_id}"
    redis_conn.delete(cancel_key)


def notify_webhook(
    webhook_url: str, job_id: str, status: str, data: Dict[str, Any]
) -> None:
    """Send webhook notification on job completion."""
    # Import the shared validation and settings
    from .webhook_safe import validate_webhook_url
    from . import settings

    # Re-validate webhook URL before sending
    if not validate_webhook_url(
        webhook_url, require_https=settings.REQUIRE_HTTPS_WEBHOOKS
    ):
        log.error(f"Webhook URL failed re-validation: {webhook_url}")
        return

    try:
        payload = {
            "job_id": job_id,
            "status": status,
            "timestamp": time.time(),
            **data,  # Include all additional data
        }

        response = requests.post(
            webhook_url,
            json=payload,
            headers={"Content-Type": "application/json"},
            timeout=10,  # Shorter timeout
            allow_redirects=False,  # Prevent redirect attacks
        )

        if response.status_code >= 400:
            log.warning(
                "Webhook notification failed for job %s: %s", job_id, response.text
            )
        else:
            log.info("Webhook notification sent for job %s", job_id)

    except Exception as exc:
        log.error("Failed to send webhook for job %s: %s", job_id, exc)


def update_job_status_sync(job_id: str, status: str, **kwargs):
    """Synchronously persist job status updates via shared SQLModel engine."""
    try:
        with Session(SYNC_DB_ENGINE) as session:
            job = session.get(Job, _job_pk(job_id))
            if not job:
                log.warning(
                    "Job %s not found while updating status to %s", job_id, status
                )
                return

            job.status = JobStatus(status)
            job.updated_at = datetime.utcnow()

            for key, value in kwargs.items():
                if hasattr(job, key):
                    setattr(job, key, value)

            session.add(job)
            session.commit()
            log.info("Updated job %s status to %s", job_id, status)
    except SQLAlchemyError as exc:  # pragma: no cover - surfaced in logs
        log.exception("Failed to update job %s status to %s: %s", job_id, status, exc)
        raise


def _run_justcall_job_sync(
    job_id: str,
    job_config: Dict[str, Any],
    account_id: Optional[str],
    redis_conn,
) -> None:
    """Run JustCall push jobs via the async worker inside the RQ worker."""
    from .workers.justcall_worker import JustCallWorker

    justcall_config = job_config.get("config") or {}
    rows = job_config.get("rows") or []
    upload_id = job_config.get("upload_id") or justcall_config.get("upload_id")
    if upload_id:
        justcall_config["upload_id"] = upload_id

    try:
        asyncio.run(
            JustCallWorker(
                account_id=str(account_id),
                job_id=job_id,
                config=justcall_config,
                rows=rows,
                redis=redis_conn,
            ).run()
        )
        update_job_status_sync(job_id, "completed")
    except Exception as exc:
        update_job_status_sync(job_id, "failed", error=str(exc))
        raise


def _run_heyreach_job_sync(
    job_id: str,
    job_config: Dict[str, Any],
    account_id: Optional[str],
    redis_conn,
) -> None:
    """Run HeyReach push jobs via the async worker inside the RQ worker."""
    from .workers.heyreach_worker import HeyReachWorker

    heyreach_config = job_config.get("config") or {}
    rows = job_config.get("rows") or []
    upload_id = job_config.get("upload_id") or heyreach_config.get("upload_id")
    if upload_id:
        heyreach_config["upload_id"] = upload_id

    try:
        asyncio.run(
            HeyReachWorker(
                account_id=str(account_id),
                job_id=job_id,
                config=heyreach_config,
                rows=rows,
                redis=redis_conn,
            ).run()
        )
        update_job_status_sync(job_id, "completed")
    except Exception as exc:
        update_job_status_sync(job_id, "failed", error=str(exc))
        raise

# ----------------------------------------------------------------------------
# Real engine integration
# ----------------------------------------------------------------------------
def get_safe_output_dir(cfg: Union[MatchConfig, DedupeConfig]) -> Path:
    """Get output directory with fallback."""
    default_dir = Path(os.getenv("TMPDIR", "/tmp")) / "fmatch" / "jobs" / cfg.job_id
    out_dir = Path(getattr(cfg, "output_dir", str(default_dir)))
    out_dir.mkdir(parents=True, exist_ok=True)
    return out_dir


def read_csv_safely(
    filepath: str, max_rows: Optional[int] = None, chunk_size: int = 50000
) -> pd.DataFrame:
    """
    Read CSV with memory safety.
    For large files, returns a sample or processes in chunks.
    """
    file_path = Path(filepath)
    if not file_path.exists():
        raise FileNotFoundError(f"File not found: {filepath}")

    # Check file size
    file_size_mb = file_path.stat().st_size / (1024 * 1024)
    log.info(f"Reading CSV file: {filepath} ({file_size_mb:.1f} MB)")

    # For small files, read normally
    if file_size_mb < 100:  # Less than 100MB
        return pd.read_csv(filepath)

    # For medium files, use max_rows if specified
    if max_rows and file_size_mb < 500:  # 100-500MB
        log.warning(f"Large file detected. Reading first {max_rows} rows only.")
        return pd.read_csv(filepath, nrows=max_rows)

    # For very large files, read in chunks and sample
    log.warning(f"Very large file ({file_size_mb:.1f} MB). Reading sample...")

    # First, get total rows efficiently
    total_rows = sum(1 for _ in open(filepath)) - 1  # Subtract header

    # Determine sample size
    sample_size = min(max_rows or 1_000_000, total_rows)

    # Read random sample
    skip_rows = sorted(
        np.random.choice(
            range(1, total_rows + 1), size=total_rows - sample_size, replace=False
        )
    )

    df = pd.read_csv(filepath, skiprows=skip_rows)
    log.info(f"Sampled {len(df)} rows from {total_rows} total rows")

    return df


def run_match_worker(
    cfg: MatchConfig, src_df: pd.DataFrame, ref_df: pd.DataFrame
) -> pd.DataFrame:
    """Run real matching using the core engine."""
    from fmatch.core.engine import process_dataframe

    # Ensure output directory exists
    out_dir = get_safe_output_dir(cfg)

    # Run the matching engine
    log.info(
        f"Running match engine with algorithm={cfg.algorithm}, threshold={cfg.threshold}"
    )
    results_df = process_dataframe(
        input_df=src_df,
        ref_df=ref_df,
        config=cfg,
        src_id_col=cfg.src_id_col,
        ref_id_col=cfg.ref_id_col,
    )

    # Save results
    if results_df is not None and not results_df.empty:
        results_path = out_dir / f"match-{cfg.job_id}.parquet"
        results_df.to_parquet(results_path, index=False)
        log.info(f"Saved {len(results_df)} matches to {results_path}")
    else:
        log.warning("No matches found")
        results_df = pd.DataFrame()

    return results_df


def run_dedupe_worker(cfg: DedupeConfig, df: pd.DataFrame) -> pd.DataFrame:
    """Run real deduplication using the core engine."""
    from fmatch.core.engine import process_dataframe

    # Ensure output directory exists
    out_dir = get_safe_output_dir(cfg)

    # Run the deduplication engine
    log.info(
        f"Running dedupe engine with algorithm={cfg.algorithm}, threshold={cfg.threshold}"
    )
    results_df = process_dataframe(
        input_df=df, ref_df=None, config=cfg, rec_id_col=cfg.rec_id_col
    )

    # Save results
    if results_df is not None and not results_df.empty:
        results_path = out_dir / f"dupes-{cfg.job_id}.parquet"
        results_df.to_parquet(results_path, index=False)
        log.info(f"Saved {len(results_df)} duplicates to {results_path}")
    else:
        log.warning("No duplicates found")
        results_df = pd.DataFrame()

    return results_df


# ----------------------------------------------------------------------------
# RQ entry‑point
# ----------------------------------------------------------------------------
def process_job(job_id: str, job_data: Dict[str, Any] = None, **kwargs) -> None:
    """
    New unified entry point for processing jobs from the v2 API.
    V2 jobs are stored in Redis, not the database.
    """
    set_job_id(job_id)
    try:
        # Import the v2 worker function
        from .worker_v2 import process_job_v2

        # Use the v2 worker for Redis-based jobs
        return process_job_v2(job_id, job_data, **kwargs)
    finally:
        clear_context()


def _hunter_results_to_dicts(results: List[Any]) -> List[Dict[str, Any]]:
    if not results:
        return []
    formatted: List[Dict[str, Any]] = []
    for row in results:
        if hasattr(row, "model_dump"):
            formatted.append(row.model_dump())
        elif isinstance(row, dict):
            formatted.append(row)
        else:
            try:
                formatted.append(dict(row))
            except Exception:
                formatted.append({"value": row})
    return formatted


def _persist_hunter_results(job_id: str, results: List[Dict[str, Any]]) -> str:
    object_id = f"uploads/results/{job_id}.parquet"
    path = resolve_object_path(object_id)
    Path(path).parent.mkdir(parents=True, exist_ok=True)

    df = pd.DataFrame(results)
    if df.empty:
        df = pd.DataFrame(
            columns=[
                "row_index",
                "salesforce_id",
                "confidence",
                "match_score",
                "match_method",
                "match_explanation",
                "display_status",
                "name_score",
                "matched_name",
                "matched_website",
                "record_url",
                "custom_fields",
                "alternative_match_count",
                "warnings",
                "status",
            ]
        )
    df.to_parquet(path, index=False)
    return object_id


def _run_id_hunter_job_sync(
    job_id: str,
    job_config: Dict[str, Any],
    account_id: Optional[str],
    redis_conn: redis.Redis,
) -> None:
    started_at = time.time()
    _set_job_meta(
        redis_conn,
        job_id,
        status="processing",
        progress=5,
        message="Starting Account Hunter job",
    )
    try:
        payload = job_config.get("request") if isinstance(job_config, dict) else None
        if not payload and isinstance(job_config, dict):
            payload = {k: v for k, v in job_config.items()}
        account = (
            account_id
            or (job_config.get("account_id") if isinstance(job_config, dict) else None)
            or (payload.get("account_id") if isinstance(payload, dict) else None)
        )
        if not account:
            raise ValueError("account_id missing for Account Hunter job")

        results, summary, warnings = asyncio.run(
            run_id_hunter_job(job_id, {**(job_config or {}), "account_id": account})
        )
        results_list = _hunter_results_to_dicts(results)
        result_object_id = _persist_hunter_results(job_id, results_list)

        matches_found = summary.get("matched", 0) if isinstance(summary, dict) else len(
            results_list
        )
        completion_message = "Account Hunter completed"
        elapsed_s = max(0.0, time.time() - started_at)

        _set_job_meta(
            redis_conn,
            job_id,
            progress=100,
            status="completed",
            message=completion_message,
            matches_found=matches_found,
            processed_rows=len(results_list),
            rows_processed=len(results_list),
            result_rows=len(results_list),
            result_object_id=result_object_id,
            results_path=result_object_id,
            results=results_list,
            summary=summary,
            warnings=warnings,
            elapsed_s=elapsed_s,
        )
        try:
            update_job_status_sync(
                job_id,
                "completed",
                results_path=result_object_id,
                matches_found=matches_found,
                error=None,
                results={"summary": summary} if summary else None,
            )
        except Exception as db_exc:
            log.warning("Unable to persist completed Hunter job %s: %s", job_id, db_exc)
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.COMPLETED,
            100,
            completion_message,
            matches_found=matches_found,
            duration_s=elapsed_s,
        )
    except Exception as exc:
        err_msg = f"Account Hunter failed: {exc}"
        log.exception(err_msg)
        _set_job_meta(
            redis_conn,
            job_id,
            progress=100,
            status="failed",
            message=err_msg,
            error=str(exc),
        )
        try:
            update_job_status_sync(job_id, "failed", error=str(exc))
        except Exception:
            pass
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.FAILED,
            100,
            err_msg,
        )


def run_matching_job(job_id: str) -> None:
    """
    Updated to fetch everything from database using job_id.
    This ensures job persistence and enables recovery after crashes.
    """
    set_job_id(job_id)
    redis_conn = create_redis_connection()  # Use connection with keepalive
    engine = SYNC_DB_ENGINE
    job_started = time.time()

    source_url = None
    reference_url = None
    webhook_url = None
    try:
        log.info("Starting job %s", job_id)

        # Check for cancellation at start
        if check_cancellation_sync(redis_conn, job_id):
            log.info("Job %s was cancelled before starting", job_id)
            emit_progress(
                redis_conn, job_id, ProgressPhase.FAILED, 100, "Job cancelled by user"
            )
            update_job_status_sync(job_id, "cancelled", error="Cancelled by user")
            return

        emit_progress(redis_conn, job_id, ProgressPhase.INITIALIZING, 5, "Starting job")
        _set_job_meta(
            redis_conn,
            job_id,
            status="processing",
            progress=5,
            processed_rows=0,
            message="Starting job",
        )

        # Fetch job from database
        with Session(engine) as session:
            job = session.get(Job, _job_pk(job_id))
            if not job:
                raise ValueError(f"Job {job_id} not found in database")

            # Set account context for structured logging
            _acct = getattr(job, "account_id", None)
            if _acct:
                set_account_id(str(_acct))

            # Update job status
            job.status = JobStatus.PROCESSING
            session.add(job)
            session.commit()

            job_config = job.config or {}

            if isinstance(job_config, dict) and job_config.get("job_type") == "justcall_push":
                account_fk = getattr(job, "account_id", None)
                session.expunge(job)
                return _run_justcall_job_sync(
                    job_id, job_config, account_fk, redis_conn
                )
            if isinstance(job_config, dict) and job_config.get("job_type") == "heyreach_push":
                account_fk = getattr(job, "account_id", None)
                session.expunge(job)
                return _run_heyreach_job_sync(
                    job_id, job_config, account_fk, redis_conn
                )

            if job.job_type == JobType.ID_HUNTER:
                account_fk = getattr(job, "account_id", None)
                session.expunge(job)
                return _run_id_hunter_job_sync(
                    job_id, job_config, account_fk, redis_conn
                )

            # Extract config from job
            config = job_config
            mode = config.get("mode", "match")
            source_url = config.get("source_url")
            reference_url = config.get("reference_url")
            webhook_url = config.get("webhook_url")

            log.info(f"Processing job {job_id} in {mode} mode")
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.LOADING_DATA,
                15,
                "Loading data files",
                mode=mode,
                source_url=source_url,
                reference_url=reference_url,
            )
            _set_job_meta(
                redis_conn,
                job_id,
                status="processing",
                progress=15,
                processed_rows=0,
                message="Loading data files",
            )

        # Check for cancellation before loading data
        if check_cancellation_sync(redis_conn, job_id):
            log.info("Job %s cancelled during initialization", job_id)
            emit_progress(
                redis_conn, job_id, ProgressPhase.FAILED, 100, "Job cancelled by user"
            )
            update_job_status_sync(job_id, "cancelled", error="Cancelled by user")
            return

        # Initialize storage backend
        from .storage import get_storage_backend

        storage = get_storage_backend()

        best_match_stats: Dict[str, Any] = {}
        if mode == "match":
            # For now, assume URLs are local file paths
            # In production, would download from S3/cloud storage
            src_df = read_csv_safely(_path_from_file_uri(source_url))
            ref_df = read_csv_safely(_path_from_file_uri(reference_url))

            # Build MatchConfig from stored config
            match_kwargs = {"apply_blocking": bool(config.get("apply_blocking", True))}
            if config.get("block_col"):
                match_kwargs["block_col"] = config.get("block_col")
            if config.get("source_block_col"):
                match_kwargs["source_block_col"] = config.get("source_block_col")
            if config.get("ref_block_col"):
                match_kwargs["ref_block_col"] = config.get("ref_block_col")

            cfg = MatchConfig(
                job_id=job_id,
                src_path=_path_from_file_uri(source_url),
                ref_path=_path_from_file_uri(reference_url),
                algorithm=config.get("algorithm", "WRatio"),
                threshold=config.get("threshold", 80),
                maps=config.get("maps", []),
                src_id_col=config.get("src_id_col"),
                ref_id_col=config.get("ref_id_col"),
                **match_kwargs,
            )

            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.PREPROCESSING,
                30,
                "Preprocessing data",
                source_rows=len(src_df),
                reference_rows=len(ref_df),
            )
            log.info(
                f"Loaded source: {len(src_df)} rows, reference: {len(ref_df)} rows"
            )
            _set_job_meta(
                redis_conn,
                job_id,
                status="processing",
                progress=30,
                processed_rows=0,
                total_rows=len(src_df),
                message="Preprocessing data",
            )

            # Check for cancellation before heavy processing
            if check_cancellation_sync(redis_conn, job_id):
                log.info("Job %s cancelled before matching", job_id)
                emit_progress(
                    redis_conn,
                    job_id,
                    ProgressPhase.FAILED,
                    100,
                    "Job cancelled by user",
                )
                update_job_status_sync(job_id, "cancelled", error="Cancelled by user")
                return

            # Use real match engine
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.MATCHING,
                50,
                "Running match engine",
                algorithm=cfg.algorithm,
                threshold=cfg.threshold,
            )
            try:
                results_df = run_match_worker(cfg, src_df, ref_df)
                if results_df is not None and not results_df.empty:
                    best_only = bool(config.get("best_match_only", True))
                    max_matches_cfg = config.get("max_matches_per_source")
                    max_matches_per_source = None
                    if max_matches_cfg is not None:
                        try:
                            max_matches_per_source = max(1, int(max_matches_cfg))
                        except Exception:
                            max_matches_per_source = None
                    target_per_source = max_matches_per_source or 1
                    if (
                        best_only
                        or max_matches_per_source is not None
                        or target_per_source == 1
                    ):
                        reduced_df, stats = apply_best_match_per_source(
                            results_df,
                            configuration=config,
                            max_per_source=target_per_source,
                            logger=log,
                        )
                        if stats.get("best_match_applied"):
                            best_match_stats = stats
                            results_df = reduced_df
                        else:
                            log.warning(
                                "best-match reduction requested but source/score columns were not detected; continuing with full candidate set."
                            )
            except RuntimeError as e:
                if "cancelled" in str(e).lower():
                    log.info("Job %s cancelled during matching", job_id)
                    emit_progress(
                        redis_conn,
                        job_id,
                        ProgressPhase.FAILED,
                        100,
                        "Job cancelled by user",
                    )
                    update_job_status_sync(
                        job_id, "cancelled", error="Cancelled by user"
                    )
                    return
                raise

        elif mode == "duplicate" or mode == "dedupe":
            # For dedupe mode, only need source file
            df = read_csv_safely(_path_from_file_uri(source_url))

            # Guard against empty data
            if len(df) == 0:
                error_msg = "No data rows found in uploaded file (empty or header-only)"
                log.error(f"Job {job_id} failed: {error_msg}")
                emit_progress(redis_conn, job_id, ProgressPhase.FAILED, 100, error_msg)
                update_job_status_sync(job_id, "failed", error=error_msg)
                return

            # Import settings for defaults
            from . import settings as saas_settings

            # Auto-detect config if missing
            rec_id_col = config.get("rec_id_col") or config.get("src_id_col")
            maps = config.get("maps", [])

            # If rec_id_col or maps are missing, try auto-detection
            if not rec_id_col or not maps:
                try:
                    from ..services.smart_auto_detector import SmartAutoDetector

                    detector = SmartAutoDetector(mode="dedupe", quick_mode=True)
                    detection_result = detector.detect(df, None)
                    if detection_result:
                        if not rec_id_col and detection_result.source_id_column:
                            rec_id_col = detection_result.source_id_column
                            log.info(f"Auto-detected rec_id_col: {rec_id_col}")
                        if not maps and detection_result.mappings:
                            maps = detection_result.mappings
                            log.info(f"Auto-detected mappings: {maps}")
                except Exception as e:
                    log.warning(f"Auto-detection failed, using defaults: {e}")
                    # Fall back to first column as ID if still missing
                    if not rec_id_col and len(df.columns) > 0:
                        rec_id_col = df.columns[0]

            # Build DedupeConfig from stored config with all required fields
            cfg = DedupeConfig(
                job_id=job_id,
                input_path=_path_from_file_uri(source_url),
                algorithm=config.get("algorithm", "WRatio"),
                threshold=config.get(
                    "threshold", saas_settings.DEFAULT_DEDUPE_THRESHOLD
                ),
                maps=maps or [],
                rec_id_col=rec_id_col,
                ruleset_name=config.get("ruleset_name", "default"),
                block_key_length=int(
                    config.get(
                        "block_key_length", saas_settings.DEFAULT_BLOCK_KEY_LENGTH
                    )
                ),
                dup_block_limit=int(
                    config.get("dup_block_limit", saas_settings.DEFAULT_DUP_BLOCK_LIMIT)
                ),
                apply_blocking=bool(config.get("apply_blocking", True)),
                block_col=config.get("block_col", "Domain"),
            )

            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.PREPROCESSING,
                30,
                "Preprocessing data",
                total_rows=len(df),
            )
            log.info(f"Loaded {len(df)} rows for deduplication")
            _set_job_meta(
                redis_conn,
                job_id,
                status="processing",
                progress=30,
                processed_rows=0,
                total_rows=len(df),
                message="Preprocessing data",
            )

            # Check for cancellation before heavy processing
            if check_cancellation_sync(redis_conn, job_id):
                log.info("Job %s cancelled before deduplication", job_id)
                emit_progress(
                    redis_conn,
                    job_id,
                    ProgressPhase.FAILED,
                    100,
                    "Job cancelled by user",
                )
                update_job_status_sync(job_id, "cancelled", error="Cancelled by user")
                return

            # Use real dedupe engine
            emit_progress(
                redis_conn,
                job_id,
                ProgressPhase.CLUSTERING,
                50,
                "Running dedupe engine",
                algorithm=cfg.algorithm,
                threshold=cfg.threshold,
            )
            try:
                results_df = run_dedupe_worker(cfg, df)
            except RuntimeError as e:
                if "cancelled" in str(e).lower():
                    log.info("Job %s cancelled during deduplication", job_id)
                    emit_progress(
                        redis_conn,
                        job_id,
                        ProgressPhase.FAILED,
                        100,
                        "Job cancelled by user",
                    )
                    update_job_status_sync(
                        job_id, "cancelled", error="Cancelled by user"
                    )
                    return
                raise

        else:
            raise ValueError(f"Unknown mode '{mode}'")

        # Final cancellation check before saving
        if check_cancellation_sync(redis_conn, job_id):
            log.info("Job %s cancelled before saving results", job_id)
            emit_progress(
                redis_conn, job_id, ProgressPhase.FAILED, 100, "Job cancelled by user"
            )
            update_job_status_sync(job_id, "cancelled", error="Cancelled by user")
            return

        # Save results location and metadata
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.SAVING_RESULTS,
            90,
            "Saving results",
            matches_found=len(results_df),
        )
        try:
            # processed_rows approximations: match -> total source rows; dedupe -> total rows processed
            processed_rows = len(src_df) if mode == "match" else len(df)
        except Exception:
            processed_rows = 0
        _set_job_meta(
            redis_conn,
            job_id,
            status="processing",
            progress=90,
            processed_rows=processed_rows,
            message="Saving results",
        )

        # Determine results path using safe output directory
        out_dir = get_safe_output_dir(cfg)
        if mode == "match":
            results_path = out_dir / f"match-{job_id}.parquet"
        else:
            results_path = out_dir / f"dupes-{job_id}.parquet"
        results_path_posix = _ensure_posix_path(results_path)

        # Update job status with results location using the synchronous helper
        elapsed_s = max(0.0, time.time() - job_started)
        update_job_status_sync(
            job_id,
            "completed",
            results_path=results_path_posix,
            matches_found=len(results_df),
            rows_processed=processed_rows,
            elapsed_s=elapsed_s,
        )

        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.COMPLETED,
            100,
            "Job completed successfully",
            matches_found=len(results_df),
            results_path=results_path_posix,
        )
        log.info(
            "Job %s completed successfully with %d results", job_id, len(results_df)
        )
        _set_job_meta(
            redis_conn,
            job_id,
            status="completed",
            state="completed",
            progress=100,
            processed_rows=processed_rows,
            results_path=results_path_posix,
            matches_found=len(results_df),
            elapsed_s=elapsed_s,
            candidates_total=best_match_stats.get("candidates_total")
            if best_match_stats
            else None,
            message="Job completed successfully",
        )

        # Clear any cancellation flag that might have been set but not processed
        clear_cancellation_flag(redis_conn, job_id)

        # Send webhook notification if configured
        if webhook_url:
            notification_data = {
                "job_id": job_id,
                "status": "completed",
                "matches_found": len(results_df),
                "rows_processed": processed_rows,
                "elapsed_s": elapsed_s,
                "candidates_total": best_match_stats.get("candidates_total")
                if best_match_stats
                else None,
                "results_url": f"/api/v1/jobs/{job_id}/results",
                "results_path": results_path_posix,
            }
            notify_webhook(webhook_url, job_id, "completed", notification_data)

    except Exception as exc:  # pragma: no cover
        log.exception("Job %s failed: %s", job_id, exc)
        emit_progress(
            redis_conn,
            job_id,
            ProgressPhase.FAILED,
            100,
            f"Job failed: {exc}",
            error=str(exc),
            error_type=type(exc).__name__,
        )
        _set_job_meta(
            redis_conn,
            job_id,
            status="failed",
            state="failed",
            progress=100,
            processed_rows=0,
            message=f"Job failed: {exc}",
            error=str(exc),
        )

        # Update job status in database using the synchronous helper
        update_job_status_sync(job_id, "failed", error=f"{type(exc).__name__}: {exc}")

        # Send webhook notification if configured
        if webhook_url:
            notification_data = {
                "job_id": job_id,
                "status": "failed",
                "error": str(exc),
                "error_type": type(exc).__name__,
            }
            notify_webhook(webhook_url, job_id, "failed", notification_data)

        raise
    finally:
        # Clean up local upload files (best effort)
        try:
            if source_url:
                _safe_unlink(_path_from_file_uri(source_url))
            if reference_url:
                _safe_unlink(_path_from_file_uri(reference_url))
        except Exception:
            pass
        clear_context()


async def _run_enrichment_job_async(job_id: str) -> None:
    from fmatch.saas import settings
    from fmatch.saas.api.progress import get_redis as get_async_redis
    from fmatch.saas.database import session_context
    from fmatch.saas.models import BYOEnrichmentJob, EnrichmentJobStatus
    from fmatch.saas.services.enrichment_worker import EnrichmentWorker, LeaseNotAcquiredError
    from fmatch.saas.services.sheets_writer import SheetsWriter
    from fmatch.saas.services.email_notifications import send_enrichment_complete_notification

    try:
        job_uuid = UUID(str(job_id))
    except Exception:
        log.error("[ENRICHMENT_WORKER] Invalid job_id: %s", job_id)
        return

    redis_conn = await get_async_redis()
    rows_key = f"byo_enrichment_rows:{job_uuid}"
    meta_key = f"byo_enrichment_meta:{job_uuid}"

    rows_raw = await redis_conn.get(rows_key)
    if not rows_raw:
        log.error("[ENRICHMENT_WORKER] No rows found for job %s", job_uuid)
        return

    try:
        rows_payload = rows_raw.decode() if isinstance(rows_raw, (bytes, bytearray)) else rows_raw
        rows = json.loads(rows_payload)
    except Exception as exc:
        log.error("[ENRICHMENT_WORKER] Failed to parse rows for job %s: %s", job_uuid, exc)
        return

    meta = {}
    meta_raw = await redis_conn.get(meta_key)
    if meta_raw:
        try:
            meta_payload = meta_raw.decode() if isinstance(meta_raw, (bytes, bytearray)) else meta_raw
            meta = json.loads(meta_payload)
        except Exception:
            meta = {}

    def _sanitize_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        cleaned = []
        for result in results or []:
            if not isinstance(result, dict):
                continue
            sanitized = dict(result)
            sanitized.pop("raw", None)
            sanitized.pop("custom", None)
            cleaned.append(sanitized)
        return cleaned

    async with session_context() as session:
        job = await session.get(BYOEnrichmentJob, job_uuid)
        if not job:
            log.error("[ENRICHMENT_WORKER] Job %s not found", job_uuid)
            return

        # Set account context for structured logging
        _ws = getattr(job, "workspace_id", None)
        if _ws:
            set_account_id(str(_ws))

        if job.status in (EnrichmentJobStatus.COMPLETED.value, EnrichmentJobStatus.FAILED.value):
            log.info("[ENRICHMENT_WORKER] Job %s already %s", job_uuid, job.status)
            return

        chunk_size = job.chunk_size or settings.BYO_ENRICHMENT_CHUNK_SIZE
        cursor = job.cursor or 0
        results_accum: List[Dict[str, Any]] = []
        task_id = f"rq:{uuid4().hex}"
        worker = EnrichmentWorker(session, job_uuid, task_id)

        try:
            while cursor < len(rows):
                chunk = rows[cursor: cursor + chunk_size]
                if not chunk:
                    break
                result = await worker.process_chunk(chunk)
                results_accum.extend(_sanitize_results(result.get("results") or []))
                cursor = int(result.get("cursor") or (cursor + len(chunk)))
                if result.get("is_complete"):
                    break
        except LeaseNotAcquiredError:
            log.warning("[ENRICHMENT_WORKER] Lease not acquired for job %s", job_uuid)
            return
        except Exception as exc:
            log.exception("[ENRICHMENT_WORKER] Job %s failed: %s", job_uuid, exc)
            job.status = EnrichmentJobStatus.FAILED.value
            job.error_message = str(exc)
            job.completed_at = datetime.utcnow()
            await session.commit()
            return

        await session.refresh(job)

        if results_accum and job.status == EnrichmentJobStatus.COMPLETED.value:
            from fmatch.saas.services.enrichment_adapters import EnrichmentStatus

            summary = dict(job.summary or {})
            provider_breakdown = summary.get("provider_breakdown") or {}
            if not provider_breakdown:
                for result in results_accum:
                    if result.get("status") != EnrichmentStatus.ENRICHED.value:
                        continue
                    provider = result.get("provider") or "unknown"
                    provider_breakdown[provider] = provider_breakdown.get(provider, 0) + 1
            summary.update(
                {
                    "rows_total": job.row_count,
                    "rows_enriched": job.rows_enriched,
                    "rows_skipped": job.rows_skipped,
                    "rows_errored": job.rows_errored,
                    "credits_consumed": job.credits_consumed,
                    "provider_breakdown": provider_breakdown,
                    "results": results_accum,
                }
            )
            job.summary = summary
            await session.commit()

        sheet_id = meta.get("sheet_id")
        sheet_name = meta.get("sheet_name")
        if sheet_id and sheet_name and results_accum:
            config = {
                "output_fields": job.output_fields or [],
                "enrichment_type": job.enrichment_type,
                "force_reenrich": bool(meta.get("force_reenrich")),
                "sheet_name": sheet_name,
            }
            writer = SheetsWriter(
                sheet_id=sheet_id,
                account_id=str(job.workspace_id),
                api_key="",
            )
            await writer.store_enrichment_results(
                sheet_name=sheet_name,
                results=results_accum,
                config=config,
                metadata={"job_id": str(job.id)},
            )

        if job.notify_email and job.status == EnrichmentJobStatus.COMPLETED.value:
            summary = dict(job.summary or {})
            if not summary.get("notification_sent"):
                try:
                    record_label = "contacts" if job.enrichment_type == "contact" else "companies"
                    await send_enrichment_complete_notification(
                        user_email=job.notify_email,
                        job_name=f"Enrichment ({job.row_count} {record_label})",
                        job_id=str(job.id),
                        total_contacts=job.row_count,
                        enriched_count=job.rows_enriched,
                        not_found_count=max(0, job.rows_processed - job.rows_enriched - job.rows_errored),
                        error_count=job.rows_errored,
                        spreadsheet_url=job.spreadsheet_url,
                        record_label=record_label,
                    )
                    summary["notification_sent"] = True
                    job.summary = summary
                    await session.commit()
                except Exception as notify_exc:
                    log.warning(
                        "[ENRICHMENT_WORKER] Failed to send notification for job %s: %s",
                        job_uuid,
                        notify_exc,
                    )

    try:
        await redis_conn.delete(rows_key, meta_key)
    except Exception:
        pass


def run_enrichment_job(job_id: str) -> None:
    """RQ entrypoint for BYO enrichment background processing."""
    set_job_id(job_id)
    try:
        asyncio.run(_run_enrichment_job_async(job_id))
    except Exception as exc:
        log.exception("[ENRICHMENT_WORKER] Unhandled failure for job %s: %s", job_id, exc)
    finally:
        clear_context()


async def _run_sheet_webhook_delivery_async(inbound_id: str) -> None:
    try:
        from fmatch.saas.api.v2.sheet_webhooks import process_queued_sheet_webhook_delivery
    except ModuleNotFoundError:
        from src.fmatch.saas.api.v2.sheet_webhooks import (
            process_queued_sheet_webhook_delivery,
        )

    await process_queued_sheet_webhook_delivery(inbound_id)


def _compute_sheet_webhook_db_retry_delay_seconds(retry_attempt: int) -> int:
    attempt = max(1, int(retry_attempt))
    delay = SHEET_WEBHOOK_DB_RETRY_BASE_SECONDS * (2 ** (attempt - 1))
    return max(
        SHEET_WEBHOOK_DB_RETRY_BASE_SECONDS,
        min(SHEET_WEBHOOK_DB_RETRY_MAX_SECONDS, delay),
    )


def _reset_sheet_webhook_loop() -> None:
    global _sheet_webhook_loop
    loop = _sheet_webhook_loop
    _sheet_webhook_loop = None
    if loop is None:
        return
    try:
        if not loop.is_closed():
            loop.close()
    except Exception:
        log.debug("[SHEET_WEBHOOK_WORKER] Failed to close shared event loop", exc_info=True)


def _get_sheet_webhook_loop() -> asyncio.AbstractEventLoop:
    global _sheet_webhook_loop
    if _sheet_webhook_loop is None or _sheet_webhook_loop.is_closed():
        _sheet_webhook_loop = asyncio.new_event_loop()
    return _sheet_webhook_loop


def _run_sheet_webhook_delivery_on_shared_loop(inbound_id: str) -> None:
    loop = _get_sheet_webhook_loop()
    try:
        asyncio.set_event_loop(loop)
        loop.run_until_complete(_run_sheet_webhook_delivery_async(inbound_id))
    finally:
        asyncio.set_event_loop(None)


def _iter_exception_chain(exc: BaseException):
    seen = set()
    stack: List[Optional[BaseException]] = [exc]
    while stack:
        current = stack.pop()
        if current is None:
            continue
        marker = id(current)
        if marker in seen:
            continue
        seen.add(marker)
        yield current
        stack.append(getattr(current, "orig", None))
        stack.append(getattr(current, "__cause__", None))
        stack.append(getattr(current, "__context__", None))


def _exception_chain_contains(exc: BaseException, markers: tuple[str, ...]) -> bool:
    lowered = tuple(marker.lower() for marker in markers)
    for current in _iter_exception_chain(exc):
        text = str(current).lower()
        if any(marker in text for marker in lowered):
            return True
    return False


def _is_transient_sheet_webhook_worker_error(
    exc: BaseException,
    is_db_capacity_error: Callable[[BaseException], bool],
) -> bool:
    if is_db_capacity_error(exc):
        return True
    return _exception_chain_contains(exc, ("event loop is closed",))


def _load_sheet_webhook_retry_helpers():
    try:
        from fmatch.saas.api.v2.sheet_webhooks import (
            _enqueue_sheet_webhook_delivery,
            _is_db_capacity_error,
        )
    except ModuleNotFoundError:
        from src.fmatch.saas.api.v2.sheet_webhooks import (
            _enqueue_sheet_webhook_delivery,
            _is_db_capacity_error,
        )
    return _enqueue_sheet_webhook_delivery, _is_db_capacity_error


def run_sheet_webhook_delivery(inbound_id: str, db_retry_attempt: int = 0) -> None:
    """RQ entrypoint for queued sheet webhook deliveries."""
    set_job_id(inbound_id)  # use inbound_id as correlation key
    safe_attempt = max(0, int(db_retry_attempt))
    try:
        _run_sheet_webhook_delivery_on_shared_loop(inbound_id)
    except Exception as exc:
        enqueue_retry, is_db_capacity_error = _load_sheet_webhook_retry_helpers()
        is_retryable = _is_transient_sheet_webhook_worker_error(exc, is_db_capacity_error)
        if _exception_chain_contains(exc, ("event loop is closed",)):
            _reset_sheet_webhook_loop()

        if is_retryable:
            if safe_attempt < SHEET_WEBHOOK_DB_RETRY_MAX_ATTEMPTS:
                next_attempt = safe_attempt + 1
                delay_seconds = _compute_sheet_webhook_db_retry_delay_seconds(next_attempt)
                retry_job_id = enqueue_retry(
                    str(inbound_id),
                    delay_seconds=delay_seconds,
                    db_retry_attempt=next_attempt,
                )
                log.warning(
                    "[SHEET_WEBHOOK_WORKER] Retryable failure for inbound %s; requeued attempt=%s/%s delay=%ss job=%s error=%s",
                    inbound_id,
                    next_attempt,
                    SHEET_WEBHOOK_DB_RETRY_MAX_ATTEMPTS,
                    delay_seconds,
                    retry_job_id,
                    exc,
                )
                return

            log.error(
                "[SHEET_WEBHOOK_WORKER] Retryable failure for inbound %s after %s retry attempts; marking failed error=%s",
                inbound_id,
                safe_attempt,
                exc,
            )
        else:
            log.error(
                "[SHEET_WEBHOOK_WORKER] Delivery failed for inbound %s: %s",
                inbound_id,
                exc,
            )
        raise
    finally:
        clear_context()


async def _run_scheduled_transform_job_async(job_id: str) -> None:
    """Background transform job runner for Sheets scheduled results."""
    from fmatch.saas import settings
    from fmatch.saas.api.progress import get_redis as get_async_redis
    from fmatch.saas.database import session_context
    from fmatch.saas.services.transform_service import batch_transform
    from fmatch.saas.services.sheets_writer import SheetsWriter
    from fmatch.saas.services.gremlin_intent_router import ResolutionTarget

    redis_conn = await get_async_redis()
    rows_key = f"scheduled_transform_rows:{job_id}"
    meta_key = f"scheduled_transform_meta:{job_id}"

    rows_raw = await redis_conn.get(rows_key)
    if not rows_raw:
        log.error("[TRANSFORM_WORKER] No rows found for job %s", job_id)
        return

    try:
        rows_payload = rows_raw.decode() if isinstance(rows_raw, (bytes, bytearray)) else rows_raw
        values = json.loads(rows_payload)
    except Exception as exc:
        log.error("[TRANSFORM_WORKER] Failed to parse rows for job %s: %s", job_id, exc)
        return

    meta = {}
    meta_raw = await redis_conn.get(meta_key)
    if meta_raw:
        try:
            meta_payload = meta_raw.decode() if isinstance(meta_raw, (bytes, bytearray)) else meta_raw
            meta = json.loads(meta_payload)
        except Exception:
            meta = {}

    op = meta.get("op") or "enrich_company_name"
    params = meta.get("params") or {}
    account_id = meta.get("account_id")
    if account_id:
        set_account_id(str(account_id))
    target_raw = meta.get("resolution_target") or None
    resolution_target = None
    if target_raw:
        try:
            resolution_target = ResolutionTarget(str(target_raw).lower())
        except Exception:
            resolution_target = ResolutionTarget.SMART

    chunk_size = int(os.getenv("TRANSFORM_MAX_ROWS", "200"))
    results_accum: List[Dict[str, Any]] = []

    def _prune_row_output(row_output: Dict[str, Any]) -> Dict[str, Any]:
        if not isinstance(row_output, dict):
            return {"value": row_output}
        keep = {
            "company_name",
            "resolution_confidence",
            "confidence",
            "resolution_source",
            "source",
            "skipped",
            "skip_reason",
            "sfdc_account_id",
            "sfdc_account_name",
        }
        return {k: row_output.get(k) for k in keep if k in row_output}

    async def _process_chunks(db_session: Optional[AsyncSession]) -> None:
        for start in range(0, len(values), chunk_size):
            chunk = values[start : start + chunk_size]
            try:
                result = await batch_transform(
                    op,
                    chunk,
                    params,
                    account_id=account_id,
                    db=db_session,
                    resolution_target=resolution_target,
                )
            except Exception as exc:
                log.exception("[TRANSFORM_WORKER] Transform chunk failed for job %s: %s", job_id, exc)
                continue

            data = result.get("data") if isinstance(result, dict) else None
            if not isinstance(data, list):
                continue

            for idx, row_output in enumerate(data):
                in_row = chunk[idx] if idx < len(chunk) else None
                cleaned = _prune_row_output(row_output if isinstance(row_output, dict) else {})
                if isinstance(in_row, dict):
                    if "row_index" in in_row:
                        cleaned["row_index"] = in_row.get("row_index")
                    elif "rowIndex" in in_row:
                        cleaned["row_index"] = in_row.get("rowIndex")
                    if "company" in in_row:
                        cleaned["input_company"] = in_row.get("company")
                    if "domain" in in_row:
                        cleaned["input_domain"] = in_row.get("domain")
                    if "website" in in_row and "input_domain" not in cleaned:
                        cleaned["input_domain"] = in_row.get("website")
                results_accum.append(cleaned)

    if account_id:
        async with session_context() as session:
            await _process_chunks(session)
    else:
        await _process_chunks(None)

    sheet_id = meta.get("sheet_id")
    sheet_name = meta.get("sheet_name")
    if not sheet_id or not sheet_name:
        log.error("[TRANSFORM_WORKER] Missing sheet info for job %s", job_id)
        return

    config = {
        "op": op,
        "row_base": meta.get("row_base"),
        "row_count": meta.get("row_count"),
        "company_col": meta.get("company_col"),
        "domain_col": meta.get("domain_col"),
        "only_fill_empty": meta.get("only_fill_empty"),
        "conversation_id": meta.get("conversation_id"),
        "scope": meta.get("scope"),
        "sheet_gid": meta.get("sheet_gid"),
        "step_id": meta.get("step_id"),
    }

    if not account_id:
        log.error(
            "[TRANSFORM_WORKER] Missing account_id for job %s; refusing to store scheduled results without tenant scoping",
            job_id,
        )
        return

    writer = SheetsWriter(sheet_id=sheet_id, account_id=str(account_id), api_key="")
    await writer.store_transform_results(
        sheet_name=sheet_name,
        results=results_accum,
        config=config,
        metadata={"job_id": job_id},
    )

    try:
        await redis_conn.delete(rows_key, meta_key)
    except Exception:
        pass


def run_scheduled_transform_job(job_id: str) -> None:
    """RQ entrypoint for scheduled transform background processing."""
    set_job_id(job_id)
    try:
        asyncio.run(_run_scheduled_transform_job_async(job_id))
    except Exception as exc:
        log.exception("[TRANSFORM_WORKER] Unhandled failure for job %s: %s", job_id, exc)
    finally:
        clear_context()


async def _run_scraper_job_async(job_id: str) -> None:
    from fmatch.saas import settings
    from fmatch.saas.database import session_context
    from fmatch.saas.models import ScraperJob, ScraperJobStatus
    from fmatch.saas.services.email_notifications import send_scraper_complete_notification
    from fmatch.saas.services.scraper_blob_storage import ScraperBlobStorage
    from fmatch.saas.workers.scraper_worker import ScraperWorker, LeaseNotAcquiredError

    try:
        job_uuid = UUID(str(job_id))
    except Exception:
        log.error("[SCRAPER_WORKER] Invalid job_id: %s", job_id)
        return

    storage = ScraperBlobStorage()

    async with session_context() as session:
        job = await session.get(ScraperJob, job_uuid)
        if not job:
            log.error("[SCRAPER_WORKER] Job %s not found", job_uuid)
            return

        # Set account context for structured logging
        _ws = getattr(job, "workspace_id", None)
        if _ws:
            set_account_id(str(_ws))

        if job.status in (ScraperJobStatus.COMPLETED.value, ScraperJobStatus.FAILED.value):
            log.info("[SCRAPER_WORKER] Job %s already %s", job_uuid, job.status)
            return

        rows = await storage.get_inputs(job.workspace_id, job_uuid)
        if not rows:
            log.error("[SCRAPER_WORKER] No rows found for job %s", job_uuid)
            return

        chunk_size = settings.SCRAPER_CHUNK_SIZE
        cursor = job.cursor or 0
        task_id = f"rq:{uuid4().hex}"
        worker = ScraperWorker(session, job_uuid, task_id)

        try:
            while cursor < len(rows):
                chunk = rows[cursor: cursor + chunk_size]
                if not chunk:
                    break
                result = await worker.process_chunk(chunk)
                cursor = int(result.get("cursor") or (cursor + len(chunk)))
                if result.get("is_complete"):
                    break
            await session.refresh(job)
            if job.notify_email and job.status == ScraperJobStatus.COMPLETED.value:
                spreadsheet_url = (
                    f"https://docs.google.com/spreadsheets/d/{job.sheet_id}"
                    if job.sheet_id
                    else None
                )
                try:
                    await send_scraper_complete_notification(
                        user_email=job.notify_email,
                        job_name=f"Web Scraper ({job.rows_total} rows)",
                        job_id=str(job.id),
                        total_rows=job.rows_total,
                        scraped_count=job.rows_scraped,
                        skipped_count=job.rows_skipped,
                        error_count=job.rows_errored,
                        spreadsheet_url=spreadsheet_url,
                    )
                except Exception as notify_exc:
                    log.warning(
                        "[SCRAPER_WORKER] Failed to send notification for job %s: %s",
                        job_uuid,
                        notify_exc,
                    )
        except LeaseNotAcquiredError:
            log.warning("[SCRAPER_WORKER] Lease not acquired for job %s", job_uuid)
            return
        except Exception as exc:
            log.exception("[SCRAPER_WORKER] Job %s failed: %s", job_uuid, exc)
            job.status = ScraperJobStatus.FAILED.value
            job.error_message = str(exc)
            job.completed_at = datetime.utcnow()
            await session.commit()
            return


def run_scraper_job(job_id: str) -> None:
    """RQ entrypoint for BYO scraper background processing."""
    set_job_id(job_id)
    try:
        asyncio.run(_run_scraper_job_async(job_id))
    except Exception as exc:
        log.exception("[SCRAPER_WORKER] Unhandled failure for job %s: %s", job_id, exc)
    finally:
        clear_context()


# ----------------------------------------------------------------------------
# CLI utility – rarely used in tests
# ----------------------------------------------------------------------------
class WindowsNoOpTimeout:
    """No-op timeout class for Windows (which doesn't support SIGALRM)"""

    def __init__(self, timeout, exception_class, job_id=None):
        pass

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

    def cancel(self):
        pass


def main():
    """Main entry point for the worker"""
    # Windows doesn't support SIGALRM, must use SimpleWorker
    if sys.platform.startswith("win"):
        from rq import SimpleWorker

        worker_cls = SimpleWorker
        print("[RQ_WORKER] Using SimpleWorker (Windows mode)")
    else:
        from rq import Worker

        worker_cls = Worker

    # Create fresh connection for worker
    worker_conn = create_redis_connection()
    queues = [Queue(name, connection=worker_conn) for name in listen]
    print(f"[RQ_WORKER] Boot: redis={REDIS_URL} queues={listen}")

    # Create worker with job_class timeout disabled on Windows
    worker = worker_cls(queues, connection=worker_conn)

    # On Windows, replace death penalty with no-op class
    if sys.platform.startswith("win"):
        worker.death_penalty_class = WindowsNoOpTimeout

    worker.work()


if __name__ == "__main__":
    main()


# ----------------------------------------------------------------------------
# PK helper: allow string job IDs or UUIDs
# ----------------------------------------------------------------------------
def _job_pk(job_id: str):
    """Return a DB primary key for Job (string UUID in our schema)."""
    return str(job_id)
