"""
FoundryMatch enrichment API endpoints.

Provides eligibility discovery and append workflows for enriching Contacts and Leads.

POST-V1: This feature is SHELVED for V1 launch.

Status: Backend APIs are complete, but frontend UI and full workflow are incomplete.
Reason: Large surface area, needs extensive testing, and UX concerns with modal workflow.
Decision: Menu items hidden. APIs preserved for post-V1 completion.

The /api/v2/enrichment/* endpoints provide:
- Eligibility scanning (scan, eligible)
- PDL enrichment preview and apply
- Salesforce field mapping and fill operations
- Usage tracking and quota management

Alternative for V1: "Contact & Company Enrichment" provides enrichment via Sheets.

Environment Variables:
    FM_ENRICHMENT_DEFAULT_LIMIT: Default number of eligible records to return (default: 250)
    FM_ENRICHMENT_MAX_LIMIT: Maximum allowed limit for eligible records (default: 2000)

    For testing with limited PDL credits, set lower limits:
        export FM_ENRICHMENT_DEFAULT_LIMIT=10
        export FM_ENRICHMENT_MAX_LIMIT=25
"""

import asyncio
import hashlib
import json
import logging
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
import math
import os
from typing import Any, Dict, List, Optional, Set, Tuple, Literal, cast

from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession

from sqlalchemy import select
from sqlalchemy.exc import ProgrammingError

from ...auth import verify_api_key
from ...config.enrichment import HEARTBEAT_SECONDS, enrichment_enabled
from ...db import AsyncSessionLocal, get_session
from ...integrations.people_data_labs import PeopleDataLabsAdapter
from ...models import EnrichmentFieldMapping
from ...services.contact_enrichment import (
    ContactEnrichmentCoordinator,
    EnrichmentMatch,
    EnrichmentWriter,
    MissingFieldScanner,
    MissingFieldCandidate,
)
from ...services.enrichment_usage_tracker import EnrichmentUsageTracker
from ...services.salesforce_gateway import SalesforceGateway, SalesforceFieldAccessError
from ...services.telemetry import track_event

logger = logging.getLogger(__name__)


def require_enrichment_enabled():
    _ensure_enrichment_enabled()


router = APIRouter(
    prefix="/api/v2/enrichment",
    tags=["enrichment"],
    dependencies=[Depends(require_enrichment_enabled)],
)

# Configurable eligibility limits - can be overridden via environment for testing
ELIGIBLE_DEFAULT_LIMIT = int(os.getenv("FM_ENRICHMENT_DEFAULT_LIMIT", "250"))
ELIGIBLE_MAX_LIMIT = int(os.getenv("FM_ENRICHMENT_MAX_LIMIT", "2000"))


def _ensure_enrichment_enabled() -> None:
    if not enrichment_enabled():
        raise HTTPException(
            status_code=404,
            detail="Enrichment features are not currently available",
        )


async def _load_custom_field_map(
    db: AsyncSession, account_id: str
) -> Dict[str, Dict[str, str]]:
    """
    Load custom field mappings for an account.

    Returns a dict like:
    {
        "Account": {"company_domain": "Website"},
        "Contact": {"email": "Work_Email__c"},
        "Lead": {"email": "Email"}
    }

    Returns empty dicts if no custom mappings are configured. Optional fields (e.g. linkedin_url)
    only appear when explicitly mapped by the tenant.
    """
    try:
        result = await db.execute(
            select(EnrichmentFieldMapping).where(
                EnrichmentFieldMapping.account_id == account_id
            )
        )
        row = result.scalar_one_or_none()
    except ProgrammingError as exc:
        message = str(exc).lower()
        if "account_mappings" in message:
            logger.warning(
                "Account mappings column missing for account %s. "
                "Falling back to contact/lead mappings only.",
                account_id,
            )
            fallback_result = await db.execute(
                select(
                    EnrichmentFieldMapping.contact_mappings,
                    EnrichmentFieldMapping.lead_mappings,
                ).where(EnrichmentFieldMapping.account_id == account_id)
            )
            fallback_row = fallback_result.first()
            if not fallback_row:
                return {"Account": {}, "Contact": {}, "Lead": {}}
            contact_map, lead_map = fallback_row
            return {
                "Account": {},
                "Contact": contact_map or {},
                "Lead": lead_map or {},
            }
        raise

    if not row:
        return {"Account": {}, "Contact": {}, "Lead": {}}

    return {
        "Account": getattr(row, "account_mappings", {}) or {},
        "Contact": row.contact_mappings or {},
        "Lead": row.lead_mappings or {},
    }


@asynccontextmanager
async def _noop_async_context():
    yield


@asynccontextmanager
async def _reservation_heartbeat(
    account_id: str,
    idempotency_key: str,
    *,
    interval_seconds: int,
):
    """
    Best-effort background heartbeat to keep a reservation lease alive during long batches.
    """
    if not account_id or not idempotency_key or interval_seconds <= 0:
        yield
        return

    stop_event = asyncio.Event()
    task: Optional[asyncio.Task] = None

    async def _beat() -> None:
        try:
            while not stop_event.is_set():
                try:
                    await asyncio.wait_for(stop_event.wait(), interval_seconds)
                    break
                except asyncio.TimeoutError:
                    pass

                try:
                    async with AsyncSessionLocal() as session:
                        tracker = EnrichmentUsageTracker(session)
                        alive = await tracker.heartbeat(account_id, idempotency_key)
                        if not alive:
                            logger.warning(
                                "Reservation heartbeat stopped (account=%s idempotency=%s).",
                                account_id,
                                idempotency_key,
                            )
                            break
                except Exception:  # pragma: no cover - defensive guard
                    logger.exception(
                        "Failed to extend enrichment reservation lease (account=%s idempotency=%s).",
                        account_id,
                        idempotency_key,
                    )
        except asyncio.CancelledError:
            pass

    try:
        task = asyncio.create_task(_beat())
        yield
    finally:
        stop_event.set()
        if task:
            task.cancel()
            try:
                await task
            except asyncio.CancelledError:
                pass


def _hash_payload(payload: Dict[str, Any]) -> str:
    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
    return hashlib.sha256(encoded).hexdigest()


def _ensure_identifier(value: Optional[str], prefix: str) -> str:
    if value:
        return value
    return f"{prefix}-{uuid.uuid4().hex}"


# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# Request/Response Models
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━


class ScanRequest(BaseModel):
    """Request to scan Salesforce for missing contact data."""

    account_id: str = Field(..., description="FoundryMatch account ID")
    sfdc_object: str = Field(
        default="Contact", description="Salesforce object (Contact or Lead)"
    )
    limit: int = Field(default=1000, ge=1, le=10000, description="Max records to scan")
    filter_b2c: bool = Field(default=True, description="Filter B2C email domains")


class CandidateResponse(BaseModel):
    """A candidate record with missing fields."""

    sfdc_id: str
    sfdc_object: str
    first_name: Optional[str]
    last_name: Optional[str]
    full_name: Optional[str]
    company_name: Optional[str]
    company_domain: Optional[str]
    email: Optional[str]
    phone: Optional[str]
    mobile_phone: Optional[str]
    title: Optional[str]
    linkedin_url: Optional[str]
    missing_fields: List[str]
    match_key: str


class ScanResponse(BaseModel):
    """Response from scan operation."""

    ok: bool = True
    candidates: List[CandidateResponse]
    total_scanned: int
    total_candidates: int
    message: Optional[str] = None


class PreviewRequest(BaseModel):
    """Request to preview enrichment matches from PDL."""

    account_id: str = Field(..., description="FoundryMatch account ID")
    job_id: Optional[str] = Field(
        None, description="Client-side job identifier (for idempotency)"
    )
    chunk_id: Optional[str] = Field(None, description="Chunk identifier within the job")
    candidate_ids: List[str] = Field(..., description="List of SFDC IDs to enrich")
    sfdc_object: str = Field(
        default="Contact", description="Salesforce object (Contact or Lead)"
    )


class EnrichmentMatchResponse(BaseModel):
    """A successful enrichment match."""

    sfdc_id: str
    sfdc_object: str
    candidate_name: str

    # Current values
    current_email: Optional[str]
    current_phone: Optional[str]
    current_mobile_phone: Optional[str]
    current_title: Optional[str]
    current_company_name: Optional[str]
    current_company_domain: Optional[str]
    current_linkedin_url: Optional[str]

    # Enriched values
    enriched_email: Optional[str]
    enriched_phone: Optional[str]
    enriched_mobile_phone: Optional[str]
    enriched_title: Optional[str]
    enriched_linkedin_url: Optional[str]
    enriched_company_name: Optional[str]
    enriched_company_domain: Optional[str]

    # Quality indicators
    confidence: str
    likelihood: Optional[float]
    source: str

    # Fields that will be updated
    fields_to_update: List[str]


def _match_to_response(match: EnrichmentMatch) -> EnrichmentMatchResponse:
    """Convert internal enrichment match to API response payload."""
    candidate = match.candidate
    candidate_name = (
        candidate.full_name
        or " ".join(filter(None, [candidate.first_name, candidate.last_name])).strip()
    )

    return EnrichmentMatchResponse(
        sfdc_id=candidate.sfdc_id,
        sfdc_object=candidate.sfdc_object,
        candidate_name=candidate_name or candidate.sfdc_id,
        current_email=candidate.email,
        current_phone=candidate.phone,
        current_mobile_phone=candidate.mobile_phone,
        current_title=candidate.title,
        current_company_name=getattr(candidate, "company_name", None),
        current_company_domain=getattr(candidate, "company_domain", None),
        current_linkedin_url=getattr(candidate, "linkedin_url", None),
        enriched_email=match.enriched_email,
        enriched_phone=match.enriched_phone,
        enriched_mobile_phone=match.enriched_mobile_phone,
        enriched_title=match.enriched_title,
        enriched_linkedin_url=match.enriched_linkedin_url,
        enriched_company_name=match.enriched_company_name,
        enriched_company_domain=match.enriched_company_domain,
        confidence=match.confidence,
        likelihood=match.pdl_likelihood,
        source=match.source,
        fields_to_update=sorted(match.fields_to_update),
    )


class PreviewResponse(BaseModel):
    """Response from preview operation."""

    ok: bool = True
    matches: List[EnrichmentMatchResponse]
    stats: Dict = Field(default_factory=dict)
    usage_estimate: Dict = Field(default_factory=dict)
    message: Optional[str] = None


class ApplyRequest(BaseModel):
    """Request to apply enrichments to Salesforce."""

    account_id: str = Field(..., description="FoundryMatch account ID")
    job_id: Optional[str] = Field(
        None, description="Client-side job identifier (for idempotency)"
    )
    chunk_id: Optional[str] = Field(None, description="Chunk identifier within the job")
    match_ids: List[str] = Field(..., description="SFDC IDs of matches to apply")
    sfdc_object: str = Field(
        default="Contact", description="Salesforce object (Contact or Lead)"
    )
    dry_run: bool = Field(default=False, description="Preview changes without writing")


class ApplyResponse(BaseModel):
    """Response from apply operation."""

    ok: bool = True
    records_written: int = 0
    records_failed: int = 0
    fields_updated: Dict[str, int] = Field(default_factory=dict)
    matches: List[EnrichmentMatchResponse] = Field(default_factory=list)
    errors: List[Dict] = Field(default_factory=list)
    audit_trail: Optional[str] = None
    usage: Optional[Dict[str, Any]] = None
    quota_snapshot: Optional[Dict[str, Any]] = None
    preflight: Optional[Dict[str, Any]] = None
    message: Optional[str] = None


# ---------------------------------------------------------------------------
# Eligibility + append models (brand-neutral contract)
# ---------------------------------------------------------------------------


class EligibleCredits(BaseModel):
    """Credit pill summary for the Sheets modal."""

    remaining: int = 0
    estimate: int = 0
    reload_url: Optional[str] = None


class EligibleSummary(BaseModel):
    """Aggregated counts for UI chips."""

    total: int = 0
    email: int = 0
    mobile: int = 0
    title: int = 0
    linkedin: int = 0
    company: int = 0


class EligibleRow(BaseModel):
    """Represents a single enrichable record."""

    object: str = Field(..., alias="object")
    record_id: str
    name: str
    account: Optional[str] = None
    fillable: List[str] = Field(default_factory=list)
    est_credits: int = 1
    reason: str = "Enrichment available"
    confidence: Optional[str] = None
    likelihood: Optional[float] = None
    current: Dict[str, Optional[str]] = Field(default_factory=dict)
    enriched: Dict[str, Optional[str]] = Field(default_factory=dict)
    source: str = "FoundryMatch Enrichment"

    class Config:
        populate_by_name = True


class EligibleResponse(BaseModel):
    """Response payload for eligibility endpoint."""

    credits: EligibleCredits
    summary: EligibleSummary
    rows: List[EligibleRow] = Field(default_factory=list)
    correlation_id: str
    meta: Dict[str, Any] = Field(default_factory=dict)


class AppendRow(BaseModel):
    """Row requested by the client for append."""

    object: str = Field(..., alias="object")
    record_id: str

    class Config:
        populate_by_name = True


class AppendRequest(BaseModel):
    """Append request body."""

    rows: List[AppendRow]
    sheet_name: Optional[str] = None
    idempotency_key: Optional[str] = None
    account_id: Optional[str] = None
    estimated_credits: Optional[int] = None


class AppendRowResult(BaseModel):
    """Detailed record returned after enrichment."""

    object: str = Field(..., alias="object")
    record_id: str
    name: str
    account: Optional[str] = None
    fillable: List[str] = Field(default_factory=list)
    current: Dict[str, Optional[str]] = Field(default_factory=dict)
    enriched: Dict[str, Optional[str]] = Field(default_factory=dict)
    confidence: Optional[str] = None
    likelihood: Optional[float] = None
    credits_used: int = 1
    reason: str = "Enrichment appended"
    source: str = "FoundryMatch Enrichment"

    class Config:
        populate_by_name = True


class AppendResponse(BaseModel):
    """Response returned to Sheets after append."""

    ok: bool = True
    written: int = 0
    skipped: int = 0
    credits_used: int = 0
    remaining: int = 0
    correlation_id: str
    appended_at: str
    rows: List[AppendRowResult] = Field(default_factory=list)
    usage: Optional[Dict[str, Any]] = None
    errors: List[Dict[str, Any]] = Field(default_factory=list)
    message: Optional[str] = None
    chunk_errors: List[Dict[str, Any]] = Field(default_factory=list)
    reload_url: Optional[str] = None


# ---------------------------------------------------------------------------
# Salesforce Fill models
# ---------------------------------------------------------------------------

FillPolicyLiteral = Literal["empty_only", "skip_if_present", "force"]
DEFAULT_FILL_POLICY: FillPolicyLiteral = "empty_only"

FILL_POLICY_LABELS: Dict[FillPolicyLiteral, str] = {
    "empty_only": "Update empty fields only",
    "skip_if_present": "Skip if field already has data",
    "force": "Always overwrite",
}


class FillRowRequest(BaseModel):
    """Row requested by the client for Salesforce fill."""

    object: str = Field(..., alias="object")
    record_id: str

    class Config:
        populate_by_name = True


class FillRequest(BaseModel):
    """Fill operation request payload."""

    account_id: Optional[str] = None
    rows: List[FillRowRequest]
    policy: FillPolicyLiteral = DEFAULT_FILL_POLICY
    dry_run: bool = True
    idempotency_key: Optional[str] = None
    estimated_credits: Optional[int] = None


class FillChange(BaseModel):
    """Single field change for a Salesforce record."""

    field: str
    from_value: Optional[str] = Field(None, alias="from")
    to_value: Optional[str] = Field(None, alias="to")

    class Config:
        populate_by_name = True


class FillRowResult(BaseModel):
    """Per-record result for fill preview/commit."""

    object: str
    record_id: str
    name: Optional[str] = None
    changes: List[FillChange] = Field(default_factory=list)
    status: Literal["preview", "updated", "skipped", "error"] = "preview"
    reason: Optional[str] = None
    credits_used: int = 0


class FillCredits(BaseModel):
    """Credit summary block for fill responses."""

    estimate: int = 0
    used: int = 0
    remaining: Optional[int] = None
    reload_url: Optional[str] = None


class FillSummary(BaseModel):
    """High-level summary for UI preview bar."""

    count: int = 0
    fields: List[str] = Field(default_factory=list)
    policy: FillPolicyLiteral = DEFAULT_FILL_POLICY


class FillResponse(BaseModel):
    """Response payload for Salesforce fill endpoint."""

    ok: bool = True
    dry_run: bool = True
    summary: FillSummary
    rows: List[FillRowResult] = Field(default_factory=list)
    credits: FillCredits
    correlation_id: str
    message: Optional[str] = None
    usage: Optional[Dict[str, Any]] = None


# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# Endpoints
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━


@router.api_route("/eligible", methods=["GET", "POST"], response_model=EligibleResponse)
async def get_enrichment_eligibility(
    raw_request: Request,
    objects: Optional[str] = None,
    skip_b2c: bool = True,
    limit: int = ELIGIBLE_DEFAULT_LIMIT,
    filters_json: Optional[str] = Query(None, alias="filters"),
    db: AsyncSession = Depends(get_session),
    api_key: str = Depends(verify_api_key),
) -> EligibleResponse:
    """Return eligible Contacts/Leads that can be enriched immediately."""
    try:
        _ensure_enrichment_enabled()

        tracker = EnrichmentUsageTracker(db)
        account_id, _user_email = tracker.resolve_account(raw_request)

        body_payload: Dict[str, Any] = {}
        if raw_request.method.upper() == "POST":
            try:
                body_payload = await raw_request.json()
                if not isinstance(body_payload, dict):
                    body_payload = {}
            except Exception:
                body_payload = {}

            body_objects = body_payload.get("objects")
            if isinstance(body_objects, list):
                objects = ",".join(str(obj) for obj in body_objects if obj)
            elif isinstance(body_objects, str):
                objects = body_objects

            if "skip_b2c" in body_payload:
                skip_b2c = bool(body_payload["skip_b2c"])
            if "limit" in body_payload:
                try:
                    limit = int(body_payload["limit"])
                except (TypeError, ValueError):
                    pass
        else:
            body_payload = {}

        filters_payload: Dict[str, Any] = {}
        raw_filters = body_payload.get("filters")
        if isinstance(raw_filters, dict):
            filters_payload = raw_filters
        elif filters_json:
            try:
                parsed_filters = json.loads(filters_json)
                if isinstance(parsed_filters, dict):
                    extracted = parsed_filters.get("filters", parsed_filters)
                    if isinstance(extracted, dict):
                        filters_payload = extracted
            except json.JSONDecodeError as exc:
                raise HTTPException(
                    status_code=400, detail=f"Invalid filters payload: {exc}"
                ) from exc

        object_tokens = [
            token.strip()
            for token in (objects or "Contact,Lead").split(",")
            if token.strip()
        ]
        normalized_objects: List[str] = []
        for token in object_tokens:
            normalized = token.capitalize()
            if normalized not in {"Contact", "Lead"}:
                logger.debug(
                    "Skipping unsupported object '%s' in eligible request", token
                )
                continue
            if normalized not in normalized_objects:
                normalized_objects.append(normalized)
        if not normalized_objects:
            normalized_objects = ["Contact"]

        limit = max(1, min(ELIGIBLE_MAX_LIMIT, int(limit)))
        per_object_limit = max(1, math.ceil(limit / len(normalized_objects)))

        gateway = await SalesforceGateway.for_tenant(account_id, db)
        custom_field_map = await _load_custom_field_map(db, account_id)
        scanner = MissingFieldScanner(
            gateway=gateway, custom_field_map=custom_field_map
        )
        coordinator = ContactEnrichmentCoordinator(pdl_adapter=PeopleDataLabsAdapter())
        salesforce_status = await gateway.get_status()
        salesforce_base_url = (
            salesforce_status.get("instance_url")
            if isinstance(salesforce_status, dict)
            else None
        )
        owner_id = _resolve_salesforce_owner_id(gateway)
        filters_by_object: Dict[str, List[str]] = {}
        skipped_filters: Dict[str, List[str]] = {}
        for sfdc_object in normalized_objects:
            filters_for_object = _filters_to_conditions(
                filters_payload, sfdc_object, owner_id
            )
            if filters_for_object:
                filters_by_object[sfdc_object] = filters_for_object

        linkedin_missing_objects: Set[str] = set()
        for sfdc_object in normalized_objects:
            linkedin_field = scanner._get_field_name(sfdc_object, "linkedin_url")
            if linkedin_field.lower() == "linkedin_url":
                linkedin_missing_objects.add(sfdc_object)

        candidates: List[MissingFieldCandidate] = []
        seen_ids: Set[str] = set()

        for sfdc_object in normalized_objects:
            extra_conditions = filters_by_object.get(sfdc_object)
            try:
                scanned = await scanner.scan(
                    account_id=account_id,
                    sfdc_object=sfdc_object,
                    limit=per_object_limit,
                    filter_b2c=skip_b2c,
                    extra_conditions=extra_conditions,
                )
            except SalesforceFieldAccessError as field_error:
                if extra_conditions:
                    logger.warning(
                        "Filters for %s reference unavailable fields; retrying without filters: %s",
                        sfdc_object,
                        field_error,
                    )
                    skipped_filters[sfdc_object] = extra_conditions
                    scanned = await scanner.scan(
                        account_id=account_id,
                        sfdc_object=sfdc_object,
                        limit=per_object_limit,
                        filter_b2c=skip_b2c,
                        extra_conditions=None,
                    )
                else:
                    raise
            except Exception as scan_error:
                error_msg = str(scan_error).lower()
                if (
                    "no such field" in error_msg
                    or "restricted" in error_msg
                    or "field access" in error_msg
                ):
                    logger.warning(
                        "Filter fields not available in Salesforce for %s, retrying without filters: %s",
                        sfdc_object,
                        scan_error,
                    )
                    # Retry without custom filters
                    scanned = await scanner.scan(
                        account_id=account_id,
                        sfdc_object=sfdc_object,
                        limit=per_object_limit,
                        filter_b2c=skip_b2c,
                        extra_conditions=None,
                    )
                else:
                    raise
            for candidate in scanned:
                if candidate.sfdc_id in seen_ids:
                    continue
                seen_ids.add(candidate.sfdc_id)
                candidates.append(candidate)

        ordered_candidates = candidates[:limit]
        candidate_index = {
            candidate.sfdc_id: idx for idx, candidate in enumerate(ordered_candidates)
        }

        rows: List[EligibleRow] = []
        stats = None

        if linkedin_missing_objects:
            logger.debug(
                "LinkedIn mapping not configured for account=%s objects=%s; linkedin fills may be absent.",
                account_id,
                sorted(linkedin_missing_objects),
            )

        if ordered_candidates:
            matches, stats = await coordinator.enrich_batch(ordered_candidates)
            for match in matches:
                candidate_id = match.candidate.sfdc_id
                if candidate_id not in candidate_index:
                    continue
                payload = _compute_match_payload(match)
                fillable = payload["fillable"]
                if not fillable:
                    continue
                rows.append(
                    EligibleRow(
                        object=match.candidate.sfdc_object,
                        record_id=candidate_id,
                        name=_extract_display_name(match.candidate),
                        account=getattr(match.candidate, "company_name", None),
                        fillable=fillable,
                        est_credits=1,
                        reason=_reason_for_match(match, fillable),
                        confidence=match.confidence,
                        likelihood=match.pdl_likelihood,
                        current=payload["current"],
                        enriched=payload["enriched"],
                    )
                )
            rows.sort(key=lambda row: candidate_index.get(row.record_id, 0))

        balances = await tracker.balances(account_id)
        remaining = int(balances.get("allowance_remaining", 0) or 0) + int(
            balances.get("pack_remaining", 0) or 0
        )
        correlation_id = _build_correlation_id("ce")

        response = EligibleResponse(
            credits=EligibleCredits(
                remaining=remaining,
                estimate=len(rows),
                reload_url=_credit_reload_url(),
            ),
            summary=_summarize_rows(rows),
            rows=rows,
            correlation_id=correlation_id,
            meta={
                "objects": normalized_objects,
                "scanned": len(ordered_candidates),
                "matches": len(rows),
                "pdl_requests": stats.pdl_requests_made if stats else 0,
                "skip_b2c": bool(skip_b2c),
                "filters_applied": filters_payload,
                "filters_skipped": skipped_filters,
                "salesforce_base_url": salesforce_base_url,
                "filters_rendered": filters_by_object,
            },
        )

        track_event(
            "enrichment.eligible",
            {
                "account_id": account_id,
                "objects": normalized_objects,
                "scanned": len(ordered_candidates),
                "matches": len(rows),
                "credits_estimate": len(rows),
                "skip_b2c": bool(skip_b2c),
            },
        )
        return response

    except HTTPException:
        raise
    except Exception as exc:
        logger.error("Eligibility lookup failed: %s", exc, exc_info=True)
        raise HTTPException(status_code=500, detail=str(exc))


@router.post("/append", response_model=AppendResponse)
async def append_enrichment_rows(
    request: AppendRequest,
    raw_request: Request,
    db: AsyncSession = Depends(get_session),
    api_key: str = Depends(verify_api_key),
    idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"),
) -> AppendResponse:
    """Return append-ready enrichment rows while metering credits."""
    try:
        _ensure_enrichment_enabled()

        if not request.rows:
            raise HTTPException(status_code=400, detail="rows cannot be empty.")

        tracker = EnrichmentUsageTracker(db)
        account_id, user_email = tracker.resolve_account(raw_request)

        state = getattr(raw_request, "state", None)
        correlation_id = getattr(state, "correlation_id", None)
        header_correlation = (
            raw_request.headers.get("X-Correlation-Id")
            if hasattr(raw_request, "headers")
            else None
        )
        if header_correlation:
            correlation_id = header_correlation
        if not correlation_id:
            correlation_id = _build_correlation_id("append")
        if state is not None:
            try:
                state.correlation_id = correlation_id
            except AttributeError:
                pass
        reload_url = _credit_reload_url()

        if request.account_id and request.account_id != account_id:
            raise HTTPException(
                status_code=403,
                detail="Authenticated account does not match request payload.",
            )

        rows_by_object: Dict[str, List[str]] = {}
        requested_pairs: List[Dict[str, str]] = []

        for row in request.rows:
            obj = (row.object or "").strip()
            record_id = row.record_id.strip()
            if not obj or not record_id:
                continue
            normalized = obj.capitalize()
            if normalized not in {"Contact", "Lead"}:
                logger.debug("Skipping unsupported object '%s' in append request", obj)
                continue
            rows_by_object.setdefault(normalized, [])
            if record_id not in rows_by_object[normalized]:
                rows_by_object[normalized].append(record_id)
                requested_pairs.append({"object": normalized, "record_id": record_id})

        if not rows_by_object:
            raise HTTPException(status_code=400, detail="No valid rows to append.")

        estimated_credits = request.estimated_credits
        if estimated_credits is None or estimated_credits <= 0:
            estimated_credits = len(requested_pairs) or len(request.rows)
        estimated_credits = int(estimated_credits)

        balances_snapshot = await tracker.balances(account_id)
        available_balance = int(
            balances_snapshot.get("allowance_remaining", 0) or 0
        ) + int(balances_snapshot.get("pack_remaining", 0) or 0)
        if estimated_credits > available_balance:
            detail = {
                "code": "INSUFFICIENT_CREDITS",
                "message": f"Estimated {estimated_credits} credits exceeds available {available_balance}.",
                "available": available_balance,
                "estimated": estimated_credits,
            }
            if reload_url:
                detail["reload_url"] = reload_url
            if correlation_id:
                detail["correlation_id"] = correlation_id
            raise HTTPException(
                status_code=status.HTTP_402_PAYMENT_REQUIRED,
                detail=detail,
            )

        gateway = await SalesforceGateway.for_tenant(account_id, db)
        custom_field_map = await _load_custom_field_map(db, account_id)
        scanner = MissingFieldScanner(
            gateway=gateway, custom_field_map=custom_field_map
        )
        coordinator = ContactEnrichmentCoordinator(pdl_adapter=PeopleDataLabsAdapter())

        linkedin_missing_objects: Set[str] = set()
        for sfdc_object in rows_by_object:
            linkedin_field = scanner._get_field_name(sfdc_object, "linkedin_url")
            if linkedin_field.lower() == "linkedin_url":
                linkedin_missing_objects.add(sfdc_object)

        candidates: List[MissingFieldCandidate] = []
        candidate_failures: Dict[Tuple[str, str], str] = {}

        for sfdc_object, record_ids in rows_by_object.items():
            fields = _select_fields_for_query(scanner, sfdc_object)
            select_clause = ", ".join(["Id"] + fields)
            id_clause = ",".join(f"'{rid}'" for rid in record_ids)
            query = (
                f"SELECT {select_clause} FROM {sfdc_object} WHERE Id IN ({id_clause})"
            )
            result = await gateway.soql(query)
            if result.get("error"):
                raise HTTPException(status_code=500, detail=result["error"])
            records = result.get("records", [])
            returned_ids = set()
            for record in records:
                record_id = record.get("Id")
                if record_id:
                    returned_ids.add(record_id)
                candidate = scanner._record_to_candidate(record, sfdc_object)
                if candidate:
                    candidates.append(candidate)
                elif record_id:
                    candidate_failures[(sfdc_object, record_id)] = (
                        "Record already has fillable data or is missing required fields."
                    )
            for record_id in record_ids:
                if record_id not in returned_ids:
                    candidate_failures[(sfdc_object, record_id)] = (
                        "Record not found in Salesforce query results."
                    )

        if linkedin_missing_objects:
            logger.debug(
                "LinkedIn mapping not configured for account=%s objects=%s; linkedin fills may be absent.",
                account_id,
                sorted(linkedin_missing_objects),
            )

        if not candidates:
            balances = await tracker.balances(account_id)
            remaining = int(balances.get("allowance_remaining", 0) or 0) + int(
                balances.get("pack_remaining", 0) or 0
            )
            requested_total = len(requested_pairs)
            error_items = [
                {"object": key[0], "record_id": key[1], "reason": reason}
                for key, reason in candidate_failures.items()
            ]
            return AppendResponse(
                ok=True,
                written=0,
                skipped=requested_total,
                credits_used=0,
                remaining=remaining,
                correlation_id=correlation_id,
                appended_at=_utc_now_iso(),
                rows=[],
                usage=None,
                errors=error_items,
                message="No eligible records found for append.",
                chunk_errors=[],
                reload_url=reload_url,
            )

        estimate = coordinator.estimate_api_requests(candidates)
        idem_key = (
            idempotency_key
            or request.idempotency_key
            or _ensure_identifier(None, "append")
        )
        payload_hash = _hash_payload(
            {
                "account_id": account_id,
                "rows": sorted(
                    requested_pairs,
                    key=lambda item: (item["object"], item["record_id"]),
                ),
                "sheet_name": request.sheet_name,
            }
        )

        preflight = await tracker.preflight(
            account_id=account_id,
            estimated_requests=estimate["requests_needed"],
            idempotency_key=idem_key,
            request_hash=payload_hash,
            reserve=True,
            job_id=None,
            chunk_id=None,
        )

        if preflight.get("replay"):
            balances = await tracker.balances(account_id)
            remaining = int(balances.get("allowance_remaining", 0) or 0) + int(
                balances.get("pack_remaining", 0) or 0
            )
            return AppendResponse(
                ok=True,
                written=0,
                skipped=0,
                credits_used=0,
                remaining=remaining,
                correlation_id=correlation_id,
                appended_at=_utc_now_iso(),
                rows=[],
                usage=preflight,
                message="Append request already processed.",
                chunk_errors=[],
                reload_url=reload_url,
            )

        if not preflight.get("allowed", False):
            status_code = int(preflight.get("status_code", 429))
            raise HTTPException(status_code=status_code, detail=preflight)

        heartbeat_context = (
            _reservation_heartbeat(
                account_id, idem_key, interval_seconds=HEARTBEAT_SECONDS
            )
            if preflight.get("reservation_id")
            else _noop_async_context()
        )

        stats = None
        finalize_usage: Optional[Dict[str, Any]] = None
        rows_payload: List[AppendRowResult] = []
        error_items: List[Dict[str, Any]] = []

        try:
            async with heartbeat_context:
                matches, stats = await coordinator.enrich_batch(candidates)

                requested_order = {
                    (row["object"], row["record_id"]): idx
                    for idx, row in enumerate(requested_pairs)
                }

                enriched_keys: Set[Tuple[str, str]] = set()

                for match in matches:
                    key = (match.candidate.sfdc_object, match.candidate.sfdc_id)
                    if key not in requested_order:
                        continue
                    payload = _compute_match_payload(match)
                    fillable = payload["fillable"]
                    if not fillable:
                        candidate_failures.setdefault(
                            key, "Provider returned no new fields."
                        )
                        continue
                    enriched_keys.add(key)
                    rows_payload.append(
                        AppendRowResult(
                            object=match.candidate.sfdc_object,
                            record_id=match.candidate.sfdc_id,
                            name=_extract_display_name(match.candidate),
                            account=getattr(match.candidate, "company_name", None),
                            fillable=fillable,
                            current=payload["current"],
                            enriched=payload["enriched"],
                            confidence=match.confidence,
                            likelihood=match.pdl_likelihood,
                            credits_used=1,
                            reason=_reason_for_match(match, fillable),
                        )
                    )

                rows_payload.sort(
                    key=lambda row: requested_order.get((row.object, row.record_id), 0)
                )

                for req in requested_pairs:
                    key = (req["object"], req["record_id"])
                    if key not in enriched_keys:
                        error_reason = (
                            candidate_failures.get(key)
                            or "No enrichment available for this record."
                        )
                        error_items.append(
                            {
                                "object": key[0],
                                "record_id": key[1],
                                "reason": error_reason,
                            }
                        )

                credits_used = stats.pdl_requests_made if stats else len(rows_payload)
                fields_filled = sum(len(row.fillable) for row in rows_payload)

                finalize_usage = await tracker.finalize(
                    account_id=account_id,
                    user_email=user_email,
                    idempotency_key=idem_key,
                    request_hash=payload_hash,
                    actual_requests=credits_used,
                    matches_found=len(rows_payload),
                    operation="append",
                    enrichment_type="sheets",
                    provider="pdl",
                    job_id=None,
                    chunk_id=None,
                    meta={
                        "records_requested": len(requested_pairs),
                        "records_enriched": len(rows_payload),
                        "fields_filled": fields_filled,
                        "sheet_name": request.sheet_name,
                    },
                )

        except Exception as exc:
            if preflight.get("reservation_id"):
                try:
                    await tracker.release(account_id, idem_key)
                except Exception:
                    logger.exception(
                        "Failed to release reservation after append error (account=%s)",
                        account_id,
                    )
            logger.error("Append enrichment failed: %s", exc, exc_info=True)
            raise

        balances = await tracker.balances(account_id)
        remaining = int(balances.get("allowance_remaining", 0) or 0) + int(
            balances.get("pack_remaining", 0) or 0
        )

        requested_total = len(requested_pairs)
        written = len(rows_payload)
        credits_used = stats.pdl_requests_made if stats else written

        response = AppendResponse(
            ok=True,
            written=written,
            skipped=max(0, requested_total - written),
            credits_used=credits_used,
            remaining=remaining,
            correlation_id=correlation_id,
            appended_at=_utc_now_iso(),
            rows=rows_payload,
            usage=finalize_usage,
            errors=error_items,
            message=f"{written} enriched record(s) ready for append."
            if written
            else "No fillable fields detected.",
            chunk_errors=[],
            reload_url=reload_url,
        )

        track_event(
            "enrichment.append",
            {
                "account_id": account_id,
                "records_requested": requested_total,
                "records_enriched": written,
                "credits_used": credits_used,
                "sheet_name": request.sheet_name,
                "correlation_id": correlation_id,
            },
        )

        return response

    except HTTPException:
        raise
    except Exception as exc:
        logger.error("Append request failed: %s", exc, exc_info=True)
        raise HTTPException(status_code=500, detail=str(exc))


@router.post("/fill", response_model=FillResponse)
async def fill_salesforce(
    request: FillRequest,
    raw_request: Request,
    db: AsyncSession = Depends(get_session),
    api_key: str = Depends(verify_api_key),
    idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"),
) -> FillResponse:
    """Preview or apply Salesforce fill updates based on enrichment data."""
    try:
        _ensure_enrichment_enabled()

        if not request.rows:
            raise HTTPException(status_code=400, detail="rows cannot be empty.")

        tracker = EnrichmentUsageTracker(db)
        account_id, user_email = tracker.resolve_account(raw_request)

        state = getattr(raw_request, "state", None)
        correlation_id = getattr(state, "correlation_id", None)
        header_correlation = (
            raw_request.headers.get("X-Correlation-Id")
            if hasattr(raw_request, "headers")
            else None
        )
        if header_correlation:
            correlation_id = header_correlation
        if not correlation_id:
            correlation_id = _build_correlation_id("fill")
        if state is not None:
            try:
                state.correlation_id = correlation_id
            except AttributeError:
                pass
        reload_url = _credit_reload_url()

        if request.account_id and request.account_id != account_id:
            raise HTTPException(
                status_code=403,
                detail="Authenticated account does not match request payload.",
            )

        policy_value = (request.policy or DEFAULT_FILL_POLICY).strip().lower()
        if policy_value not in FILL_POLICY_LABELS:
            raise HTTPException(
                status_code=400,
                detail=f"Unsupported fill policy '{request.policy}'.",
            )
        policy = cast(FillPolicyLiteral, policy_value)
        dry_run = bool(request.dry_run)

        rows_by_object: Dict[str, List[str]] = {}
        requested_pairs: List[Dict[str, str]] = []

        for row in request.rows:
            obj = (row.object or "").strip()
            record_id = (row.record_id or "").strip()
            if not obj or not record_id:
                continue
            normalized = obj.capitalize()
            if normalized not in {"Contact", "Lead"}:
                logger.debug("Skipping unsupported object '%s' in fill request", obj)
                continue
            rows_by_object.setdefault(normalized, [])
            if record_id not in rows_by_object[normalized]:
                rows_by_object[normalized].append(record_id)
                requested_pairs.append({"object": normalized, "record_id": record_id})

        if not rows_by_object:
            raise HTTPException(status_code=400, detail="No valid rows to fill.")

        estimated_credits = request.estimated_credits
        if estimated_credits is None or estimated_credits <= 0:
            estimated_credits = len(requested_pairs) or len(request.rows)
        estimated_credits = int(estimated_credits)

        balances_snapshot = await tracker.balances(account_id)
        available_balance = int(
            balances_snapshot.get("allowance_remaining", 0) or 0
        ) + int(balances_snapshot.get("pack_remaining", 0) or 0)
        if estimated_credits > available_balance:
            detail = {
                "code": "INSUFFICIENT_CREDITS",
                "message": f"Estimated {estimated_credits} credits exceeds available {available_balance}.",
                "available": available_balance,
                "estimated": estimated_credits,
            }
            if reload_url:
                detail["reload_url"] = reload_url
            if correlation_id:
                detail["correlation_id"] = correlation_id
            raise HTTPException(
                status_code=status.HTTP_402_PAYMENT_REQUIRED,
                detail=detail,
            )

        gateway = await SalesforceGateway.for_tenant(account_id, db)
        custom_field_map = await _load_custom_field_map(db, account_id)
        scanner = MissingFieldScanner(
            gateway=gateway, custom_field_map=custom_field_map
        )
        coordinator = ContactEnrichmentCoordinator(pdl_adapter=PeopleDataLabsAdapter())

        candidates: List[MissingFieldCandidate] = []
        candidate_context: Dict[Tuple[str, str], Dict[str, Any]] = {}
        candidate_failures: Dict[Tuple[str, str], str] = {}
        field_map_cache: Dict[str, Dict[str, str]] = {}

        for sfdc_object, record_ids in rows_by_object.items():
            fields = _select_fields_for_query(scanner, sfdc_object)
            select_clause = ", ".join(["Id"] + fields)
            id_clause = ",".join(f"'{rid}'" for rid in record_ids)
            query = (
                f"SELECT {select_clause} FROM {sfdc_object} WHERE Id IN ({id_clause})"
            )
            result = await gateway.soql(query)
            if result.get("error"):
                raise HTTPException(status_code=500, detail=result["error"])
            records = result.get("records", [])
            returned_ids = set()
            for record in records:
                record_id = record.get("Id")
                if not record_id:
                    continue
                returned_ids.add(record_id)
                candidate = _record_to_fill_candidate(scanner, record, sfdc_object)
                if not candidate:
                    candidate_failures[(sfdc_object, record_id)] = (
                        "Record missing required name fields."
                    )
                    continue
                candidates.append(candidate)
                key = (sfdc_object, record_id)
                if sfdc_object not in field_map_cache:
                    field_map_cache[sfdc_object] = _build_field_map(
                        scanner, sfdc_object
                    )
                candidate_context[key] = {
                    "candidate": candidate,
                    "field_map": field_map_cache[sfdc_object],
                }
            for record_id in record_ids:
                if record_id not in returned_ids:
                    candidate_failures[(sfdc_object, record_id)] = (
                        "Record not found in Salesforce query results."
                    )

        estimate = coordinator.estimate_api_requests(candidates)
        requests_needed = int(estimate.get("requests_needed", len(candidates)))

        idem_key = (
            idempotency_key
            or request.idempotency_key
            or _ensure_identifier(None, "fill")
        )
        payload_hash = _hash_payload(
            {
                "account_id": account_id,
                "rows": sorted(
                    requested_pairs,
                    key=lambda item: (item["object"], item["record_id"]),
                ),
                "policy": policy,
                "dry_run": dry_run,
            }
        )

        preflight = None
        if requests_needed > 0:
            preflight = await tracker.preflight(
                account_id=account_id,
                estimated_requests=requests_needed,
                idempotency_key=idem_key,
                request_hash=payload_hash,
                reserve=not dry_run,
                job_id=None,
                chunk_id=None,
            )

            if preflight.get("replay"):
                remaining_snapshot = await tracker.balances(account_id)
                remaining = int(
                    remaining_snapshot.get("allowance_remaining", 0) or 0
                ) + int(remaining_snapshot.get("pack_remaining", 0) or 0)
                response = FillResponse(
                    ok=True,
                    dry_run=dry_run,
                    summary=FillSummary(count=0, fields=[], policy=policy),
                    rows=[],
                    credits=FillCredits(
                        estimate=estimated_credits,
                        used=0,
                        remaining=remaining,
                        reload_url=reload_url,
                    ),
                    correlation_id=correlation_id,
                    message="Fill request already processed.",
                    usage=preflight,
                )
                return response

            if not preflight.get("allowed", False):
                status_code = int(preflight.get("status_code", 429))
                raise HTTPException(status_code=status_code, detail=preflight)

        track_event(
            "fill.preview.requested" if dry_run else "fill.commit.requested",
            {
                "account_id": account_id,
                "correlation_id": correlation_id,
                "records_requested": len(requested_pairs),
                "policy": policy,
                "dry_run": dry_run,
                "estimated_credits": estimated_credits,
                "estimated_requests": requests_needed,
            },
        )

        heartbeat_context = (
            _reservation_heartbeat(
                account_id, idem_key, interval_seconds=HEARTBEAT_SECONDS
            )
            if (preflight and preflight.get("reservation_id") and not dry_run)
            else _noop_async_context()
        )

        stats = None
        matches: List[EnrichmentMatch] = []
        rows_payload: List[Dict[str, Any]] = []
        row_index_by_key: Dict[Tuple[str, str], int] = {}
        update_batches: Dict[str, List[Dict[str, Any]]] = {}
        summary_fields: Set[str] = set()
        rows_with_changes = 0
        bulk_errors: List[Dict[str, Any]] = []

        try:
            async with heartbeat_context:
                if candidates:
                    matches, stats = await coordinator.enrich_batch(candidates)
        except Exception as exc:
            if not dry_run and preflight and preflight.get("reservation_id"):
                try:
                    await tracker.release(account_id, idem_key)
                except Exception:
                    logger.exception(
                        "Failed to release reservation after fill error (account=%s)",
                        account_id,
                    )
            logger.error("Fill enrichment failed: %s", exc, exc_info=True)
            raise

        matches_by_key: Dict[Tuple[str, str], EnrichmentMatch] = {}
        for match in matches or []:
            key = (match.candidate.sfdc_object, match.candidate.sfdc_id)
            if key not in matches_by_key:
                matches_by_key[key] = match

        for pair in requested_pairs:
            key = (pair["object"], pair["record_id"])
            context = candidate_context.get(key)
            candidate = context["candidate"] if context else None
            field_map = context["field_map"] if context else {}
            match = matches_by_key.get(key)
            reason = "ok"
            status_value = "preview" if dry_run else "updated"
            credits_used_row = 0
            row_summary_fields: Set[str] = set()
            update_record: Dict[str, Any] = {}
            changes_for_row: List[Dict[str, Any]] = []

            if not candidate:
                reason = candidate_failures.get(key, "record_not_available")
                status_value = "skipped"
            elif not match:
                reason = "no_candidate"
                status_value = "skipped"
            else:
                payload = _compute_match_payload(match)
                changes_for_row, update_record, row_summary_fields, skip_reason = (
                    _derive_fill_changes(
                        payload,
                        policy,
                        field_map,
                    )
                )

                if skip_reason:
                    reason = skip_reason
                    status_value = "skipped"
                    update_record = {}
                    changes_for_row = []
                elif not changes_for_row:
                    reason = "no_changes"
                    status_value = "skipped"
                    update_record = {}
                else:
                    rows_with_changes += 1
                    summary_fields.update(row_summary_fields)
                    if not dry_run:
                        credits_used_row = 1
                        update_batches.setdefault(pair["object"], []).append(
                            {"Id": pair["record_id"], **update_record}
                        )

            display_name = _extract_display_name(candidate) if candidate else key[1]

            row_payload = {
                "object": pair["object"],
                "record_id": pair["record_id"],
                "name": display_name,
                "changes": changes_for_row,
                "status": status_value,
                "reason": reason,
                "credits_used": credits_used_row if not dry_run else 0,
                "summary_fields": list(row_summary_fields)
                if row_summary_fields
                else [],
            }
            rows_payload.append(row_payload)
            row_index_by_key[key] = len(rows_payload) - 1

        if not dry_run and update_batches:
            for sfdc_object, records in update_batches.items():
                if not records:
                    continue
                try:
                    result = await gateway.bulk_update(sfdc_object, records)
                except Exception as exc:
                    logger.exception("Bulk update failed for %s: %s", sfdc_object, exc)
                    for record in records:
                        key = (sfdc_object, record.get("Id"))
                        idx = row_index_by_key.get(key)
                        if idx is not None:
                            rows_payload[idx]["status"] = "error"
                            rows_payload[idx]["reason"] = "salesforce_error"
                            rows_payload[idx]["credits_used"] = 0
                    bulk_errors.append(
                        {
                            "object": sfdc_object,
                            "error": str(exc),
                            "ids": [record.get("Id") for record in records],
                        }
                    )
                    continue

                success_ids = set()
                failure_messages: Dict[str, str] = {}

                if result.get("error"):
                    message = result["error"]
                    for record in records:
                        record_id = record.get("Id")
                        if not record_id:
                            continue
                        failure_messages[record_id] = message or "Salesforce bulk error"
                else:
                    for entry in result.get("successfulResults", []) or []:
                        rec_id = entry.get("id")
                        if rec_id:
                            success_ids.add(rec_id)
                    for entry in result.get("failedResults", []) or []:
                        rec_id = entry.get("id")
                        errors = (
                            entry.get("errors")
                            if isinstance(entry.get("errors"), list)
                            else None
                        )
                        message = ""
                        if errors:
                            message = errors[0].get("message", "")
                        failure_messages[rec_id or ""] = (
                            message or "Salesforce update failed"
                        )

                for record in records:
                    rec_id = record.get("Id")
                    if not rec_id:
                        continue
                    key = (sfdc_object, rec_id)
                    idx = row_index_by_key.get(key)
                    if idx is None:
                        continue
                    if rec_id in success_ids:
                        rows_payload[idx]["status"] = "updated"
                        rows_payload[idx]["reason"] = "ok"
                        rows_payload[idx]["credits_used"] = 1
                    else:
                        rows_payload[idx]["status"] = "error"
                        rows_payload[idx]["reason"] = failure_messages.get(
                            rec_id, "salesforce_error"
                        )
                        rows_payload[idx]["credits_used"] = 0
                        bulk_errors.append(
                            {
                                "object": sfdc_object,
                                "record_id": rec_id,
                                "error": failure_messages.get(
                                    rec_id, "salesforce_error"
                                ),
                            }
                        )

        if not dry_run:
            summary_fields = set()
            for row in rows_payload:
                if row.get("status") != "updated":
                    continue
                for field_key in row.get("summary_fields") or []:
                    if field_key:
                        summary_fields.add(field_key)

        credits_used_total = (
            0 if dry_run else (stats.pdl_requests_made if stats else requests_needed)
        )
        updated_count = sum(1 for row in rows_payload if row["status"] == "updated")
        effective_summary_count = rows_with_changes if dry_run else updated_count
        summary_list = _ordered_summary_fields(summary_fields)

        if dry_run:
            message = (
                f"Preview ready — {rows_with_changes} record(s) would change."
                if rows_with_changes
                else "Dry run found no fields to update."
            )
        else:
            if updated_count:
                message = f"{updated_count} record(s) updated in Salesforce."
            else:
                message = "No Salesforce updates were applied."

        if bulk_errors:
            track_event(
                "fill.commit.chunk_failed",
                {
                    "account_id": account_id,
                    "correlation_id": correlation_id,
                    "errors": len(bulk_errors),
                },
            )

        usage_result = None
        if not dry_run and requests_needed > 0:
            try:
                usage_result = await tracker.finalize(
                    account_id=account_id,
                    user_email=user_email,
                    idempotency_key=idem_key,
                    request_hash=payload_hash,
                    actual_requests=credits_used_total,
                    matches_found=updated_count,
                    operation="fill",
                    enrichment_type="sheets",
                    provider="pdl",
                    job_id=None,
                    chunk_id=None,
                    meta={
                        "records_requested": len(requested_pairs),
                        "records_updated": updated_count,
                        "policy": policy,
                        "bulk_errors": len(bulk_errors),
                    },
                )
            except Exception as exc:
                logger.exception("Failed to finalize fill usage: %s", exc)
                if preflight and preflight.get("reservation_id"):
                    try:
                        await tracker.release(account_id, idem_key)
                    except Exception:
                        logger.exception(
                            "Failed to release reservation after finalize error (account=%s)",
                            account_id,
                        )
                raise

        if dry_run:
            remaining = available_balance
        else:
            balances = await tracker.balances(account_id)
            remaining = int(balances.get("allowance_remaining", 0) or 0) + int(
                balances.get("pack_remaining", 0) or 0
            )

        response = FillResponse(
            ok=True,
            dry_run=dry_run,
            summary=FillSummary(
                count=effective_summary_count, fields=summary_list, policy=policy
            ),
            rows=[
                FillRowResult(**{k: v for k, v in row.items() if k != "summary_fields"})
                for row in rows_payload
            ],
            credits=FillCredits(
                estimate=estimated_credits,
                used=0 if dry_run else credits_used_total,
                remaining=remaining,
                reload_url=reload_url,
            ),
            correlation_id=correlation_id,
            message=message,
            usage=usage_result,
        )

        track_event(
            "fill.preview.responded" if dry_run else "fill.commit.completed",
            {
                "account_id": account_id,
                "correlation_id": correlation_id,
                "records_with_changes": rows_with_changes,
                "records_updated": updated_count,
                "summary_fields": summary_list,
                "policy": policy,
                "dry_run": dry_run,
                "credits_used": 0 if dry_run else credits_used_total,
            },
        )

        return response

    except HTTPException:
        raise
    except Exception as exc:
        logger.error("Fill request failed: %s", exc, exc_info=True)
        raise HTTPException(status_code=500, detail=str(exc))


@router.post("/scan", response_model=ScanResponse)
async def scan_for_missing_fields(
    request: ScanRequest,
    db: AsyncSession = Depends(get_session),
    api_key: str = Depends(verify_api_key),
) -> ScanResponse:
    """
    Scan Salesforce for Contacts/Leads with missing enrichable fields.

    Returns list of candidates with:
        - Missing email, phone, mobile, or title
        - Valid company context for PDL matching
        - B2C domains filtered out (optional)
    """
    try:
        _ensure_enrichment_enabled()
        logger.info(
            f"Scanning {request.sfdc_object} for missing fields "
            f"(account={request.account_id}, limit={request.limit})"
        )

        # Initialize services
        gateway = await SalesforceGateway.for_tenant(request.account_id, db)
        custom_field_map = await _load_custom_field_map(db, request.account_id)
        scanner = MissingFieldScanner(
            gateway=gateway, custom_field_map=custom_field_map
        )

        # Scan for candidates
        candidates = await scanner.scan(
            account_id=request.account_id,
            sfdc_object=request.sfdc_object,
            limit=request.limit,
            filter_b2c=request.filter_b2c,
        )

        # Convert to response format
        candidate_responses = [
            CandidateResponse(
                sfdc_id=c.sfdc_id,
                sfdc_object=c.sfdc_object,
                first_name=c.first_name,
                last_name=c.last_name,
                full_name=c.full_name,
                company_name=c.company_name,
                company_domain=c.company_domain,
                email=c.email,
                phone=c.phone,
                mobile_phone=c.mobile_phone,
                title=c.title,
                linkedin_url=getattr(c, "linkedin_url", None),
                missing_fields=list(c.missing_fields),
                match_key=c.match_key,
            )
            for c in candidates
        ]

        # Track telemetry
        track_event(
            "contact_enrichment.scan",
            {
                "account_id": request.account_id,
                "object": request.sfdc_object,
                "candidates_found": len(candidates),
            },
        )

        return ScanResponse(
            ok=True,
            candidates=candidate_responses,
            total_scanned=request.limit,
            total_candidates=len(candidates),
            message=f"Found {len(candidates)} records with missing fields",
        )

    except Exception as e:
        logger.error(f"Scan failed: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))


@router.post("/preview", response_model=PreviewResponse)
async def preview_enrichments(
    request: PreviewRequest,
    raw_request: Request,
    db: AsyncSession = Depends(get_session),
    api_key: str = Depends(verify_api_key),
    idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"),
) -> PreviewResponse:
    """
    Preview the expected enrichment cost and limits without calling PDL.
    """
    try:
        _ensure_enrichment_enabled()

        tracker = EnrichmentUsageTracker(db)
        account_id, user_email = tracker.resolve_account(raw_request)
        if account_id != request.account_id:
            raise HTTPException(
                status_code=403,
                detail="Authenticated account does not match request payload.",
            )

        job_id = _ensure_identifier(request.job_id, "preview-job")
        chunk_id = _ensure_identifier(request.chunk_id, "preview-chunk")
        idem_key = idempotency_key or _ensure_identifier(None, "preview-key")

        logger.info(
            "Estimating enrichment preview for %s candidates (account=%s, job=%s, chunk=%s)",
            len(request.candidate_ids),
            account_id,
            job_id,
            chunk_id,
        )

        gateway = await SalesforceGateway.for_tenant(account_id, db)
        custom_field_map = await _load_custom_field_map(db, account_id)
        scanner = MissingFieldScanner(
            gateway=gateway, custom_field_map=custom_field_map
        )
        coordinator = ContactEnrichmentCoordinator(pdl_adapter=PeopleDataLabsAdapter())

        ids_str = "','".join(request.candidate_ids)
        query = (
            f"SELECT Id, FirstName, LastName, Name, Email, Phone, MobilePhone, Title, "
            f"Account.Name, Account.Website FROM {request.sfdc_object} WHERE Id IN ('{ids_str}')"
        )
        result = await gateway.soql(query)
        if result.get("error"):
            raise HTTPException(status_code=500, detail=result["error"])

        records = result.get("records", [])
        candidates: List[MissingFieldCandidate] = []
        for rec in records:
            candidate = scanner._record_to_candidate(rec, request.sfdc_object)
            if candidate:
                candidates.append(candidate)

        if not candidates:
            return PreviewResponse(
                ok=True,
                matches=[],
                stats={"candidates_submitted": 0, "estimated_requests": 0},
                usage_estimate={},
                message="No valid candidates found for enrichment.",
            )

        estimate = coordinator.estimate_api_requests(candidates)
        payload_hash = _hash_payload(
            {
                "account_id": account_id,
                "job_id": job_id,
                "chunk_id": chunk_id,
                "candidate_ids": sorted(request.candidate_ids),
                "sfdc_object": request.sfdc_object,
            }
        )

        preflight_key = idem_key
        preflight = await tracker.preflight(
            account_id=account_id,
            estimated_requests=estimate["requests_needed"],
            idempotency_key=preflight_key,
            request_hash=payload_hash,
            reserve=False,
            job_id=job_id,
            chunk_id=chunk_id,
        )

        stats = {
            "candidates_submitted": len(candidates),
            "unique_candidates": estimate["unique_candidates"],
            "cache_hits": estimate["cache_hits"],
            "estimated_requests": estimate["requests_needed"],
        }

        track_event(
            "contact_enrichment.preview_estimate",
            {
                "account_id": account_id,
                "candidates": len(candidates),
                "requests_needed": estimate["requests_needed"],
            },
        )

        return PreviewResponse(
            ok=True,
            matches=[],
            stats=stats,
            usage_estimate=preflight,
            message="Preview provides a cost estimate only. Run apply to fetch and review matches.",
        )

    except HTTPException:
        raise
    except Exception as e:
        logger.error("Preview failed: %s", e, exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))


@router.post("/apply", response_model=ApplyResponse)
async def apply_enrichments(
    request: ApplyRequest,
    raw_request: Request,
    db: AsyncSession = Depends(get_session),
    api_key: str = Depends(verify_api_key),
    idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"),
) -> ApplyResponse:
    """Apply enrichments to Salesforce and meter the usage ledger."""
    try:
        _ensure_enrichment_enabled()

        tracker = EnrichmentUsageTracker(db)
        account_id, user_email = tracker.resolve_account(raw_request)
        if account_id != request.account_id:
            raise HTTPException(
                status_code=403,
                detail="Authenticated account does not match request payload.",
            )

        job_id = _ensure_identifier(request.job_id, "apply-job")
        chunk_id = _ensure_identifier(request.chunk_id, "apply-chunk")
        idem_key = idempotency_key or _ensure_identifier(None, "apply-key")

        logger.info(
            "Applying %s enrichments (account=%s, job=%s, chunk=%s, dry_run=%s)",
            len(request.match_ids),
            account_id,
            job_id,
            chunk_id,
            request.dry_run,
        )

        gateway = await SalesforceGateway.for_tenant(account_id, db)
        custom_field_map = await _load_custom_field_map(db, account_id)
        scanner = MissingFieldScanner(
            gateway=gateway, custom_field_map=custom_field_map
        )
        coordinator = ContactEnrichmentCoordinator(pdl_adapter=PeopleDataLabsAdapter())
        writer = EnrichmentWriter(gateway=gateway, scanner=scanner)

        ids_str = "','".join(request.match_ids)
        query = (
            f"SELECT Id, FirstName, LastName, Name, Email, Phone, MobilePhone, Title, "
            f"Account.Name, Account.Website FROM {request.sfdc_object} WHERE Id IN ('{ids_str}')"
        )
        result = await gateway.soql(query)
        if result.get("error"):
            raise HTTPException(status_code=500, detail=result["error"])

        records = result.get("records", [])
        candidates: List[MissingFieldCandidate] = []
        for rec in records:
            candidate = scanner._record_to_candidate(rec, request.sfdc_object)
            if candidate:
                candidates.append(candidate)

        if not candidates:
            return ApplyResponse(
                ok=False,
                records_written=0,
                records_failed=0,
                fields_updated={},
                matches=[],
                preflight=None,
                usage=None,
                message="No valid candidates found.",
            )

        estimate = coordinator.estimate_api_requests(candidates)
        payload_hash = _hash_payload(
            {
                "account_id": account_id,
                "job_id": job_id,
                "chunk_id": chunk_id,
                "match_ids": sorted(request.match_ids),
                "dry_run": request.dry_run,
                "sfdc_object": request.sfdc_object,
            }
        )

        preflight = await tracker.preflight(
            account_id=account_id,
            estimated_requests=estimate["requests_needed"],
            idempotency_key=idem_key,
            request_hash=payload_hash,
            reserve=True,
            job_id=job_id,
            chunk_id=chunk_id,
        )

        if preflight.get("replay"):
            logger.info(
                "Apply replay detected for job=%s chunk=%s",
                request.job_id,
                request.chunk_id,
            )
            quota_snapshot = await tracker.balances(account_id)
            return ApplyResponse(
                ok=True,
                records_written=0,
                records_failed=0,
                fields_updated={},
                matches=[],
                preflight=preflight,
                usage=None,
                quota_snapshot=quota_snapshot,
                message="Chunk already finalized. Skipping duplicate apply.",
            )

        if not preflight.get("allowed", False):
            status_code = preflight.get("status_code", 429)
            raise HTTPException(status_code=status_code, detail=preflight)

        stats: Optional[Any] = None
        finalize_usage: Optional[Dict[str, Any]] = None
        reservation_active = preflight.get("reservation_id") is not None
        match_responses: List[EnrichmentMatchResponse] = []

        heartbeat_context = (
            _reservation_heartbeat(
                account_id, idem_key, interval_seconds=HEARTBEAT_SECONDS
            )
            if reservation_active
            else _noop_async_context()
        )

        try:
            async with heartbeat_context:
                matches, stats = await coordinator.enrich_batch(candidates)

                match_responses = [_match_to_response(m) for m in matches]

                base_meta = {
                    "records_scanned": len(records),
                    "candidates_found": len(candidates),
                    "duration_ms": stats.duration_ms if stats else None,
                    "sfdc_object": request.sfdc_object,
                }

                if not matches:
                    meta = {
                        **base_meta,
                        "records_written": 0,
                        "fields_updated": 0,
                        "error": "No enrichment matches found.",
                    }
                    finalize_usage = await tracker.finalize(
                        account_id=account_id,
                        user_email=user_email,
                        idempotency_key=idem_key,
                        request_hash=payload_hash,
                        actual_requests=stats.pdl_requests_made if stats else 0,
                        matches_found=0 if stats is None else stats.matches_found,
                        operation="apply",
                        enrichment_type=request.sfdc_object.lower(),
                        provider="pdl",
                        job_id=job_id,
                        chunk_id=chunk_id,
                        meta=meta,
                    )
                    reservation_active = False
                    quota_snapshot = await tracker.balances(account_id)
                    return ApplyResponse(
                        ok=False,
                        records_written=0,
                        records_failed=0,
                        fields_updated={},
                        matches=[],
                        preflight=preflight,
                        usage=finalize_usage,
                        quota_snapshot=quota_snapshot,
                        message="No enrichment matches found.",
                    )

                # First, finalize usage to get usage_id (before writing to enable archival)
                preliminary_meta = {
                    **base_meta,
                    "status": "writing",
                }
                finalize_usage_preliminary = await tracker.finalize(
                    account_id=account_id,
                    user_email=user_email,
                    idempotency_key=idem_key,
                    request_hash=payload_hash,
                    actual_requests=stats.pdl_requests_made,
                    matches_found=stats.matches_found,
                    operation="apply",
                    enrichment_type=request.sfdc_object.lower(),
                    provider="pdl",
                    job_id=job_id,
                    chunk_id=chunk_id,
                    meta=preliminary_meta,
                )
                usage_id = finalize_usage_preliminary.get("usage_id")

                # Now write enrichments with db and usage_id to enable archival
                write_result = await writer.write_enrichments(
                    account_id=account_id,
                    matches=matches,
                    dry_run=request.dry_run,
                    db=db,  # ✅ enables archive flow
                    usage_id=usage_id,  # ✅ connects vault rows to ledger
                )

                total_fields_updated = (
                    sum(write_result.fields_updated.values())
                    if write_result.fields_updated
                    else 0
                )

                # Update usage with final write results
                meta = {
                    **base_meta,
                    "records_written": write_result.records_written,
                    "fields_updated": total_fields_updated,
                    "status": "complete",
                }

                finalize_usage = await tracker.finalize(
                    account_id=account_id,
                    user_email=user_email,
                    idempotency_key=idem_key,
                    request_hash=payload_hash,
                    actual_requests=stats.pdl_requests_made,
                    matches_found=stats.matches_found,
                    operation="apply",
                    enrichment_type=request.sfdc_object.lower(),
                    provider="pdl",
                    job_id=job_id,
                    chunk_id=chunk_id,
                    meta=meta,
                )
                reservation_active = False
        except Exception as exc:
            if reservation_active and stats:
                error_meta = {
                    "records_scanned": len(records),
                    "candidates_found": len(candidates),
                    "records_written": 0,
                    "fields_updated": 0,
                    "duration_ms": stats.duration_ms if stats else None,
                    "sfdc_object": request.sfdc_object,
                    "error": str(exc),
                }
                try:
                    finalize_usage = await tracker.finalize(
                        account_id=account_id,
                        user_email=user_email,
                        idempotency_key=idem_key,
                        request_hash=payload_hash,
                        actual_requests=stats.pdl_requests_made,
                        matches_found=stats.matches_found,
                        operation="apply",
                        enrichment_type=request.sfdc_object.lower(),
                        provider="pdl",
                        job_id=job_id,
                        chunk_id=chunk_id,
                        meta=error_meta,
                    )
                    reservation_active = False
                except Exception:
                    logger.exception(
                        "Failed to finalize usage after apply error; enqueueing outbox."
                    )
                    outbox_id = await tracker.enqueue_ledger_outbox(
                        account_id=account_id,
                        user_email=user_email,
                        idempotency_key=idem_key,
                        request_hash=payload_hash,
                        actual_requests=stats.pdl_requests_made,
                        matches_found=stats.matches_found,
                        operation="apply",
                        enrichment_type=request.sfdc_object.lower(),
                        provider="pdl",
                        job_id=job_id,
                        chunk_id=chunk_id,
                        meta=error_meta,
                    )
                    if outbox_id is not None:
                        logger.info(
                            "Enqueued enrichment ledger outbox %s for account=%s chunk=%s",
                            outbox_id,
                            account_id,
                            chunk_id,
                        )
            elif reservation_active:
                await tracker.release(account_id, idem_key)
            raise

        quota_snapshot = await tracker.balances(account_id)

        track_event(
            "contact_enrichment.apply",
            {
                "account_id": account_id,
                "records_written": write_result.records_written,
                "records_failed": write_result.records_failed,
                "fields_updated": write_result.fields_updated,
                "dry_run": request.dry_run,
            },
        )

        message = (
            f"Successfully updated {write_result.records_written} records"
            if write_result.success
            else f"Partial success: {write_result.records_written} succeeded, {write_result.records_failed} failed"
        )

        return ApplyResponse(
            ok=write_result.success,
            records_written=write_result.records_written,
            records_failed=write_result.records_failed,
            fields_updated=write_result.fields_updated,
            matches=match_responses,
            errors=write_result.errors,
            audit_trail=write_result.audit_trail,
            usage=finalize_usage,
            quota_snapshot=quota_snapshot,
            preflight=preflight,
            message=message,
        )

    except HTTPException:
        raise
    except Exception as e:
        logger.error("Apply failed: %s", e, exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))


FILL_FIELD_SPECS: List[Dict[str, str]] = [
    {
        "key": "email",
        "logical": "email",
        "mode": "email",
        "summary": "email",
        "response_field": "email",
    },
    {
        "key": "phone",
        "logical": "phone",
        "mode": "phone",
        "summary": "phone",
        "response_field": "phone",
    },
    {
        "key": "mobile",
        "logical": "mobile_phone",
        "mode": "phone",
        "summary": "mobile",
        "response_field": "mobile_phone",
    },
    {
        "key": "title",
        "logical": "title",
        "mode": "text",
        "summary": "title",
        "response_field": "title",
    },
    {
        "key": "linkedin",
        "logical": "linkedin_url",
        "mode": "url",
        "summary": "linkedin",
        "response_field": "linkedin",
    },
    {
        "key": "company_name",
        "logical": "company_name",
        "mode": "text",
        "summary": "company",
        "response_field": "company_name",
    },
    {
        "key": "company_domain",
        "logical": "company_domain",
        "mode": "domain",
        "summary": "company",
        "response_field": "company_domain",
    },
]

FILLABLE_FIELD_ORDER = ["email", "phone", "mobile", "title", "linkedin", "company"]


def _utc_now_iso() -> str:
    """Return a UTC ISO8601 timestamp without microseconds."""
    return (
        datetime.now(timezone.utc)
        .replace(microsecond=0)
        .isoformat()
        .replace("+00:00", "Z")
    )


def _build_correlation_id(prefix: str) -> str:
    """Generate a human-readable correlation identifier."""
    return f"{prefix}_{_utc_now_iso()}_{uuid.uuid4().hex[:6]}"


def _normalize_value(value: Optional[str], mode: str = "text") -> Optional[str]:
    """Normalize values for safe equality checks."""
    if value is None:
        return None
    if isinstance(value, str):
        trimmed = value.strip()
        if not trimmed:
            return None
    else:
        trimmed = str(value).strip()
        if not trimmed:
            return None

    if mode == "email":
        return trimmed.lower()
    if mode == "phone":
        digits = "".join(ch for ch in trimmed if ch.isdigit())
        return digits or None
    if mode == "url":
        return trimmed.lower()
    if mode == "domain":
        lowered = trimmed.lower()
        for prefix in ("https://", "http://"):
            if lowered.startswith(prefix):
                lowered = lowered[len(prefix) :]
        if lowered.startswith("www."):
            lowered = lowered[4:]
        return lowered.split("/", 1)[0]
    if mode == "text":
        return trimmed.lower()
    return trimmed


def _value_changed(
    new_value: Optional[str], old_value: Optional[str], *, mode: str
) -> bool:
    """Return True when the enriched value is materially different."""
    normalized_new = _normalize_value(new_value, mode)
    if not normalized_new:
        return False
    normalized_old = _normalize_value(old_value, mode)
    if normalized_old is None:
        return True
    return normalized_new != normalized_old


def _extract_display_name(candidate: MissingFieldCandidate) -> str:
    """Build a friendly display name for UI consumption."""
    if candidate.full_name:
        return candidate.full_name
    parts = [candidate.first_name or "", candidate.last_name or ""]
    combined = " ".join(part for part in parts if part).strip()
    return combined or candidate.sfdc_id


def _compute_match_payload(match: EnrichmentMatch) -> Dict[str, Any]:
    """Summarize fillable fields and value deltas for a match."""
    candidate = match.candidate
    current = {
        "email": candidate.email,
        "phone": candidate.phone,
        "mobile": candidate.mobile_phone,
        "title": candidate.title,
        "linkedin": getattr(candidate, "linkedin_url", None),
        "company_name": getattr(candidate, "company_name", None),
        "company_domain": getattr(candidate, "company_domain", None),
    }

    # Sanitize enriched values - PDL sometimes returns booleans instead of strings
    def _sanitize_str(value: Any) -> Optional[str]:
        if value is None:
            return None
        if isinstance(value, bool):
            return None  # Treat boolean flags as missing data
        if isinstance(value, str):
            return value
        return str(value)  # Convert other types to string

    enriched = {
        "email": _sanitize_str(match.enriched_email),
        "phone": _sanitize_str(match.enriched_phone),
        "mobile": _sanitize_str(match.enriched_mobile_phone),
        "title": _sanitize_str(match.enriched_title),
        "linkedin": _sanitize_str(match.enriched_linkedin_url),
        "company_name": _sanitize_str(match.enriched_company_name),
        "company_domain": _sanitize_str(match.enriched_company_domain),
    }

    fillable_flags: Dict[str, bool] = {
        "email": _value_changed(enriched["email"], current["email"], mode="email"),
        "phone": _value_changed(enriched["phone"], current["phone"], mode="phone"),
        "mobile": _value_changed(enriched["mobile"], current["mobile"], mode="phone"),
        "title": _value_changed(enriched["title"], current["title"], mode="text"),
        "linkedin": _value_changed(
            enriched["linkedin"], current["linkedin"], mode="url"
        ),
        "company_name": _value_changed(
            enriched["company_name"], current["company_name"], mode="text"
        ),
        "company_domain": _value_changed(
            enriched["company_domain"], current["company_domain"], mode="domain"
        ),
    }

    fillable_set = set()
    for key, changed in fillable_flags.items():
        if not changed:
            continue
        if key in {"company_name", "company_domain"}:
            fillable_set.add("company")
        else:
            fillable_set.add(key)

    ordered_fillable = [
        field for field in FILLABLE_FIELD_ORDER if field in fillable_set
    ]

    return {
        "fillable": ordered_fillable,
        "current": current,
        "enriched": enriched,
    }


def _record_to_fill_candidate(
    scanner: MissingFieldScanner,
    record: Dict[str, Any],
    sfdc_object: str,
) -> Optional[MissingFieldCandidate]:
    """Convert a queried record into a candidate without requiring missing fields."""

    def _get_value(logical_field: str) -> Optional[str]:
        actual = scanner._get_field_name(sfdc_object, logical_field)
        if "." in actual:
            parts = actual.split(".")
            value: Any = record
            for part in parts:
                if not isinstance(value, dict):
                    return None
                value = value.get(part)
            return value
        return record.get(actual)

    try:
        first_name = _get_value("first_name")
        last_name = _get_value("last_name")
        if not first_name or not last_name:
            return None

        candidate = MissingFieldCandidate(
            sfdc_id=record["Id"],
            sfdc_object=sfdc_object,
            first_name=first_name,
            last_name=last_name,
            full_name=_get_value("full_name"),
            company_name=_get_value("company_name"),
            company_domain=scanner._normalize_domain(_get_value("company_domain")),
            email=_get_value("email"),
            phone=_get_value("phone"),
            mobile_phone=_get_value("mobile_phone"),
            title=_get_value("title"),
            linkedin_url=_get_value("linkedin_url"),
        )

        candidate.missing_fields.clear()
        enrichable_fields = (
            scanner.get_enrichable_fields()
            if hasattr(scanner, "get_enrichable_fields")
            else MissingFieldScanner.ENRICHABLE_FIELDS
        )
        for logical_field in enrichable_fields:
            current_value = getattr(candidate, logical_field, None)
            if not current_value or str(current_value).strip() == "":
                candidate.missing_fields.add(logical_field)

        return candidate
    except Exception as exc:
        logger.warning(
            "Failed to normalize record %s for fill: %s", record.get("Id"), exc
        )
        return None


def _build_field_map(scanner: MissingFieldScanner, sfdc_object: str) -> Dict[str, str]:
    """Return logical->actual Salesforce field mapping for fill operations."""
    mapping: Dict[str, str] = {}
    active_fields = (
        scanner.get_enrichable_fields()
        if hasattr(scanner, "get_enrichable_fields")
        else MissingFieldScanner.ENRICHABLE_FIELDS
    )
    for spec in FILL_FIELD_SPECS:
        logical = spec["logical"]
        if logical not in active_fields:
            continue
        try:
            actual = scanner._get_field_name(sfdc_object, logical)
        except Exception:
            actual = logical
        mapping[logical] = actual
    return mapping


def _derive_fill_changes(
    payload: Dict[str, Dict[str, Optional[str]]],
    policy: FillPolicyLiteral,
    field_map: Dict[str, str],
) -> Tuple[
    List[Dict[str, Optional[str]]], Dict[str, Optional[str]], Set[str], Optional[str]
]:
    """
    Convert enrichment payload into Salesforce patch instructions.

    Returns (changes, update_record, summary_fields, skip_reason)
    """
    applicable_specs: List[Tuple[Dict[str, str], str]] = []
    for spec in FILL_FIELD_SPECS:
        logical = spec["logical"]
        actual = field_map.get(logical)
        if not actual:
            continue
        if "." in actual:
            continue  # Cannot update relationship targets
        if logical == "linkedin_url" and actual.lower() == "linkedin_url":
            continue  # Unmapped LinkedIn custom field
        applicable_specs.append((spec, actual))

    if not applicable_specs:
        return [], {}, set(), "unsupported_fields"

    if policy == "skip_if_present":
        for spec, _actual in applicable_specs:
            current_val = payload["current"].get(spec["key"])
            if _normalize_value(current_val, spec["mode"]):
                return [], {}, set(), "already_has_value"

    changes: List[Dict[str, Optional[str]]] = []
    update_record: Dict[str, Optional[str]] = {}
    summary_fields: Set[str] = set()

    for spec, actual in applicable_specs:
        key = spec["key"]
        mode = spec["mode"]
        current_val = payload["current"].get(key)
        enriched_val = payload["enriched"].get(key)

        if not _value_changed(enriched_val, current_val, mode=mode):
            continue

        if policy == "empty_only" and _normalize_value(current_val, mode):
            continue

        changes.append(
            {
                "field": spec["response_field"],
                "from": current_val,
                "to": enriched_val,
            }
        )
        update_record[actual] = enriched_val
        summary_fields.add(spec["summary"])

    if not changes:
        return [], update_record, summary_fields, None

    return changes, update_record, summary_fields, None


def _ordered_summary_fields(fields: Set[str]) -> List[str]:
    """Return summary fields ordered for UI display."""
    if not fields:
        return []
    ordered = [field for field in FILLABLE_FIELD_ORDER if field in fields]
    extras = sorted(field for field in fields if field not in FILLABLE_FIELD_ORDER)
    return ordered + extras


def _reason_for_match(match: EnrichmentMatch, fillable: List[str]) -> str:
    """Short reason string for grid column."""
    confidence = (match.confidence or "").lower()
    if confidence == "verified":
        return "Verified match"
    if confidence == "likely":
        return "High-confidence match"
    if fillable:
        primary = fillable[0].capitalize()
        return f"{primary} ready"
    return "Enrichment available"


def _summarize_rows(rows: List[EligibleRow]) -> EligibleSummary:
    """Aggregate fillable counts for chips."""
    summary = {
        "total": len(rows),
        "email": 0,
        "mobile": 0,
        "title": 0,
        "linkedin": 0,
        "company": 0,
    }
    for row in rows:
        fillable_set = set(row.fillable)
        if "email" in fillable_set:
            summary["email"] += 1
        if "phone" in fillable_set or "mobile" in fillable_set:
            summary["mobile"] += 1
        if "title" in fillable_set:
            summary["title"] += 1
        if "linkedin" in fillable_set:
            summary["linkedin"] += 1
        if "company" in fillable_set:
            summary["company"] += 1
    return EligibleSummary(**summary)


def _credit_reload_url() -> Optional[str]:
    """Optional billing portal URL surfaced to Sheets."""
    return os.getenv("FM_BILLING_PORTAL_URL")


def _select_fields_for_query(
    scanner: MissingFieldScanner, sfdc_object: str
) -> List[str]:
    """Fields required to reconstruct candidates when refetching by Id."""
    logical_fields = [
        "first_name",
        "last_name",
        "full_name",
        "email",
        "phone",
        "mobile_phone",
        "title",
        "company_name",
        "company_domain",
    ]
    if hasattr(scanner, "get_enrichable_fields"):
        enrichable = set(scanner.get_enrichable_fields())
    else:
        enrichable = set(MissingFieldScanner.ENRICHABLE_FIELDS)
    if "linkedin_url" in enrichable:
        logical_fields.append("linkedin_url")
    fields: List[str] = []
    for logical in logical_fields:
        actual = scanner._get_field_name(sfdc_object, logical)
        if not actual:
            continue
        if actual.lower() == logical.lower() and logical == "linkedin_url":
            # Skip unmapped LinkedIn custom field
            continue
        if actual not in fields:
            fields.append(actual)
    return fields


def _resolve_salesforce_owner_id(gateway: SalesforceGateway) -> Optional[str]:
    """Best-effort lookup of the connected Salesforce user id for owner-scoped filters."""
    integration = getattr(gateway, "_integration", None)
    if integration is None:
        return None
    config = getattr(integration, "config", None)
    if isinstance(config, dict):
        owner_id = config.get("user_id") or config.get("owner_id")
        if owner_id:
            return owner_id
        user_block = config.get("user")
        if isinstance(user_block, dict):
            return user_block.get("id") or user_block.get("user_id")
    return None


def _filters_to_conditions(
    filters: Dict[str, Any], sfdc_object: str, owner_id: Optional[str]
) -> List[str]:
    """Convert filter payload into SOQL conditions per object."""
    if not filters or not isinstance(filters, dict):
        return []

    conditions: List[str] = []

    if sfdc_object == "Contact":
        account_filters = filters.get("account")
        if isinstance(account_filters, dict):
            for field, value in account_filters.items():
                field_path = f"Account.{field}"
                clause = _render_filter_condition(field_path, value)
                if clause:
                    conditions.append(clause)
    elif sfdc_object == "Lead":
        lead_filters = filters.get("lead")
        if isinstance(lead_filters, dict):
            for field, value in lead_filters.items():
                clause = _render_filter_condition(field, value)
                if clause:
                    conditions.append(clause)

    owner_scope = filters.get("owner_scope")
    if owner_scope == "me" and owner_id:
        clause = _render_filter_condition("OwnerId", owner_id, allow_list=False)
        if clause:
            conditions.append(clause)

    return conditions


def _render_filter_condition(
    field_path: str, raw_value: Any, *, allow_list: bool = True
) -> Optional[str]:
    """Render a single SOQL condition for provided value."""
    if not field_path or not _is_safe_field_reference(field_path):
        return None

    if isinstance(raw_value, list) and allow_list:
        formatted_items = [
            _format_filter_value(item) for item in raw_value if item is not None
        ]
        formatted_items = [item for item in formatted_items if item]
        if not formatted_items:
            return None
        return f"{field_path} IN ({', '.join(formatted_items)})"

    formatted_value = _format_filter_value(raw_value)
    if not formatted_value:
        return None
    return f"{field_path} = {formatted_value}"


def _format_filter_value(value: Any) -> Optional[str]:
    """Format literals safely for inclusion in SOQL."""
    if value is None:
        return None
    if isinstance(value, bool):
        return "true" if value else "false"
    if isinstance(value, (int, float)):
        return str(value)
    string_value = _escape_soql_literal(str(value))
    return f"'{string_value}'"


def _escape_soql_literal(value: str) -> str:
    """Escape single quotes for SOQL literals."""
    return value.replace("\\", "\\\\").replace("'", "\\'")


def _is_safe_field_reference(field_path: str) -> bool:
    """Validate that the field path is safe to embed in SOQL."""
    if not field_path:
        return False
    if ".." in field_path or field_path.startswith(".") or field_path.endswith("."):
        return False
    allowed_chars = set(
        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_."
    )
    return all(ch in allowed_chars for ch in field_path)


@router.get("/health")
async def health_check() -> Dict:
    """Check if contact enrichment services are available."""
    pdl_adapter = PeopleDataLabsAdapter()

    return {
        "ok": True,
        "pdl_available": pdl_adapter.available(),
        "services": {
            "scanner": "ok",
            "coordinator": "ok",
            "writer": "ok",
        },
    }
