"""Salesforce OAuth Integration API endpoints"""

from fastapi import (
    APIRouter,
    Depends,
    HTTPException,
    Query,
    BackgroundTasks,
    Request,
    Header,
    Body,
    status,
)
from fastapi.responses import HTMLResponse
from fastapi.responses import RedirectResponse, JSONResponse
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from typing import Optional, Dict, Any, List, Set, Tuple
from urllib.parse import urlencode, urlparse
import secrets
import jwt
import httpx
import asyncio
import time
import logging
import json
import html
import re
from datetime import datetime, timedelta
import os
import hashlib
import base64
import uuid
import pandas as pd

try:
    import redis.asyncio as redis_async
except Exception:  # pragma: no cover - redis optional during tests
    redis_async = None

from pydantic import BaseModel, Field

from ...db import get_session, AsyncSessionLocal
from ...auth_security import (
    ADMIN_EMAILS,
    AdminContext,
    require_account,
    require_admin,
    require_user,
)
from ...models import (
    Account,
    Integration,
    IntegrationStatus,
    IntegrationProvider,
    Job,
    JobStatus,
    JobType,
)
from ...security.crypto import encrypt_str, decrypt_str
from ... import settings
from ...services.context_service import ContextService
from ..progress import get_redis
from ...operators.match_salesforce_account import ScoringBackend
from ...services.salesforce_gateway import (
    SalesforceGateway,
    SalesforceAuthError,
    SalesforceConnectionError,
    SalesforceAllowlistError,
    SalesforceFieldAccessError,
    SalesforceNotConfigured,
    SalesforceRateLimitError,
    get_salesforce_gateway,
)
from ...services.soql_query_policy import (
    evaluate_safe_query_scope,
)
# Lazy import for sfdc_bulk_matcher to avoid DuckDB init at module load time
# This module chain (sfdc_bulk_matcher -> sfdc_account_cache -> duckdb) is heavy
_sfdc_bulk_matcher_module = None

def _get_sfdc_bulk_matcher():
    """Lazy loader for sfdc_bulk_matcher module."""
    global _sfdc_bulk_matcher_module
    if _sfdc_bulk_matcher_module is None:
        from ...services import sfdc_bulk_matcher as _m
        _sfdc_bulk_matcher_module = _m
    return _sfdc_bulk_matcher_module

def _get_SfdcBulkMatchWorker():
    return _get_sfdc_bulk_matcher().SfdcBulkMatchWorker

# Constants - define locally to avoid import, values match sfdc_bulk_matcher.py
SFDC_BULK_STATE_KEY = "sfdc_bulk:job:{job_id}:state"
SFDC_BULK_PROGRESS_KEY = "sfdc_bulk:job:{job_id}:progress"
SFDC_BULK_RESULT_KEY = "sfdc_bulk:job:{job_id}:result"
SFDC_BULK_IDEMPOTENCY_KEY = "sfdc_bulk:{account_id}:{key}"
OAUTH_STATE_KEY_PREFIX = "oauth_state:"
OAUTH_STATE_TTL_SECONDS = 600
_DEV_OAUTH_STATE_CACHE: Dict[str, Tuple[float, Dict[str, Any]]] = {}


def _oauth_state_key(state_id: str) -> str:
    return f"{OAUTH_STATE_KEY_PREFIX}{state_id}"


def _prune_dev_oauth_state_cache(now_ts: Optional[float] = None) -> None:
    now = now_ts or time.time()
    expired = [k for k, (expires_at, _) in _DEV_OAUTH_STATE_CACHE.items() if expires_at <= now]
    for key in expired:
        _DEV_OAUTH_STATE_CACHE.pop(key, None)


async def _store_oauth_state_payload(
    redis_client: Any, state_id: str, payload: Dict[str, Any]
) -> None:
    serialized = json.dumps(payload, separators=(",", ":"), default=str)
    if redis_client:
        await redis_client.set(
            _oauth_state_key(state_id),
            serialized,
            ex=OAUTH_STATE_TTL_SECONDS,
        )
        return
    if _is_dev_env():
        _prune_dev_oauth_state_cache()
        _DEV_OAUTH_STATE_CACHE[state_id] = (time.time() + OAUTH_STATE_TTL_SECONDS, payload)
        return
    raise HTTPException(
        status_code=503,
        detail="OAuth state storage unavailable. Please try again later.",
    )


async def _consume_oauth_state_payload(redis_client: Any, state_id: str) -> Dict[str, Any]:
    if redis_client:
        key = _oauth_state_key(state_id)
        raw = await redis_client.get(key)
        await redis_client.delete(key)
        if not raw:
            raise HTTPException(status_code=400, detail="Invalid or expired OAuth state")
        raw_text = raw.decode("utf-8") if isinstance(raw, (bytes, bytearray)) else str(raw)
        try:
            payload = json.loads(raw_text)
        except Exception as exc:
            log.warning("Failed to parse OAuth state payload for key %s: %s", key, exc)
            raise HTTPException(status_code=400, detail="Invalid OAuth state") from exc
        if not isinstance(payload, dict):
            raise HTTPException(status_code=400, detail="Invalid OAuth state")
        return payload

    if _is_dev_env():
        _prune_dev_oauth_state_cache()
        entry = _DEV_OAUTH_STATE_CACHE.pop(state_id, None)
        if not entry:
            raise HTTPException(status_code=400, detail="Invalid or expired OAuth state")
        expires_at, payload = entry
        if expires_at <= time.time():
            raise HTTPException(status_code=400, detail="Invalid or expired OAuth state")
        if not isinstance(payload, dict):
            raise HTTPException(status_code=400, detail="Invalid OAuth state")
        return payload

    raise HTTPException(
        status_code=503,
        detail="OAuth state storage unavailable. Please try again later.",
    )


async def _ensure_account(account_id: Optional[str], db: AsyncSession) -> None:
    """Ensure an account row exists for the given account_id (idempotent)."""
    if not account_id:
        return
    # Fast path: already exists
    existing = await db.get(Account, account_id)
    if existing:
        return
    stmt = pg_insert(Account).values(
        id=account_id,
        name=f"Clerk Org {account_id}",
        clerk_org_id=account_id,
        telemetry_opt_in=True,
        credits_balance=0,
        purchased_credits=0,
        data_region="us-east-1",
        paid_seat_count=1,
    )
    # Avoid race conditions if another request inserts simultaneously
    stmt = stmt.on_conflict_do_nothing(index_elements=[Account.id])
    await db.execute(stmt)
    await db.flush()


def _resolve_salesforce_auth_host(integration: Integration) -> str:
    cfg = integration.config or {}
    token_host = cfg.get("auth_host")
    token_host = _normalize_salesforce_custom_domain(token_host) if token_host else None
    if token_host:
        return token_host
    env = (cfg.get("environment") or getattr(settings, "SF_ENV", "production")).lower()
    if env in {"sandbox", "test"}:
        return "https://test.salesforce.com"
    return "https://login.salesforce.com"


def _looks_like_sandbox_domain(domain: Optional[str]) -> bool:
    if not domain:
        return False
    lowered = domain.lower()
    return "sandbox" in lowered or "test" in lowered


def _is_dev_env() -> bool:
    return (getattr(settings, "ENV", "") or "").strip().lower() in (
        "dev",
        "development",
        "local",
    )


def _truncate_for_log(value: Optional[str], limit: int = 200) -> Optional[str]:
    if value is None:
        return None
    text = str(value)
    if len(text) <= limit:
        return text
    return text[:limit] + "..."


def _require_signing_secret(purpose: str) -> str:
    """Return a signing secret or fail closed outside dev.

    Never default to a hardcoded secret in non-dev environments, or signed tokens
    (OAuth state / embed tokens) become forgeable.
    """
    candidates = _candidate_signing_secrets(include_dev_fallback=True)
    if candidates:
        return candidates[0]
    raise HTTPException(
        500,
        f"{purpose} signing secret not configured (ENCRYPTION_KEY/JWT_SECRET_KEY)",
    )


def _candidate_signing_secrets(include_dev_fallback: bool = False) -> List[str]:
    """Return unique signing secrets in priority order.

    We include both raw and stripped variants to tolerate newline/whitespace drift
    across environments and secret stores.
    """
    candidates: List[str] = []
    for raw in (
        getattr(settings, "ENCRYPTION_KEY", None),
        getattr(settings, "JWT_SECRET_KEY", None),
    ):
        if raw is None:
            continue
        text = str(raw)
        for candidate in (text, text.strip()):
            if candidate and candidate not in candidates:
                candidates.append(candidate)

    if not candidates and include_dev_fallback and _is_dev_env():
        candidates.append("dev-secret")

    return candidates


def _decode_oauth_state_token(state_token: str) -> Dict[str, Any]:
    """Decode OAuth state using all configured signing secrets."""
    candidates = _candidate_signing_secrets(include_dev_fallback=True)
    if not candidates:
        raise HTTPException(
            500,
            "OAuth state signing secret not configured (ENCRYPTION_KEY/JWT_SECRET_KEY)",
        )

    last_error: Optional[Exception] = None
    for secret in candidates:
        try:
            return jwt.decode(state_token, secret, algorithms=["HS256"])
        except jwt.ExpiredSignatureError:
            raise HTTPException(400, "State token expired. Please try connecting again.")
        except jwt.InvalidTokenError as exc:
            last_error = exc

    log.error(
        "Invalid state token signature (attempted_keys=%s): %s",
        len(candidates),
        last_error,
    )
    raise HTTPException(400, "Invalid state token")


def _normalize_salesforce_custom_domain(custom_domain: Optional[str]) -> Optional[str]:
    if not custom_domain:
        return None
    text = str(custom_domain).strip()
    if not text:
        return None
    if "://" not in text:
        text = f"https://{text}"
    parsed = urlparse(text)
    if (parsed.scheme or "").lower() != "https":
        return None
    if parsed.username or parsed.password:
        return None
    if parsed.port and parsed.port != 443:
        return None
    host = parsed.hostname or parsed.path
    if not host:
        return None
    host = host.strip().lower().rstrip(".")

    if host.endswith(".sandbox.lightning.force.com"):
        base = host[: -len(".sandbox.lightning.force.com")]
        return f"https://{base}.my.salesforce.com"
    if host.endswith(".lightning.force.com"):
        base = host[: -len(".lightning.force.com")]
        return f"https://{base}.my.salesforce.com"
    if host.endswith(".my.salesforce.com"):
        return f"https://{host}"
    if host.endswith(".salesforce.com") or host.endswith(".force.com"):
        return f"https://{host}"
    if "." not in host:
        host = f"{host}.my.salesforce.com"

    # Only allow Salesforce-owned domains to avoid SSRF and secret exfiltration.
    allowed_suffixes = ("salesforce.com", "my.salesforce.com", "force.com")
    for suffix in allowed_suffixes:
        suffix = suffix.lower().lstrip(".").rstrip(".")
        if host == suffix or host.endswith("." + suffix):
            return f"https://{host}"
    return None


def _select_oauth_host(environment: str, custom_domain: Optional[str]) -> str:
    env = (environment or "production").lower()
    if env == "custom":
        normalized = _normalize_salesforce_custom_domain(custom_domain)
        if not normalized:
            raise HTTPException(400, "Invalid Salesforce custom domain")
        return normalized
    if env == "sandbox":
        return "https://test.salesforce.com"
    return "https://login.salesforce.com"


def _oauth_error_html(error_message: str) -> HTMLResponse:
    frontend_origin = settings.FRONTEND_ORIGIN
    message = (error_message or "Salesforce authorization failed").strip()
    safe_message = html.escape(message)
    message_json = json.dumps(message)
    html_content = f"""
    <!DOCTYPE html>
    <html>
    <head>
        <title>Connection Failed</title>
        <style>
            body {{
                font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
                display: flex;
                justify-content: center;
                align-items: center;
                height: 100vh;
                margin: 0;
                background: #f5f5f5;
            }}
            .error {{
                text-align: center;
                padding: 2rem;
                background: white;
                border-radius: 10px;
                box-shadow: 0 2px 10px rgba(0,0,0,0.1);
                max-width: 400px;
            }}
            h1 {{ color: #dc3545; margin: 0 0 1rem 0; }}
            p {{ color: #666; }}
            button {{
                margin-top: 1rem;
                padding: 0.5rem 1.5rem;
                background: #007bff;
                color: white;
                border: none;
                border-radius: 5px;
                cursor: pointer;
            }}
        </style>
    </head>
    <body>
        <div class="error">
            <h1>❌ Connection Failed</h1>
            <p>{safe_message}</p>
            <button onclick="window.close()">Close Window</button>
        </div>
        <script>
            // Notify parent window of failure
            if (window.opener) {{
                window.opener.postMessage({{
                    type: 'salesforce_connected',
                    success: false,
                    error: {message_json}
                }}, '{frontend_origin}');
            }}
        </script>
    </body>
    </html>
    """
    return HTMLResponse(content=html_content, status_code=400)


async def _revoke_salesforce_token(token: str, token_host: str) -> None:
    if not token:
        return
    url = f"{token_host}/services/oauth2/revoke"
    try:
        async with httpx.AsyncClient(timeout=10) as client:
            response = await client.post(url, data={"token": token})
        if response.status_code not in (200, 204):
            log.warning(
                "Salesforce token revocation returned %s for host %s",
                response.status_code,
                token_host,
            )
    except Exception as exc:
        log.warning("Salesforce token revocation failed for host %s: %s", token_host, exc)
from .gremlin import (
    ScanRequest,
    get_gremlin_tier,
    get_sheets_service_from_token,
    _resolve_range,
    _fetch_sheet_grids,
    _list_sheet_titles,
    _http_error,
    _extract_error_code,
    _internal_error_response,
    _log_sheets_event,
    _request_id,
)
from g_gremlin.a1 import col_index_to_letter, parse_a1_range
from ...services.engine_client import EngineClient
from ...api.intelligence_client import IntelligenceClient
from ...services.salesforce_schema_index import SalesforceSchemaIndexer
from ...repos.salesforce_schema_index_repo import SalesforceSchemaIndexRepo
from ...monitoring.oauth_metrics import track_wait_until, track_connection

# Main router for integration endpoints
router = APIRouter(prefix="/api/v2/integrations/salesforce", tags=["salesforce"])

# Secondary router for data operations
api_router = APIRouter(prefix="/api/v2/salesforce", tags=["salesforce-api"])
log = logging.getLogger(__name__)


def _currency_field_names(describe_payload: dict) -> list[str]:
    fields = describe_payload.get("fields") or []
    allowed = {"currency", "double", "int", "percent", "number"}
    names: list[str] = []
    for field in fields:
        ftype = str(field.get("type") or "").lower()
        if ftype in allowed:
            name = field.get("name")
            if name:
                names.append(name)
    return names


async def _describe_has_field(gateway: SalesforceGateway, sobject: str, field_name: str) -> bool:
    try:
        desc = await gateway.describe(sobject)
    except Exception as exc:
        log.debug("Failed to describe %s for %s lookup: %s", sobject, field_name, exc)
        return False
    if not isinstance(desc, dict):
        return False
    return any(
        str(field.get("name") or "") == field_name
        for field in (desc.get("fields") or [])
    )


async def _count_query(sf: SalesforceGateway, soql: str) -> int:
    res = await sf.soql(soql)
    if isinstance(res, dict):
        if "totalSize" in res:
            return int(res.get("totalSize") or 0)
        if "records" in res:
            return len(res.get("records") or [])
    return 0


def _aggregate_count(res: Any) -> Optional[int]:
    if not isinstance(res, dict):
        return None
    records = res.get("records") or []
    if not records:
        return None
    first = records[0]
    for key in ("expr0", "expr1"):
        if key in first and first[key] is not None:
            try:
                return int(first[key])
            except (TypeError, ValueError):
                return None
    return None

class SalesforceEnrichRequest(BaseModel):
    spreadsheet_id: str
    sheet_name: Optional[str] = None
    range: Optional[str] = None
    object: str
    id_column: Optional[str] = None
    match_mode: str = Field("id", description="id|email|domain|name")
    match_status_column: Optional[str] = None
    status_filter: Optional[str] = None
    fields: List[str] = Field(default_factory=list)


class SalesforceEnrichResponse(BaseModel):
    spreadsheet_id: str
    sheet_name: str
    rows_enriched: int
    fields_added: List[str] = Field(default_factory=list)


class MatchAccountsRequest(BaseModel):
    spreadsheet_id: str
    sheet_name: Optional[str] = None
    range: Optional[str] = None
    company_column: str
    domain_column: Optional[str] = None
    threshold: float = 0.85
    values: Optional[List[List[Any]]] = None


class MatchAccountResult(BaseModel):
    row_index: int
    company_name: Optional[str] = None
    account_id: Optional[str] = None
    account_name: Optional[str] = None
    match_score: Optional[float] = None
    match_method: Optional[str] = None


class MatchAccountsResponse(BaseModel):
    spreadsheet_id: str
    sheet_name: str
    rows_scanned: int
    matched: int
    low_confidence: int
    no_match: int
    matches: List[MatchAccountResult] = Field(default_factory=list)


class SfdcBulkMatchRequest(BaseModel):
    spreadsheet_id: str
    sheet_name: Optional[str] = None
    range: Optional[str] = None
    company_column: str
    domain_column: Optional[str] = None
    threshold: int = Field(default=80, ge=0, le=100)
    idempotency_key: Optional[str] = None
    values: Optional[List[List[Any]]] = None


class SfdcBulkMatchResponse(BaseModel):
    job_id: str
    status: str
    progress: Optional[int] = None
    phase: Optional[str] = None
    estimated_completion_seconds: Optional[int] = None


class SfdcBulkMatchResult(BaseModel):
    job_id: str
    total_rows: int
    matched_rows: int
    unmatched_rows: int
    ambiguous_rows: int
    results: List[Dict[str, Any]]
    cache_hit_rate: float
    processing_time_ms: int

# Simple in-memory describe cache: {(account_id, object_lower): (payload, timestamp)}
_DESCRIBE_CACHE: dict[tuple[str, str], tuple[dict, float]] = {}
_DESCRIBE_TTL_SEC = 20 * 60  # 20 minutes
_DESCRIBE_CACHE_REDIS = None


def _get_describe_cache_redis():
    global _DESCRIBE_CACHE_REDIS
    if redis_async is None:
        return None
    redis_url = getattr(settings, "REDIS_URL", None)
    if not redis_url:
        return None
    if _DESCRIBE_CACHE_REDIS is False:
        return None
    if _DESCRIBE_CACHE_REDIS is None:
        try:
            _DESCRIBE_CACHE_REDIS = redis_async.from_url(
                redis_url, decode_responses=True
            )
        except Exception as exc:  # pragma: no cover
            log.warning("Redis describe cache disabled: %s", exc)
            _DESCRIBE_CACHE_REDIS = False
    return (
        _DESCRIBE_CACHE_REDIS
        if _DESCRIBE_CACHE_REDIS not in (None, False)
        else None
    )


def _describe_cache_key(account_id: str, sobj_lower: str) -> str:
    return f"sf:describe:{account_id}:{sobj_lower}"


async def _read_describe_cache(account_id: str, sobj_lower: str) -> Optional[dict]:
    redis_client = _get_describe_cache_redis()
    if not redis_client:
        return None
    try:
        raw = await redis_client.get(_describe_cache_key(account_id, sobj_lower))
        if raw:
            return json.loads(raw)
    except Exception as exc:  # pragma: no cover
        log.warning("Failed to read Redis describe cache: %s", exc)
    return None


async def _write_describe_cache(
    account_id: str, sobj_lower: str, payload: dict, now: float
):
    _DESCRIBE_CACHE[(account_id, sobj_lower)] = (payload, now)
    redis_client = _get_describe_cache_redis()
    if not redis_client:
        return
    try:
        await redis_client.setex(
            _describe_cache_key(account_id, sobj_lower),
            _DESCRIBE_TTL_SEC,
            json.dumps(payload),
        )
    except Exception as exc:  # pragma: no cover
        log.warning("Failed to persist describe cache to Redis: %s", exc)


# ---------------------------------------------------------------------------
# Sheets enrich endpoint (Google Sheets -> Salesforce bridge)
# ---------------------------------------------------------------------------


@api_router.post("/enrich", response_model=SalesforceEnrichResponse)
async def enrich_sheet_from_salesforce(
    payload: SalesforceEnrichRequest,
    request: Request,
    service=Depends(get_sheets_service_from_token),
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
):
    """
    Enrich a Google Sheet range with Salesforce fields via the Salesforce bridge.
    """
    start_time = time.perf_counter()
    error_code: Optional[str] = None
    sheet_name: Optional[str] = None
    row_count: Optional[int] = None
    try:
        tier = await get_gremlin_tier(account_id, db)
        if tier == "free":
            upgrade_url = settings.APP_URL or "https://foundrymatch.com/pricing"
            raise HTTPException(
                status_code=status.HTTP_402_PAYMENT_REQUIRED,
                detail={
                    "error": "UPGRADE_REQUIRED",
                    "message": "Upgrade to Pro to let Gremlin execute fixes and enrich from Salesforce.",
                    "upgrade_url": upgrade_url,
                },
            )

        try:
            sf_status = await gateway.get_status()
        except (SalesforceAuthError, SalesforceNotConfigured):
            error_code = "AUTH_REQUIRED"
            return _salesforce_error_response(
                status.HTTP_401_UNAUTHORIZED,
                "AUTH_REQUIRED",
                "Connect Salesforce in FoundryMatch before using Enrich.",
                request,
            )
        except SalesforceConnectionError:
            log.exception(
                "Salesforce status check failed",
                extra={"account_id": account_id},
            )
            error_code = "INTERNAL_ERROR"
            return _internal_error_response(request)

        if not (sf_status and sf_status.get("connected")):
            error_code = "AUTH_REQUIRED"
            return _salesforce_error_response(
                status.HTTP_401_UNAUTHORIZED,
                "AUTH_REQUIRED",
                "Connect Salesforce in FoundryMatch before using Enrich.",
                request,
            )

        try:
            sheet_titles = _list_sheet_titles(service, payload.spreadsheet_id)
        except Exception as err:  # HttpError handled in helper
            raise _http_error(err, "Unable to list sheets") from err

        scan_payload = ScanRequest(
            spreadsheet_id=payload.spreadsheet_id,
            sheet_name=payload.sheet_name,
            range=payload.range,
        )
        sheet_name, resolved_range = _resolve_range(scan_payload, sheet_titles)

        try:
            values_grid, _ = _fetch_sheet_grids(service, payload.spreadsheet_id, resolved_range)
        except Exception as err:
            raise _http_error(err, "Unable to read sheet") from err

        if not values_grid:
            raise HTTPException(status_code=400, detail={"error": "INVALID_RANGE", "message": "Sheet range is empty."})

        headers = [str(h) if h is not None else "" for h in (values_grid[0] or [])]
        if not headers or not any(headers):
            raise HTTPException(status_code=400, detail={"error": "INVALID_HEADER", "message": "Header row is empty."})

        rows = _rows_from_grid(values_grid[1:], headers)
        filter_applied = bool(payload.match_status_column and payload.status_filter is not None)
        if filter_applied:
            if payload.match_status_column not in headers:
                raise HTTPException(
                    status_code=400,
                    detail={"error": "INVALID_CONFIG", "message": "match_status_column not found in sheet headers."},
                )
            target_rows = [
                row
                for row in rows
                if str(row.get(payload.match_status_column) or "") == str(payload.status_filter)
            ]
        else:
            target_rows = rows
        match_column = _resolve_match_column(payload, headers)
        if not match_column:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "Unable to determine match column for enrichment."},
            )

        row_count = len(target_rows)
        match_values = _extract_match_values(target_rows, match_column, payload.match_mode)
        if not match_values:
            if filter_applied:
                return SalesforceEnrichResponse(
                    spreadsheet_id=payload.spreadsheet_id,
                    sheet_name=sheet_name,
                    rows_enriched=0,
                    fields_added=[],
                )
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "No matchable values found in the selected range."},
            )

        limit = _row_limit_for_tier(tier)
        if row_count > limit:
            error_code = "ROW_LIMIT_EXCEEDED"
            return _salesforce_error_response(
                status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
                "ROW_LIMIT_EXCEEDED",
                "This action exceeds your row limit for this plan. Try a smaller selection or upgrade.",
                request,
                extra={"limit": limit},
            )

        fields_to_add = _normalize_sfdc_fields(payload.fields)
        if not fields_to_add:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_FIELDS", "message": "At least one Salesforce field is required."},
            )

        try:
            records_by_key = await _fetch_salesforce_records(
                gateway,
                payload.object,
                payload.match_mode,
                match_column,
                match_values,
                fields_to_add,
            )
        except SalesforceRateLimitError:
            error_code = "RATE_LIMITED"
            return _salesforce_error_response(
                status.HTTP_429_TOO_MANY_REQUESTS,
                "RATE_LIMITED",
                "Salesforce is temporarily rate limiting requests. Try again later or reduce the selection.",
                request,
            )
        except (SalesforceFieldAccessError, SalesforceAllowlistError, ValueError):
            error_code = "INVALID_CONFIG"
            return _salesforce_error_response(
                status.HTTP_400_BAD_REQUEST,
                "INVALID_CONFIG",
                "The Salesforce enrich configuration is invalid. Check object, match column, and fields.",
                request,
            )
        except SalesforceAuthError:
            error_code = "AUTH_REQUIRED"
            return _salesforce_error_response(
                status.HTTP_401_UNAUTHORIZED,
                "AUTH_REQUIRED",
                "Connect Salesforce in FoundryMatch before using Enrich.",
                request,
            )
        except SalesforceConnectionError:
            error_code = "INTERNAL_ERROR"
            log.exception(
                "Salesforce connection error during enrich",
                extra={"account_id": account_id, "spreadsheet_id": payload.spreadsheet_id, "sheet_name": sheet_name},
            )
            return _internal_error_response(request)
        except HTTPException as exc:
            error_code = _extract_error_code(exc.detail)
            raise
        except Exception:
            error_code = "INTERNAL_ERROR"
            log.exception(
                "Salesforce enrich failed",
                extra={
                    "account_id": account_id,
                    "spreadsheet_id": payload.spreadsheet_id,
                    "sheet_name": sheet_name,
                },
            )
            return _internal_error_response(request)

        fields_added = fields_to_add
        new_rows: List[List[Any]] = []
        target_flags = None
        if filter_applied:
            target_flags = [
                str(row.get(payload.match_status_column) or "") == str(payload.status_filter) for row in rows
            ]
        enriched_count = 0
        for idx, row in enumerate(rows):
            should_enrich = True if not filter_applied else bool(target_flags and target_flags[idx])
            if should_enrich:
                key = _normalize_match_value(row.get(match_column), payload.match_mode)
                record = records_by_key.get(key)
                if record:
                    enriched_count += 1
                new_rows.append([_extract_field_value(record, field) for field in fields_added])
            else:
                new_rows.append(["" for _ in fields_added])

        write_range = _build_enrich_write_range(resolved_range, len(headers), len(fields_added), len(new_rows))
        values_to_write: List[List[Any]] = [fields_added]
        values_to_write.extend(new_rows)

        try:
            _write_enriched_columns(service, payload.spreadsheet_id, write_range, values_to_write)
        except Exception as err:
            raise _http_error(err, "Failed to write enriched data") from err

        return SalesforceEnrichResponse(
            spreadsheet_id=payload.spreadsheet_id,
            sheet_name=sheet_name,
            rows_enriched=enriched_count,
            fields_added=fields_added,
        )
    except HTTPException as exc:
        error_code = _extract_error_code(exc.detail) or f"HTTP_{exc.status_code}"
        config_errors = {
            "INVALID_CONFIG",
            "INVALID_MATCH",
            "INVALID_FIELDS",
            "INVALID_RANGE",
            "INVALID_HEADER",
            "NO_MATCH_VALUES",
        }
        if exc.status_code in (400, 422) and error_code in config_errors:
            error_code = "INVALID_CONFIG"
            return _salesforce_error_response(
                exc.status_code,
                "INVALID_CONFIG",
                "The Salesforce enrich configuration is invalid. Check object, match column, and fields.",
                request,
            )
        raise
    except Exception:
        error_code = "INTERNAL_ERROR"
        log.exception(
            "Salesforce enrich failed unexpectedly",
            extra={"account_id": account_id, "spreadsheet_id": payload.spreadsheet_id, "sheet_name": sheet_name},
        )
        return _internal_error_response(request)
    finally:
        duration_ms = int((time.perf_counter() - start_time) * 1000)
        _log_sheets_event(
            feature="sfdc_enrich",
            account_id=account_id,
            spreadsheet_id=payload.spreadsheet_id,
            sheet_name=sheet_name or payload.sheet_name,
            row_count=row_count,
            duration_ms=duration_ms,
            status="success" if error_code is None else "error",
            error=error_code,
            request_id=_request_id(request),
        )


@api_router.post("/match/accounts", response_model=MatchAccountsResponse)
async def match_accounts_from_salesforce(
    payload: MatchAccountsRequest,
    request: Request,
    service=Depends(get_sheets_service_from_token),
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
):
    """
    Match messy company names/domains to Salesforce Accounts and return deterministic results for client-side writeback.
    """
    start_time = time.perf_counter()
    error_code: Optional[str] = None
    sheet_name: Optional[str] = None
    row_count: Optional[int] = None
    try:
        tier = await get_gremlin_tier(account_id, db)
        if tier == "free":
            upgrade_url = settings.APP_URL or "https://foundrymatch.com/pricing"
            raise HTTPException(
                status_code=status.HTTP_402_PAYMENT_REQUIRED,
                detail={
                    "error": "UPGRADE_REQUIRED",
                    "message": "Upgrade to Pro to let Gremlin execute fixes and enrich from Salesforce.",
                    "upgrade_url": upgrade_url,
                },
            )

        try:
            sf_status = await gateway.get_status()
        except (SalesforceAuthError, SalesforceNotConfigured):
            error_code = "AUTH_REQUIRED"
            return _salesforce_error_response(
                status.HTTP_401_UNAUTHORIZED,
                "AUTH_REQUIRED",
                "Connect Salesforce in FoundryMatch before using Match.",
                request,
            )
        except SalesforceConnectionError:
            log.exception(
                "Salesforce status check failed",
                extra={"account_id": account_id},
            )
            error_code = "INTERNAL_ERROR"
            return _internal_error_response(request)

        if not (sf_status and sf_status.get("connected")):
            error_code = "AUTH_REQUIRED"
            return _salesforce_error_response(
                status.HTTP_401_UNAUTHORIZED,
                "AUTH_REQUIRED",
                "Connect Salesforce in FoundryMatch before using Match.",
                request,
            )

        use_client_data = payload.values is not None and len(payload.values) > 0
        values_grid: List[List[Any]] = []
        resolved_range: Optional[str] = None
        sheet_titles: List[str] = []

        if use_client_data:
            sheet_name = payload.sheet_name or "Sheet1"
            resolved_range = payload.range or f"{sheet_name}!A1:Z{len(payload.values)}"
            values_grid = _normalize_client_values(payload.values)
        else:
            if service is None:
                raise HTTPException(
                    status_code=400,
                    detail={"error": "INVALID_CONFIG", "message": "values payload required when Sheets token is missing."},
                )
            try:
                sheet_titles = _list_sheet_titles(service, payload.spreadsheet_id)
            except Exception as err:  # HttpError handled in helper
                raise _http_error(err, "Unable to list sheets") from err

            sheet_name = payload.sheet_name or (sheet_titles[0] if sheet_titles else "")
            if not sheet_name:
                raise HTTPException(
                    status_code=400,
                    detail={"error": "INVALID_RANGE", "message": "Sheet name could not be resolved."},
                )

            resolved_range = _resolve_match_range(payload, sheet_name)

            try:
                values_grid, _ = _fetch_sheet_grids(service, payload.spreadsheet_id, resolved_range)
            except Exception as err:
                raise _http_error(err, "Unable to read sheet") from err

        if not values_grid:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_RANGE", "message": "Sheet range is empty."},
            )

        headers = [str(h) if h is not None else "" for h in (values_grid[0] or [])]
        if not headers or not any(headers):
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_HEADER", "message": "Header row is empty."},
            )
        if payload.company_column not in headers:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "Company column not found in headers."},
            )
        if payload.domain_column and payload.domain_column not in headers:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "Domain column not found in headers."},
            )

        rows = _rows_from_grid(values_grid[1:], headers)
        row_count = len(rows)
        try:
            parsed_range = parse_a1_range(resolved_range or f"{sheet_name}!A1:Z")
            data_start_row = parsed_range.start_row
        except Exception:
            data_start_row = 1
        if row_count == 0:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_RANGE", "message": "No data rows found to match."},
            )

        limit = _row_limit_for_tier(tier)
        if row_count > limit:
            error_code = "ROW_LIMIT_EXCEEDED"
            return _salesforce_error_response(
                status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
                "ROW_LIMIT_EXCEEDED",
                "This action exceeds your row limit for this plan. Try a smaller selection or upgrade.",
                request,
                extra={"limit": limit},
            )

        source_records: List[Dict[str, Any]] = []
        for idx, row in enumerate(rows):
            row_index = data_start_row + idx + 1  # account for header row within range
            row_id = str(row_index)
            company = row.get(payload.company_column)
            domain = row.get(payload.domain_column) if payload.domain_column else None
            if (company is None or str(company).strip() == "") and (domain is None or str(domain).strip() == ""):
                continue
            source_records.append(
                {
                    "row_id": row_id,
                    "company": company,
                    "company_name": company,
                    "domain": domain,
                }
            )

        if not source_records:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "No matchable company or domain values found."},
            )

        source_df = pd.DataFrame(source_records)
        if source_df.empty:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "No matchable company or domain values found."},
            )

        try:
            account_records = await _fetch_salesforce_accounts(gateway)
        except SalesforceRateLimitError:
            error_code = "RATE_LIMITED"
            return _salesforce_error_response(
                status.HTTP_429_TOO_MANY_REQUESTS,
                "RATE_LIMITED",
                "Salesforce is temporarily rate limiting requests. Try again later or reduce the selection.",
                request,
            )
        except SalesforceAuthError:
            error_code = "AUTH_REQUIRED"
            return _salesforce_error_response(
                status.HTTP_401_UNAUTHORIZED,
                "AUTH_REQUIRED",
                "Connect Salesforce in FoundryMatch before using Match.",
                request,
            )
        except SalesforceConnectionError:
            error_code = "INTERNAL_ERROR"
            log.exception(
                "Salesforce connection error during account fetch",
                extra={"account_id": account_id, "spreadsheet_id": payload.spreadsheet_id, "sheet_name": sheet_name},
            )
            return _internal_error_response(request)
        except Exception:
            error_code = "INTERNAL_ERROR"
            log.exception(
                "Salesforce account fetch failed",
                extra={"account_id": account_id, "spreadsheet_id": payload.spreadsheet_id, "sheet_name": sheet_name},
            )
            return _internal_error_response(request)

        target_df = pd.DataFrame(account_records or [])
        if target_df.empty:
            target_df = pd.DataFrame(columns=["Id", "Name", "Website"])
        if "Id" not in target_df.columns:
            target_df["Id"] = None
        if "Name" not in target_df.columns:
            target_df["Name"] = None
        if "Website" not in target_df.columns:
            target_df["Website"] = None
        target_df["account_id"] = target_df["Id"]
        target_df["account_name"] = target_df.get("Name")
        target_df["website"] = target_df.get("Website")
        if target_df.empty:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "No Salesforce Accounts available to match."},
            )

        threshold = _normalize_threshold_value(payload.threshold)

        intel = IntelligenceClient()
        config = await intel.auto_configure(source_df, target_df, goal="lookup")
        mappings = config.get("field_mappings") or config.get("mappings")
        if not mappings:
            raise HTTPException(
                status_code=502, detail="Unable to auto-configure matching configuration."
            )

        engine = EngineClient()
        try:
            engine_res = await engine.match_records(
                source_df,
                target_df,
                mappings=mappings,
                threshold=threshold,
                blocking_strategy=config.get("blocking"),
                source_id_column="row_id",
                reference_id_column="Id",
            )
        except ValueError:
            error_code = "INVALID_CONFIG"
            return _salesforce_error_response(
                status.HTTP_400_BAD_REQUEST,
                "INVALID_CONFIG",
                "Unable to run matcher with the current configuration.",
                request,
            )

        matches_df = _normalize_match_dataframe(engine_res.matches if engine_res else None)
        best_matches = _best_match_by_source(matches_df)
        match_counts = _count_matches_by_source(matches_df)
        account_lookup = {
            str(rec.get("Id") or rec.get("id") or rec.get("ID")): rec for rec in account_records or [] if isinstance(rec, dict)
        }

        matches_payload: List[Dict[str, Any]] = []
        matched_count = 0
        low_confidence_count = 0
        no_match_count = 0

        for idx, row in enumerate(rows):
            row_index = data_start_row + idx + 1
            rid = str(row_index)
            match_row = best_matches.get(rid)
            candidate_count = match_counts.get(rid, 0)
            confidence = _normalize_match_score(match_row.get("score") if match_row else None)
            account_id_val = ""
            account_name_val = ""
            status_val = "no_match"

            if match_row is not None:
                account_id_val = str(
                    match_row.get("target_id")
                    or match_row.get("sf_id")
                    or match_row.get("reference_id")
                    or match_row.get("ref_id")
                    or ""
                )
                account_name_val = ""
                if account_id_val and account_id_val in account_lookup:
                    account_name_val = account_lookup[account_id_val].get("Name") or account_lookup[account_id_val].get("name") or ""
                if candidate_count > 1:
                    status_val = "low_confidence"
                elif confidence is not None and confidence >= threshold:
                    status_val = "matched"
                elif confidence is not None and confidence >= 0.5:
                    status_val = "low_confidence"
                else:
                    status_val = "no_match"
            else:
                status_val = "no_match"

            if status_val == "matched":
                matched_count += 1
            elif status_val == "low_confidence":
                low_confidence_count += 1
            else:
                no_match_count += 1

            match_method = _infer_match_method(
                row.get(payload.domain_column) if payload.domain_column else None,
                account_lookup.get(account_id_val),
                status_val,
            )
            match_score = None if confidence is None else round(confidence, 3)
            matches_payload.append(
                {
                    "row_index": row_index,
                    "company_name": row.get(payload.company_column),
                    "account_id": account_id_val or None,
                    "account_name": account_name_val or None,
                    "match_score": match_score,
                    "match_method": match_method,
                }
            )

        return MatchAccountsResponse(
            spreadsheet_id=payload.spreadsheet_id,
            sheet_name=sheet_name,
            rows_scanned=row_count,
            matched=matched_count,
            low_confidence=low_confidence_count,
            no_match=no_match_count,
            matches=matches_payload,
        )
    except HTTPException as exc:
        error_code = _extract_error_code(exc.detail) or f"HTTP_{exc.status_code}"
        config_errors = {
            "INVALID_CONFIG",
            "INVALID_RANGE",
            "INVALID_HEADER",
        }
        if exc.status_code in (400, 422) and error_code in config_errors:
            error_code = "INVALID_CONFIG"
            return _salesforce_error_response(
                exc.status_code,
                "INVALID_CONFIG",
                "The Salesforce match configuration is invalid. Check columns and range.",
                request,
            )
        raise
    except Exception:
        error_code = "INTERNAL_ERROR"
        log.exception(
            "Salesforce match failed unexpectedly",
            extra={"account_id": account_id, "spreadsheet_id": payload.spreadsheet_id, "sheet_name": sheet_name},
        )
        return _internal_error_response(request)
    finally:
        duration_ms = int((time.perf_counter() - start_time) * 1000)
        _log_sheets_event(
            feature="sfdc_match_accounts",
            account_id=account_id,
            spreadsheet_id=payload.spreadsheet_id,
            sheet_name=sheet_name or payload.sheet_name,
            row_count=row_count,
            duration_ms=duration_ms,
            status="success" if error_code is None else "error",
            error=error_code,
            request_id=_request_id(request),
        )


@api_router.post("/match/bulk", response_model=SfdcBulkMatchResponse)
async def start_bulk_match(
    payload: SfdcBulkMatchRequest,
    request: Request,
    service=Depends(get_sheets_service_from_token),
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    redis=Depends(get_redis),
) -> SfdcBulkMatchResponse:
    if not getattr(settings, "SFDC_BULK_MATCH_ENABLED", False):
        raise HTTPException(status_code=404, detail="Bulk SFDC matching is disabled")

    if payload.threshold < 0 or payload.threshold > 100:
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_CONFIG", "message": "threshold must be between 0 and 100"},
        )

    # Check SFDC connection
    try:
        sf_status = await gateway.get_status()
    except (SalesforceAuthError, SalesforceNotConfigured):
        return _salesforce_error_response(
            status.HTTP_401_UNAUTHORIZED,
            "AUTH_REQUIRED",
            "Connect Salesforce in FoundryMatch before using Match.",
            request,
        )
    except SalesforceConnectionError:
        return _internal_error_response(request)

    if not (sf_status and sf_status.get("connected")):
        return _salesforce_error_response(
            status.HTTP_401_UNAUTHORIZED,
            "AUTH_REQUIRED",
            "Connect Salesforce in FoundryMatch before using Match.",
            request,
        )

    # Idempotency shortcut (reuse existing job)
    if redis and payload.idempotency_key:
        idem_key = SFDC_BULK_IDEMPOTENCY_KEY.format(
            account_id=account_id, key=payload.idempotency_key
        )
        existing = await redis.get(idem_key)
        if existing:
            job_id_existing = (
                existing.decode("utf-8") if hasattr(existing, "decode") else str(existing)
            )
            state = await redis.get(SFDC_BULK_STATE_KEY.format(job_id=job_id_existing))
            progress = await _read_bulk_progress(redis, job_id_existing)
            return SfdcBulkMatchResponse(
                job_id=job_id_existing,
                status=(state.decode("utf-8") if isinstance(state, (bytes, bytearray)) else state)
                or "queued",
                progress=progress.get("percent") if progress else None,
                phase=progress.get("phase") if progress else None,
                estimated_completion_seconds=None,
            )

    _, rows, data_start_row = await _extract_bulk_rows(payload, service)
    row_count = len(rows)
    if len(rows) < 2000:
        raise HTTPException(
            status_code=400,
            detail={
                "error": "ROW_LIMIT_TOO_SMALL",
                "message": "Use inline matching for fewer than 2,000 rows.",
            },
        )

    # Prepare rows for matcher
    prepared_rows: List[Dict[str, Any]] = []
    for idx, row in enumerate(rows):
        company = row.get(payload.company_column)
        domain = row.get(payload.domain_column) if payload.domain_column else None
        if (company is None or str(company).strip() == "") and (domain is None or str(domain).strip() == ""):
            continue
        prepared_rows.append(
            {
                "row_id": data_start_row + idx + 1,
                "company": company,
                "domain": domain,
            }
        )

    if not prepared_rows:
        raise HTTPException(
            status_code=400,
            detail={
                "error": "INVALID_CONFIG",
                "message": "No matchable company/domain rows found.",
            },
        )
    if len(prepared_rows) < 2000:
        raise HTTPException(
            status_code=400,
            detail={
                "error": "ROW_LIMIT_TOO_SMALL",
                "message": "Use inline matching for fewer than 2,000 rows.",
            },
        )

    integration_id = _resolve_integration_id(gateway) or account_id

    # Create job record
    job_record = Job(
        account_id=account_id,
        status=JobStatus.QUEUED,
        job_type=JobType.SFDC_BULK_MATCH,
        config={
            "spreadsheet_id": payload.spreadsheet_id,
            "sheet_name": payload.sheet_name,
            "company_column": payload.company_column,
            "domain_column": payload.domain_column,
            "row_count": len(prepared_rows),
            "threshold": payload.threshold,
        },
    )
    db.add(job_record)
    await db.commit()
    await db.refresh(job_record)

    SfdcBulkMatchWorker = _get_SfdcBulkMatchWorker()
    worker = SfdcBulkMatchWorker(redis=redis)
    job_id, _ = await worker.start_job(
        account_id=account_id,
        integration_id=integration_id,
        gateway=gateway,
        rows=prepared_rows,
        threshold=payload.threshold,
        scoring_backend=ScoringBackend.ENGINE,
        idempotency_key=payload.idempotency_key,
        job_record_id=job_record.id,
    )

    progress = await _read_bulk_progress(redis, job_id)
    return SfdcBulkMatchResponse(
        job_id=job_id,
        status="queued",
        progress=progress.get("percent") if progress else 0,
        phase=progress.get("phase") if progress else "prefetching",
        estimated_completion_seconds=None,
    )


@api_router.get("/match/bulk/{job_id}", response_model=SfdcBulkMatchResponse)
async def get_bulk_match_status(
    job_id: str,
    account_id: str = Depends(require_account),
    redis=Depends(get_redis),
) -> SfdcBulkMatchResponse:
    if not getattr(settings, "SFDC_BULK_MATCH_ENABLED", False):
        raise HTTPException(status_code=404, detail="Bulk SFDC matching is disabled")
    if not redis:
        raise HTTPException(status_code=503, detail="Redis is required for bulk matching")

    state = await redis.get(SFDC_BULK_STATE_KEY.format(job_id=job_id))
    if not state:
        raise HTTPException(status_code=404, detail="Job not found")
    progress = await _read_bulk_progress(redis, job_id)
    state_val = state.decode("utf-8") if isinstance(state, (bytes, bytearray)) else str(state)
    return SfdcBulkMatchResponse(
        job_id=job_id,
        status=state_val,
        progress=progress.get("percent") if progress else None,
        phase=progress.get("phase") if progress else None,
        estimated_completion_seconds=None,
    )


@api_router.get("/match/bulk/{job_id}/result", response_model=SfdcBulkMatchResult)
async def get_bulk_match_result(
    job_id: str,
    account_id: str = Depends(require_account),
    redis=Depends(get_redis),
) -> SfdcBulkMatchResult:
    if not getattr(settings, "SFDC_BULK_MATCH_ENABLED", False):
        raise HTTPException(status_code=404, detail="Bulk SFDC matching is disabled")
    if not redis:
        raise HTTPException(status_code=503, detail="Redis is required for bulk matching")

    raw = await redis.get(SFDC_BULK_RESULT_KEY.format(job_id=job_id))
    if not raw:
        state = await redis.get(SFDC_BULK_STATE_KEY.format(job_id=job_id))
        if state and (state.decode("utf-8") if hasattr(state, "decode") else state) in ("queued", "processing"):
            raise HTTPException(status_code=202, detail="Job not complete")
        raise HTTPException(status_code=404, detail="Result not found")

    payload = json.loads(raw.decode("utf-8") if isinstance(raw, (bytes, bytearray)) else raw)
    return SfdcBulkMatchResult(**payload)



# Status constants to avoid magic strings and ensure consistency
class StatusValues:
    """Normalized status values for integration state comparisons"""

    CONNECTED = "CONNECTED"
    DISCONNECTED = "DISCONNECTED"
    # Note: DB may store lowercase or mixed case, always normalize with .upper()
    # This ensures consistent comparisons regardless of how the enum is stored


class SalesforceEmbedTokenResponse(BaseModel):
    token: str
    expires_at: str
    expires_in: int
    connected: Optional[bool] = None
    account_id: Optional[str] = None


# Helper: compute a redirect_uri that matches Connected App configuration
def _compute_redirect_uri(request: Request) -> str:
    """Prefer explicit env var if set; otherwise derive from the request.

    Dev-friendly: allows http://localhost callbacks when configured in env and
    in the Connected App. This avoids changing the callback when using ngrok.
    """
    try:
        env_uri = (
            settings.SALESFORCE_REDIRECT_URI or settings.SF_REDIRECT_URI or ""
        ).strip()
    except Exception:
        env_uri = ""

    if env_uri:
        if _is_dev_env():
            try:
                parsed = urlparse(env_uri)
                env_host = (parsed.hostname or "").strip().lower()
                req_host = ((request.url.hostname or "") if request and request.url else "").strip().lower()
                if env_host and req_host and env_host != req_host:
                    log.warning(
                        "Ignoring SALESFORCE_REDIRECT_URI in development due host mismatch: env_host=%s request_host=%s",
                        env_host,
                        req_host,
                    )
                    return str(request.url_for("salesforce_oauth_callback"))
            except Exception:
                pass
        return env_uri

    if not _is_dev_env():
        raise HTTPException(
            status_code=500,
            detail="SALESFORCE_REDIRECT_URI must be configured in non-development environments",
        )

    # Dev-only fallback to dynamic URL from the incoming request
    return str(request.url_for("salesforce_oauth_callback"))


@api_router.get("/describe")
async def describe_object(
    object: str = Query(
        ..., description="Lead | Account | Contact | AccountContactRelation"
    ),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
    soft: bool = Query(
        True, description="Return exists:false instead of 500 for missing objects"
    ),
):
    """Return Salesforce object describe with a 20-minute cache.

    Response: { fields: [{name,label,type,picklistValues,...}], lastFetched }
    """
    sobj = (object or "").strip()
    if sobj not in ("Lead", "Account", "Contact", "AccountContactRelation"):
        raise HTTPException(
            400,
            "object must be 'Lead' | 'Account' | 'Contact' | 'AccountContactRelation'",
        )

    sobj_lower = sobj.lower()
    key = (account_id, sobj_lower)
    now = time.time()
    cached_payload = await _read_describe_cache(account_id, sobj_lower)
    if cached_payload:
        _DESCRIBE_CACHE[key] = (cached_payload, now)
        return {
            "fields": cached_payload.get("fields", []),
            "lastFetched": cached_payload.get("lastFetched")
            or datetime.utcnow().isoformat(),
        }
    cached = _DESCRIBE_CACHE.get(key)
    if cached and (now - cached[1]) < _DESCRIBE_TTL_SEC:
        payload, ts = cached
        last_fetched = payload.get("lastFetched") or datetime.fromtimestamp(ts).isoformat()
        return {
            "fields": payload.get("fields", []),
            "lastFetched": last_fetched,
        }

    try:
        raw = await sf.describe(sobj)
        fields = []
        for f in raw.get("fields", []):
            fields.append(
                {
                    "name": f.get("name"),
                    "label": f.get("label"),
                    "type": f.get("type"),
                    "picklistValues": f.get("picklistValues", []),
                    "referenceTo": f.get("referenceTo", []),
                    "filterable": f.get("filterable", False),
                }
            )
        payload = {"fields": fields, "lastFetched": datetime.utcnow().isoformat()}
        await _write_describe_cache(account_id, sobj_lower, payload, now)
        return {
            "exists": True,
            "object": sobj,
            "fields": fields,
            "lastFetched": payload["lastFetched"],
        }
    except Exception as e:
        # Detect Salesforce "object not found" errors and soft-fail if requested
        msg = str(e) if e else ""
        status = None
        try:
            status = getattr(
                getattr(e, "response", None), "status_code", None
            ) or getattr(e, "status_code", None)
        except Exception:
            status = None
        is_missing = (
            status == 404
            or ("INVALID_TYPE" in msg)
            or ("NOT_FOUND" in msg)
            or ("404" in msg)
        )
        if soft and is_missing:
            return {"exists": False, "object": sobj, "fields": []}
        log.error(f"Describe failed for {sobj}: {e}")
        raise HTTPException(500, "Describe failed")


@api_router.get("/describe/{sobject}")
async def describe_object_path(
    sobject: str,
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
) -> Dict[str, Any]:
    """Compatibility route for clients still using a path param."""
    try:
        return await gateway.describe(sobject)
    except HTTPException:
        raise
    except Exception as e:
        log.exception(f"Failed to describe Salesforce object {sobject}: {e}")
        raise HTTPException(status_code=500, detail="Failed to describe object")


@api_router.get("/org-info")
async def salesforce_org_info(
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Return Salesforce org metadata for GRR wizard."""
    is_multi = await _describe_has_field(gateway, "Organization", "IsMultiCurrencyEnabled")

    opp_count = await _count_query(gateway, "SELECT COUNT() FROM Opportunity WHERE IsWon = true")
    account_count = None
    try:
        acct_res = await gateway.soql(
            "SELECT COUNT_DISTINCT(AccountId) FROM Opportunity WHERE IsWon = true AND AccountId != null",
        )
        account_count = _aggregate_count(acct_res)
    except Exception as exc:
        log.warning("Failed to estimate account count: %s", exc)
    oli_count = await _count_query(
        gateway, "SELECT COUNT() FROM OpportunityLineItem WHERE Opportunity.IsWon = true"
    )
    has_oli = oli_count > 0

    enabled_currencies: list[str] = []
    corporate_currency: Optional[str] = None
    if is_multi:
        try:
            currency_res = await gateway.soql(
                "SELECT IsoCode, IsActive, IsCorporate FROM CurrencyType WHERE IsActive = true LIMIT 200"
            )
            if isinstance(currency_res, dict):
                for row in currency_res.get("records") or []:
                    code = row.get("IsoCode")
                    if code:
                        enabled_currencies.append(code)
                    if row.get("IsCorporate"):
                        corporate_currency = code
        except Exception as exc:
            log.warning("Failed to load CurrencyType info: %s", exc)

    currency_fields_available: list[str] = []
    try:
        target_obj = "OpportunityLineItem" if has_oli else "Opportunity"
        desc = await gateway.describe(target_obj)
        if isinstance(desc, dict):
            currency_fields_available = _currency_field_names(desc)
    except Exception as exc:
        log.warning("Failed to describe for currency fields: %s", exc)

    return {
        "is_multi_currency": is_multi,
        "corporate_currency": corporate_currency,
        "enabled_currencies": enabled_currencies,
        "has_opportunity_line_items": has_oli,
        "account_count_estimate": account_count or 0,
        "opportunity_count_estimate": opp_count,
        "oli_count_estimate": oli_count,
        "currency_fields_available": currency_fields_available,
    }


# COMMENTED OUT - Duplicate route, using the more complete version on line 1557
# @api_router.get("/schema/{object_type}")
# async def schema(
#     object_type: str,
#     sf: SalesforceGateway = Depends(get_salesforce_gateway),
# ):
#     """Soft-fail describe to simply report existence without 500-ing the UI.
#
#     Returns: { exists: bool, object: str, fields: list }
#     """
#     try:
#         desc = await sf.describe(object_type)
#         return {"exists": True, "object": object_type, "fields": desc.get("fields", [])}
#     except Exception as e:
#         txt = str(e)
#         if any(x in txt for x in ("INVALID_TYPE", "NOT_FOUND", "404")):
#             return {"exists": False, "object": object_type, "fields": []}
#         # Some orgs return 404 in response.status_code
#         try:
#             if getattr(getattr(e, "response", None), "status_code", None) == 404:
#                 return {"exists": False, "object": object_type, "fields": []}
#         except Exception:
#             pass
#         raise


@router.get("/connect")
async def start_salesforce_connect(
    request: Request,
    environment: str = Query(
        "production", pattern="^(production|sandbox|developer|custom)$"
    ),
    custom_domain: Optional[str] = Query(None),
    user_id: str = Depends(require_user),
    account_id: str = Depends(require_account),
    db: AsyncSession = Depends(get_session),
    redis=Depends(get_redis),
) -> Dict[str, str]:
    """Generate OAuth URL with signed state token for CSRF protection"""
    await _ensure_account(account_id, db)

    # Check if we have an existing integration with a refresh token
    result = await db.execute(
        select(Integration).where(
            Integration.account_id == account_id,
            Integration.provider == IntegrationProvider.SALESFORCE,
        )
    )
    existing_integration = result.scalar_one_or_none()
    has_refresh_token = existing_integration and existing_integration.refresh_token

    # Determine the Salesforce OAuth host based on environment
    # IMPORTANT: For cross-org OAuth to work, authorization MUST go through
    # login.salesforce.com (production) or test.salesforce.com (sandbox).
    # The custom domain is only used for API calls AFTER authentication.
    # Salesforce returns the correct instance_url in the token response.

    # Store the user-provided custom domain for reference (used in state token)
    user_custom_domain = custom_domain

    if environment == "custom" and not custom_domain:
        raise HTTPException(400, "Custom domain required for custom environment")
    sf_host = _select_oauth_host(environment, custom_domain)
    if environment == "custom":
        log.info(
            "Using custom domain for OAuth: %s (from %s)",
            sf_host,
            custom_domain,
        )

    # Generate PKCE code verifier and challenge
    code_verifier = (
        base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8").rstrip("=")
    )
    code_challenge = (
        base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode("utf-8")).digest())
        .decode("utf-8")
        .rstrip("=")
    )
    state_id = secrets.token_urlsafe(32)
    await _store_oauth_state_payload(
        redis,
        state_id,
        {
            "account_id": str(account_id),
            "user_id": user_id,
            "environment": environment,
            "auth_host": sf_host,
            "custom_domain": user_custom_domain,
            "code_verifier": code_verifier,
            "created_at": datetime.utcnow().isoformat(),
        },
    )

    # Create signed state token for CSRF protection and session tracking
    state_data = {
        "account_id": str(account_id),
        "user_id": user_id,
        "environment": environment,
        "auth_host": sf_host,  # OAuth host (login.salesforce.com or test.salesforce.com)
        "custom_domain": user_custom_domain,  # User-provided domain for reference
        "nonce": secrets.token_urlsafe(16),
        "state_id": state_id,
        "exp": (datetime.utcnow() + timedelta(minutes=10)).timestamp(),
        "origin": request.headers.get("Referer") or str(request.client.host)
        if getattr(request, "client", None)
        else None,
    }

    # Use encryption key or JWT secret for signing (never fall back to a hardcoded
    # secret in production, or the state token becomes forgeable).
    secret_key = _require_signing_secret("OAuth state")
    state_token = jwt.encode(state_data, secret_key, algorithm="HS256")

    # Build OAuth authorization URL
    client_id = settings.SALESFORCE_CLIENT_ID or settings.SF_CLIENT_ID
    redirect_uri = _compute_redirect_uri(request)

    if not client_id:
        raise HTTPException(
            500, "SALESFORCE_CLIENT_ID not configured in environment variables"
        )

    # Build OAuth parameters with proper scopes
    params = {
        "response_type": "code",
        "client_id": client_id,
        "redirect_uri": redirect_uri,
        "scope": "api refresh_token",  # Minimal essential scopes
        "state": state_token,
        "code_challenge": code_challenge,
        "code_challenge_method": "S256",
    }

    # If we don't have a refresh token, force consent to ensure SF issues one
    if not has_refresh_token:
        params["prompt"] = "consent"
        log.info(
            "No refresh token found, adding prompt=consent to force new refresh token"
        )
    else:
        params["prompt"] = "login"  # Just login if we already have a refresh token

    auth_url = f"{sf_host}/services/oauth2/authorize?{urlencode(params)}"

    # Do NOT log the full auth URL: it contains a signed state token that includes
    # PKCE verifier and session metadata. Logging it risks token leakage via logs.
    log.info(
        "Generated Salesforce OAuth URL (account_id=%s env=%s host=%s prompt=%s)",
        account_id,
        environment,
        sf_host,
        params.get("prompt"),
    )
    log.debug("Salesforce OAuth redirect_uri=%s", redirect_uri)

    return {
        "auth_url": auth_url,
        "environment": environment,
        "host": sf_host,
        "redirect_uri": redirect_uri,
    }


@router.get("/redirect")
async def redirect_to_salesforce_auth(
    request: Request,
    environment: str = Query(
        "production", pattern="^(production|sandbox|developer|custom)$"
    ),
    custom_domain: Optional[str] = Query(None),
    user_id: str = Depends(require_user),
    account_id: str = Depends(require_account),
    db: AsyncSession = Depends(get_session),
    redis=Depends(get_redis),
):
    """Redirect-based start for OAuth (gconnector-style).

    Builds the same authorization URL as /connect and returns a 302 redirect so
    clients can simply navigate to this endpoint without parsing JSON first.
    """
    await _ensure_account(account_id, db)
    # Reuse the logic in start_salesforce_connect: copy minimal pieces here.
    # Determine SF host
    # Store user-provided custom domain for state token
    user_custom_domain = custom_domain

    # IMPORTANT: For cross-org OAuth, authorization MUST go through central login endpoints
    if environment == "custom" and not custom_domain:
        raise HTTPException(400, "Custom domain required for custom environment")
    sf_host = _select_oauth_host(environment, custom_domain)

    # PKCE
    code_verifier = (
        base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8").rstrip("=")
    )
    code_challenge = (
        base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode("utf-8")).digest())
        .decode("utf-8")
        .rstrip("=")
    )
    state_id = secrets.token_urlsafe(32)
    await _store_oauth_state_payload(
        redis,
        state_id,
        {
            "account_id": str(account_id),
            "user_id": user_id,
            "environment": environment,
            "auth_host": sf_host,
            "custom_domain": user_custom_domain,
            "code_verifier": code_verifier,
            "created_at": datetime.utcnow().isoformat(),
        },
    )

    # Signed state
    secret_key = _require_signing_secret("OAuth state")
    state_data = {
        "account_id": str(account_id),
        "user_id": user_id,
        "environment": environment,
        "auth_host": sf_host,
        "custom_domain": user_custom_domain,  # User-provided domain for reference
        "state_id": state_id,
        "exp": (datetime.utcnow() + timedelta(minutes=10)).timestamp(),
        "ts": int(time.time()),
        "origin": request.headers.get("Referer") or str(request.client.host)
        if getattr(request, "client", None)
        else None,
    }
    state_token = jwt.encode(state_data, secret_key, algorithm="HS256")

    client_id = settings.SALESFORCE_CLIENT_ID or settings.SF_CLIENT_ID
    redirect_uri = _compute_redirect_uri(request)
    if not client_id:
        raise HTTPException(
            500, "SALESFORCE_CLIENT_ID not configured in environment variables"
        )

    # Force consent if no refresh token yet
    has_refresh_token = False
    try:
        row = await db.execute(
            select(Integration).where(
                Integration.account_id == str(account_id),
                Integration.provider == IntegrationProvider.SALESFORCE,
            )
        )
        integ = row.scalar_one_or_none()
        has_refresh_token = bool(integ and integ.refresh_token)
    except Exception:
        has_refresh_token = False

    params = {
        "response_type": "code",
        "client_id": client_id,
        "redirect_uri": redirect_uri,
        "scope": "api refresh_token",  # Minimal essential scopes
        "state": state_token,
        "code_challenge": code_challenge,
        "code_challenge_method": "S256",
        "prompt": "login" if has_refresh_token else "consent",
    }

    auth_url = f"{sf_host}/services/oauth2/authorize?{urlencode(params)}"
    return RedirectResponse(url=auth_url, status_code=302)


@router.get("/callback", name="salesforce_oauth_callback")
async def oauth_callback(
    request: Request,
    code: Optional[str] = Query(None),
    state: str = Query(...),
    error: Optional[str] = Query(None),
    error_description: Optional[str] = Query(None),
    background_tasks: BackgroundTasks = BackgroundTasks(),
    db: AsyncSession = Depends(get_session),
    redis=Depends(get_redis),
):
    """Handle OAuth callback from Salesforce"""

    # Handle OAuth errors from Salesforce
    if error:
        log.error("Salesforce OAuth error: %s - %s", error, error_description)
        error_msg = error_description or error or "Salesforce authorization failed"
        if error_description and "Cross-org" in error_description:
            error_msg = "Salesforce authorization failed. Please try again or contact support."
        return _oauth_error_html(error_msg)

    if not code:
        return _oauth_error_html("Authorization code not received from Salesforce")

    try:
        # Verify and decode state token
        try:
            state_data = _decode_oauth_state_token(state)
            account_id = state_data["account_id"]
            user_id = state_data.get("user_id")
            environment = state_data.get("environment", "production")
            state_id = str(state_data.get("state_id") or "").strip()
            if not state_id:
                raise HTTPException(400, "Invalid state token")
            stored_state = await _consume_oauth_state_payload(redis, state_id)
            code_verifier = stored_state.get("code_verifier")
            if not code_verifier:
                raise HTTPException(400, "Invalid state token")
            if str(stored_state.get("account_id") or "") != str(account_id):
                log.warning(
                    "OAuth state account mismatch for state_id=%s token_account=%s stored_account=%s",
                    state_id,
                    account_id,
                    stored_state.get("account_id"),
                )
                raise HTTPException(400, "Invalid state token")
            user_id = stored_state.get("user_id") or user_id
            environment = stored_state.get("environment") or environment

            # Use the account_id from the signed state token as-is so that
            # OAuth connections are recorded against the same tenant that
            # initiated the flow (API key, bearer token, or dev fallback).
        except HTTPException:
            raise
        except Exception as e:
            log.error("Invalid state token: %s", e)
            raise HTTPException(400, "Invalid state token")

        await _ensure_account(account_id, db)

        auth_host = stored_state.get("auth_host") or state_data.get("auth_host")
        custom_domain = stored_state.get("custom_domain") or state_data.get(
            "custom_domain"
        )
        token_host_raw = auth_host or _select_oauth_host(environment, custom_domain)
        token_host = _normalize_salesforce_custom_domain(token_host_raw)
        if not token_host:
            log.error("Blocked invalid Salesforce token host: %s", token_host_raw)
            return _oauth_error_html(
                "Invalid Salesforce authorization host. Please retry connecting."
            )

        # Exchange authorization code for tokens
        # Use the same redirect URI that was used to initiate auth
        redirect_uri = _compute_redirect_uri(request)
        token_data = {
            "grant_type": "authorization_code",
            "code": code,
            "client_id": settings.SALESFORCE_CLIENT_ID or settings.SF_CLIENT_ID,
            "client_secret": settings.SALESFORCE_CLIENT_SECRET
            or settings.SF_CLIENT_SECRET,
            "redirect_uri": redirect_uri,
        }

        # Add PKCE verifier if present
        if code_verifier:
            token_data["code_verifier"] = code_verifier

        async with httpx.AsyncClient() as client:
            token_response = await client.post(
                f"{token_host}/services/oauth2/token", data=token_data, timeout=30
            )

        if not token_response.is_success:
            detail = None
            try:
                body = token_response.json()
                if isinstance(body, dict):
                    detail = body.get("error") or body.get("error_description")
            except Exception:
                detail = None
            detail = detail or _truncate_for_log(token_response.text, 200) or "unknown"
            log.error(
                "Token exchange failed: status=%s detail=%s",
                token_response.status_code,
                detail,
            )
            raise HTTPException(400, f"Token exchange failed: {detail}")

        tokens = token_response.json()
        log.info(f"Token response keys: {tokens.keys()}")
        log.info(
            f"Instance URL from tokens: {tokens.get('instance_url', 'NOT PROVIDED')}"
        )

        # Get user info from Salesforce
        instance_url = tokens.get("instance_url")
        if not instance_url:
            log.warning(
                "No instance_url in token response, will try to determine from user info"
            )
            # For developer orgs, extract from the ID URL if available
            if "id" in tokens:
                import re

                id_url = str(tokens.get("id") or "")
                # Prefer the /id/ form, but fall back to any https://<host> match.
                match = re.search(r"https://([^/]+)/id/", id_url) or re.search(
                    r"https://([^/]+)", id_url
                )
                if match:
                    domain = match.group(1)
                    if domain not in ["login.salesforce.com", "test.salesforce.com"]:
                        instance_url = f"https://{domain}"
                        log.info("Extracted instance URL from id: %s", instance_url)

        instance_url = (
            _normalize_salesforce_custom_domain(instance_url) if instance_url else None
        )
        if not instance_url:
            log.error(
                "Missing or invalid instance_url in Salesforce token exchange (account_id=%s)",
                account_id,
            )
            return _oauth_error_html(
                "Salesforce did not return a valid instance URL. Please retry connecting."
            )

        async with httpx.AsyncClient() as client:
            user_info_response = await client.get(
                f"{instance_url}/services/oauth2/userinfo",
                headers={"Authorization": f"Bearer {tokens['access_token']}"},
                timeout=30,
            )

        if user_info_response and not user_info_response.is_success:
            log.warning(
                "Failed to get user info from Salesforce: status=%s",
                user_info_response.status_code,
            )
            # Continue anyway, we have the tokens
            user_info = {}
        elif user_info_response:
            user_info = user_info_response.json()
        else:
            user_info = {}

        # Check if integration already exists for this account
        log.info("=== OAuth callback: Checking for existing integration ===")
        log.info(f"  account_id: {account_id}")
        log.info("  provider: SALESFORCE")
        existing_result = await db.execute(
            select(Integration).where(
                Integration.account_id == account_id,
                Integration.provider == IntegrationProvider.SALESFORCE,
            )
        )
        integration = existing_result.scalar_one_or_none()
        log.info(f"  Found existing: {integration is not None}")

        if not integration:
            # Create new integration
            integration = Integration(
                account_id=account_id, provider=IntegrationProvider.SALESFORCE
            )
            db.add(integration)

        # Update integration with new tokens and info
        integration.access_token = (
            encrypt_str(tokens["access_token"])
            if settings.ENCRYPTION_KEY
            else tokens["access_token"]
        )

        # Only update refresh_token if we got a new one (don't overwrite with None/empty)
        new_refresh = tokens.get("refresh_token")
        if new_refresh:
            integration.refresh_token = (
                encrypt_str(new_refresh) if settings.ENCRYPTION_KEY else new_refresh
            )
            log.info("Saved new refresh_token from OAuth callback")
        elif not integration.refresh_token:
            log.warning(
                "No refresh_token received and no existing refresh_token - user may need to reconnect with consent"
            )
        integration.instance_url = instance_url
        integration.status = IntegrationStatus.CONNECTED
        integration.scopes = [
            "api",
            "id",
            "refresh_token",
            "offline_access",
            "openid",
            "profile",
            "email",
            "web",
        ]
        # Salesforce tokens typically expire in 2 hours
        expires_in = tokens.get("expires_in", 7200)  # Default to 2 hours
        if isinstance(expires_in, str):
            expires_in = int(expires_in)
        integration.expires_at = datetime.utcnow() + timedelta(seconds=expires_in)
        integration.updated_at = datetime.utcnow()

        # Store additional metadata in config field and clear any reconnect flags
        integration.config = {
            "environment": environment,
            "auth_host": token_host,  # Store the auth host used for token exchange
            "org_id": user_info.get("organization_id"),
            "org_name": user_info.get(
                "organization_id"
            ),  # Will be updated on first API call
            "user_id": user_info.get("user_id"),
            "username": user_info.get("preferred_username") or user_info.get("email"),
            "connected_by_user_id": user_id,
            "connected_at": datetime.utcnow().isoformat(),
            # Clear any reconnect flags on successful connection
            "needs_reconnect": False,
            "reconnect_reason": None,
            "reconnect_message": None,
        }

        # Store IDs before commit to avoid lazy loading issues
        integration_id = str(integration.id) if integration.id else str(uuid.uuid4())
        if not integration.id:
            integration.id = integration_id

        await db.commit()

        # Read-after-write confirmation to verify the row is CONNECTED
        await db.refresh(integration)
        final_status = getattr(integration.status, "value", integration.status)
        log.info(f"Post-commit status for {account_id}: {final_status}")

        # Clear the gateway cache to ensure fresh status (with lock)
        if hasattr(SalesforceGateway, "_global_lock"):
            async with SalesforceGateway._global_lock:
                # Clear the specific account
                SalesforceGateway._instances.pop(account_id, None)
                # Also try clearing common dev account variations
                SalesforceGateway._instances.pop(settings.DEV_ACCOUNT_ID, None)
                SalesforceGateway._instances.pop(str(account_id), None)

                # Log what was cleared
                log.info(
                    f"Cleared gateway cache for tenant {account_id} after OAuth callback"
                )
                log.info("Also cleared dev-account-id cache entry if present")

                # In development, clear ALL cached instances to be safe
                if settings.ENV.lower() in ["development", "dev"]:
                    cleared_count = len(SalesforceGateway._instances)
                    SalesforceGateway._instances.clear()
                    log.info(
                        f"Development mode: Cleared all {cleared_count} cached gateway instances"
                    )

        # Add delay to ensure DB commit completes and is visible
        import asyncio

        await asyncio.sleep(0.5)
        log.info("Waited 0.5s for DB commit to complete")
        log.info(
            f"Cache state after clear and wait: {list(SalesforceGateway._instances.keys()) if hasattr(SalesforceGateway, '_instances') else 'no cache'}"
        )

        log.info(
            f"Successfully connected Salesforce for account {account_id}, org: {user_info.get('organization_id')}"
        )

        # Track successful connection
        track_connection(
            account_id=account_id,
            success=True,
            duration_ms=0,  # OAuth flow duration could be tracked if needed
        )

        if settings.AI_SCHEMA_INDEX_ON_CONNECT:

            async def _warm_schema_index(target_account: str) -> None:
                try:
                    async with AsyncSessionLocal() as session:
                        gateway_instance = await SalesforceGateway.for_tenant(
                            str(target_account), session
                        )
                        indexer = SalesforceSchemaIndexer(session)
                        await indexer.ensure_quick_index(
                            org_id=str(target_account), gateway=gateway_instance
                        )
                except Exception as exc:
                    log.warning(
                        "Failed to build schema index for %s after OAuth: %s",
                        target_account,
                        exc,
                    )

            asyncio.create_task(_warm_schema_index(account_id))

        # Bootstrap organization profile from Salesforce data
        try:
            # Import OrgProfile here to avoid circular imports
            from ...models import OrgProfile

            # Use the existing db session instead of creating a new one
            # This avoids the greenlet context issue
            result = await db.execute(
                select(OrgProfile).where(OrgProfile.account_id == account_id)
            )
            org_profile = result.scalar_one_or_none()

            if not org_profile:
                org_profile = OrgProfile(
                    account_id=account_id,
                    org_id=user_info.get("organization_id"),
                    instance_url=integration.instance_url,
                    username=user_info.get("preferred_username")
                    or user_info.get("email"),
                )
                db.add(org_profile)
            else:
                # Update with Salesforce info
                org_profile.org_id = user_info.get("organization_id")
                org_profile.instance_url = integration.instance_url
                org_profile.username = user_info.get(
                    "preferred_username"
                ) or user_info.get("email")
                org_profile.updated_at = datetime.utcnow()

            await db.commit()
            log.info(
                f"Updated org profile for account {account_id} with Salesforce info"
            )

            # Also bootstrap business context from signup info
            context_service = ContextService(db)
            username = user_info.get("preferred_username") or user_info.get("email", "")
            org_name = user_info.get("name", "")

            if username:
                await context_service.bootstrap_from_signup(
                    account_id=account_id, email=username, company_name=org_name
                )
                log.info(f"Bootstrapped business context for account {account_id}")

        except Exception as e:
            log.warning(f"Failed to bootstrap org profile: {e}")
            # Non-critical, continue anyway

        # Trigger field discovery in background with stored IDs
        background_tasks.add_task(
            trigger_discovery_and_survey, integration_id, account_id
        )

        # Return HTML that closes the popup and notifies the parent window
        frontend_origin = (
            settings.FRONTEND_ORIGIN
        )  # Defaults to http://localhost:3004 in settings
        html_content = f"""
        <!DOCTYPE html>
        <html>
        <head>
            <title>Salesforce Connected</title>
            <style>
                body {{
                    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
                    display: flex;
                    justify-content: center;
                    align-items: center;
                    height: 100vh;
                    margin: 0;
                    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
                    color: white;
                }}
                .message {{
                    text-align: center;
                    padding: 2rem;
                    background: rgba(255, 255, 255, 0.1);
                    border-radius: 10px;
                    backdrop-filter: blur(10px);
                }}
                h1 {{ margin: 0 0 1rem 0; }}
                p {{ margin: 0; opacity: 0.9; }}
            </style>
        </head>
        <body>
            <div class="message">
                <h1>✅ Successfully Connected!</h1>
                <p>You can close this window now.</p>
            </div>
            <script>
                // Notify parent window
                if (window.opener) {{
                    window.opener.postMessage({{
                        type: 'salesforce_connected',
                        success: true
                    }}, '{frontend_origin}');
                }}
                // Auto-close after a short delay
                setTimeout(() => {{
                    window.close();
                }}, 1500);
            </script>
        </body>
        </html>
        """

        return HTMLResponse(content=html_content)

    except HTTPException as exc:
        detail = getattr(exc, "detail", None) or str(exc)
        log.error("Salesforce OAuth callback error: %s", detail)
        return _oauth_error_html(
            "Salesforce authorization failed. Please try connecting again."
        )
    except Exception as e:
        import traceback

        log.error(f"Salesforce OAuth callback error: {e}")
        log.error(f"Full traceback: {traceback.format_exc()}")
        return _oauth_error_html(
            "Salesforce authorization failed. Please try connecting again."
        )


@router.get("/status")
async def get_integration_status(
    fresh: bool = Query(False, description="Force cache bust if fresh=true"),
    wait_until: Optional[str] = Query(
        None, description="Wait until status is 'connected' or 'disconnected'"
    ),
    timeout_ms: int = Query(
        1500, ge=0, le=5000, description="Max time to wait in milliseconds"
    ),
    account_id: str = Depends(require_account),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    """Get detailed Salesforce integration status including reconnect state

    Uses centralized SalesforceGateway for all Salesforce operations.
    The gateway handles token refresh and circuit breaker status automatically.

    Args:
        fresh: If fresh=1, force exact tenant resolution from auth and clear cache
        account_id: Account ID from auth dependency (canonical source)
        db: Database session
    """

    # Use the account_id from the centralized auth dependency
    acct = account_id
    log.info(f"/status for account_id={acct} fresh={fresh} wait_until={wait_until}")

    async def _db_read():
        """Read integration from DB and return (status, integration)"""
        row = await db.execute(
            select(Integration).where(
                Integration.account_id == acct,
                Integration.provider == IntegrationProvider.SALESFORCE,
            )
        )
        integ = row.scalar_one_or_none()
        if not integ:
            return None, None
        status_value = getattr(integ.status, "value", integ.status)
        return status_value, integ

    # 1) DB-first with optional wait-until for read-after-write consistency
    if wait_until in {"connected", "disconnected"}:
        start_time = time.monotonic()
        deadline = start_time + (timeout_ms / 1000.0)
        target = (
            StatusValues.CONNECTED
            if wait_until == "connected"
            else StatusValues.DISCONNECTED
        )
        backoff_ms = 50  # Start with 50ms
        poll_count = 0

        # Telemetry: Track wait-until usage
        telemetry_data = {
            "account_id": acct,
            "wait_until": wait_until,
            "timeout_ms": timeout_ms,
            "start_time": start_time,
        }

        log.info(f"Starting wait-until={target} for {acct}, timeout={timeout_ms}ms")

        while True:
            poll_count += 1
            status_value, integ = await _db_read()
            # Normalize for comparison (handle both "connected" and "CONNECTED")
            if status_value and status_value.upper() == target:
                elapsed_ms = int((time.monotonic() - start_time) * 1000)

                # Telemetry: Record successful wait completion
                telemetry_data.update(
                    {
                        "elapsed_ms": elapsed_ms,
                        "poll_count": poll_count,
                        "success": True,
                        "final_status": status_value,
                    }
                )
                log.info(
                    f"Wait-until satisfied after {elapsed_ms}ms: found {target} for {acct} (polls={poll_count})"
                )
                log.info(f"Wait-until telemetry: {telemetry_data}")

                # Track metrics
                track_wait_until(
                    account_id=acct,
                    wait_until=wait_until,
                    timeout_ms=timeout_ms,
                    elapsed_ms=elapsed_ms,
                    poll_count=poll_count,
                    success=True,
                )
                if status_value and status_value.upper() == StatusValues.CONNECTED:
                    return {
                        "connected": True,
                        "needs_reconnect": False,
                        "instance_url": integ.instance_url,
                        "expires_in_sec": None,
                        "message": "Connection active",
                        "reason": "healthy",
                    }
                else:
                    return {
                        "connected": False,
                        "needs_reconnect": False,
                        "message": "Salesforce integration has been disconnected",
                        "reason": "disconnected",
                    }

            if time.monotonic() >= deadline:
                # Telemetry: Record timeout
                elapsed_ms = int((time.monotonic() - start_time) * 1000)
                telemetry_data.update(
                    {
                        "elapsed_ms": elapsed_ms,
                        "poll_count": poll_count,
                        "success": False,
                        "timeout": True,
                        "final_status": status_value,
                    }
                )
                log.warning(
                    f"Wait-until timeout after {elapsed_ms}ms for {acct} (polls={poll_count})"
                )
                log.info(f"Wait-until telemetry: {telemetry_data}")

                # Track metrics
                track_wait_until(
                    account_id=acct,
                    wait_until=wait_until,
                    timeout_ms=timeout_ms,
                    elapsed_ms=elapsed_ms,
                    poll_count=poll_count,
                    success=False,
                )
                break

            # Check deadline BEFORE sleeping to avoid overshooting timeout
            now = time.monotonic()
            if now >= deadline:
                # Already handled timeout above, just break
                break

            # Sleep only for remaining time or backoff, whichever is shorter
            remaining = deadline - now
            sleep_time = min(backoff_ms / 1000.0, remaining)
            if sleep_time > 0:
                await asyncio.sleep(sleep_time)

            # Exponential backoff: 50ms -> 75ms -> 112ms -> ... -> max 500ms
            backoff_ms = min(int(backoff_ms * 1.5), 500)

    # 2) One DB read without waiting (fast path)
    status_value, integ = await _db_read()
    # Normalize status for comparison (DB may store lowercase/mixed case)
    if status_value and status_value.upper() == StatusValues.CONNECTED:
        return {
            "connected": True,
            "needs_reconnect": False,
            "instance_url": integ.instance_url,
            "expires_in_sec": None,
            "message": "Connection active",
            "reason": "healthy",
        }
    elif status_value and status_value.upper() == StatusValues.DISCONNECTED:
        return {
            "connected": False,
            "needs_reconnect": False,
            "message": "Salesforce integration has been disconnected",
            "reason": "disconnected",
        }

    # 3) Optional cache bust (post-OAuth)
    if fresh and hasattr(SalesforceGateway, "_global_lock"):
        async with SalesforceGateway._global_lock:
            SalesforceGateway._instances.pop(acct, None)
            log.info(f"Cleared gateway cache for tenant {acct} due to fresh=true")

    # 4) Gateway fallback (steady-state / refresh)
    try:
        gw = await SalesforceGateway.for_tenant(acct, db)
        return await gw.get_status()
    except Exception as e:
        log.error(f"Failed to get Salesforce status for {acct}: {e}")
        return {
            "connected": False,
            "needs_reconnect": True,
            "reason": "error",
            "message": "Status unavailable",
        }


@router.post("/embed-token", response_model=SalesforceEmbedTokenResponse)
async def create_embed_token(
    request: Request,
    db: AsyncSession = Depends(get_session),
    user_id: str = Depends(require_user),
    account_id: str = Depends(require_account),
):
    """Issue a short-lived embed token for trusted surfaces (e.g., Google Sheets)."""

    ttl_seconds = max(60, int(settings.GSHEETS_EMBED_TOKEN_TTL_SECONDS or 600))
    expires_at = datetime.utcnow() + timedelta(seconds=ttl_seconds)

    # Validate Salesforce connectivity before issuing tokens
    try:
        gateway = await SalesforceGateway.for_tenant(account_id, db)
        status = await gateway.get_status()
        if not status.get("connected"):
            raise HTTPException(
                status_code=409,
                detail="Salesforce is not connected for this account",
            )
    except HTTPException:
        raise
    except Exception as exc:
        log.error("Embed token issuance failed for account %s: %s", account_id, exc)
        raise HTTPException(500, "Unable to verify Salesforce integration") from exc

    secret = settings.ENCRYPTION_KEY or settings.JWT_SECRET_KEY
    if not secret:
        secret = _require_signing_secret("Embed token")
    payload = {
        "iss": "foundrymatch",
        "aud": "gsheets",
        "sub": user_id,
        "account_id": account_id,
        "email": user_id,  # Use user_id as email for now
        "scope": "salesforce.embed",
        "exp": expires_at,
        "iat": datetime.utcnow(),
        "jti": uuid.uuid4().hex,
    }

    token = jwt.encode(payload, secret, algorithm="HS256")
    response_body = SalesforceEmbedTokenResponse(
        token=token,
        expires_at=expires_at.isoformat() + "Z",
        expires_in=ttl_seconds,
        connected=True,
        account_id=account_id,
    )

    response = JSONResponse(response_body.model_dump())
    response.headers["Cache-Control"] = "no-store"
    return response


@router.post("/test")
async def test_connection(
    user_id: str = Depends(require_user),
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
) -> Dict[str, Any]:
    """Test the Salesforce connection - returns ok: true on success

    Uses centralized SalesforceGateway for connection testing.
    """

    integration = None
    status: Dict[str, Any] = {}

    try:
        raw_status = await gateway.get_status()
        if isinstance(raw_status, dict):
            status = raw_status
        else:
            log.warning(
                "Gateway.get_status returned unexpected payload: %r", raw_status
            )
            status = {}

        connected = bool(status.get("connected"))
        needs_reconnect = bool(status.get("needs_reconnect"))
        message = status.get("message") or (
            "Connection is active and working!"
            if connected
            else "Unable to verify connection"
        )

        try:
            await gateway._ensure_integration()  # type: ignore[attr-defined]
        except Exception as integration_err:
            log.debug(
                "Failed to eagerly load integration while testing connection: %s",
                integration_err,
            )
        finally:
            integration = getattr(gateway, "_integration", None)

        environment = "production"
        org_id = None
        username = None
        instance_url = status.get("instance_url")

        if integration is not None:
            cfg = getattr(integration, "config", None) or {}
            environment = cfg.get("environment", environment)
            org_id = cfg.get("org_id") or cfg.get("organization_id")
            username = cfg.get("username") or cfg.get("user_email")
            if not instance_url:
                instance_url = getattr(integration, "instance_url", None)

        return {
            "ok": connected,
            "connected": connected,
            "needs_reconnect": needs_reconnect,
            "status": "success" if connected else "error",
            "message": message,
            "org_id": org_id,
            "username": username,
            "instance_url": instance_url,
            "environment": environment,
        }

    except Exception as e:
        log.exception("Error in test_connection")
        return {
            "ok": False,
            "status": "error",
            "message": "Error checking connection status",
            "error": str(e),
            "details": str(e),
            "connected": False,
            "needs_reconnect": False,
            "instance_url": status.get("instance_url") if status else None,
            "username": getattr(integration, "user_email", None)
            if integration
            else None,
            "org_id": getattr(integration, "org_id", None) if integration else None,
        }


@router.post("/disconnect")
async def disconnect_salesforce(
    account_id: str = Depends(require_account), db: AsyncSession = Depends(get_session)
) -> Dict[str, str]:
    """Disconnect Salesforce integration"""

    log.info(f"Disconnect called for account: {account_id}")

    result = await db.execute(
        select(Integration).where(
            Integration.account_id == str(account_id),
            Integration.provider == IntegrationProvider.SALESFORCE,
        )
    )
    integration = result.scalar_one_or_none()

    if integration:
        token_host = _resolve_salesforce_auth_host(integration)
        refresh_token = SalesforceGateway.safe_decrypt(integration.refresh_token)
        access_token = SalesforceGateway.safe_decrypt(integration.access_token)
        token_to_revoke = refresh_token or access_token
        if token_to_revoke:
            await _revoke_salesforce_token(token_to_revoke, token_host)

        # Clear sensitive data
        integration.status = IntegrationStatus.DISCONNECTED
        integration.access_token = None
        integration.refresh_token = None
        integration.expires_at = None

        # Keep config for audit trail but mark as disconnected
        if integration.config:
            integration.config["disconnected_at"] = datetime.utcnow().isoformat()
            integration.config["disconnected_by"] = "dev_user"  # In dev mode

        await db.commit()

        # Clear the gateway cache to ensure fresh status (with lock)
        if hasattr(SalesforceGateway, "_global_lock"):
            async with SalesforceGateway._global_lock:
                # Clear the specific account and common variations
                SalesforceGateway._instances.pop(account_id, None)
                SalesforceGateway._instances.pop(settings.DEV_ACCOUNT_ID, None)
                SalesforceGateway._instances.pop(str(account_id), None)
                log.info(
                    f"Cleared gateway cache for tenant {account_id} after disconnect"
                )

                # In development, clear ALL cached instances to be safe
                if settings.ENV.lower() in ["development", "dev"]:
                    cleared_count = len(SalesforceGateway._instances)
                    SalesforceGateway._instances.clear()
                    log.info(
                        f"Development mode: Cleared all {cleared_count} cached gateway instances after disconnect"
                    )

        log.info(f"Salesforce disconnected for account {account_id}")

        return {"status": "disconnected", "message": "Salesforce has been disconnected"}

    return {"status": "not_found", "message": "No Salesforce integration found"}


# Duplicate route removed - using get_integration_status above
# which now uses the centralized SalesforceGateway


async def refresh_token(integration: Integration, db: AsyncSession) -> bool:
    """Helper function to refresh Salesforce OAuth token"""

    if not integration.refresh_token:
        return False

    try:
        refresh_token_value = (
            decrypt_str(integration.refresh_token)
            if settings.ENCRYPTION_KEY
            else integration.refresh_token
        )

        token_host = _resolve_salesforce_auth_host(integration)

        async with httpx.AsyncClient() as client:
            refresh_response = await client.post(
                f"{token_host}/services/oauth2/token",
                data={
                    "grant_type": "refresh_token",
                    "refresh_token": refresh_token_value,
                    "client_id": settings.SALESFORCE_CLIENT_ID or settings.SF_CLIENT_ID,
                    "client_secret": settings.SALESFORCE_CLIENT_SECRET
                    or settings.SF_CLIENT_SECRET,
                },
                timeout=30,
            )

        if refresh_response.is_success:
            new_tokens = refresh_response.json()
            integration.access_token = (
                encrypt_str(new_tokens["access_token"])
                if settings.ENCRYPTION_KEY
                else new_tokens["access_token"]
            )
            integration.expires_at = datetime.utcnow() + timedelta(
                seconds=new_tokens.get("expires_in", 7200)
            )
            integration.updated_at = datetime.utcnow()

            await db.commit()
            return True
        else:
            detail = None
            try:
                body = refresh_response.json()
                if isinstance(body, dict):
                    detail = body.get("error") or body.get("error_description")
            except Exception:
                detail = None
            detail = detail or _truncate_for_log(refresh_response.text, 200) or "unknown"
            log.error(
                "Token refresh failed: status=%s detail=%s",
                refresh_response.status_code,
                detail,
            )
            return False

    except Exception as e:
        log.error(f"Token refresh failed: {e}")

    return False


async def trigger_discovery_and_survey(integration_id: str, account_id: str):
    """Trigger field discovery and instant survey after OAuth completes."""
    try:
        import httpx

        # First trigger discovery
        async with httpx.AsyncClient() as client:
            # Trigger discovery - pass integration_id as query parameter
            discovery_response = await client.post(
                f"http://localhost:8000/api/v2/discovery/trigger-on-connect?integration_id={integration_id}",
                timeout=10,
            )
            if discovery_response.is_success:
                log.info(f"Field discovery triggered for integration {integration_id}")
            else:
                log.warning(f"Failed to trigger discovery: {discovery_response.text}")

            # Then trigger instant survey (it will use discovered fields if available)
            survey_response = await client.post(
                f"http://localhost:8000/api/v2/surveys/trigger-after-oauth?integration_id={integration_id}",
                timeout=10,
            )
            if survey_response.is_success:
                log.info(f"Instant survey triggered for integration {integration_id}")
            else:
                log.warning(f"Failed to trigger instant survey: {survey_response.text}")

    except Exception as e:
        log.error(f"Error triggering discovery and survey: {e}")
        # Don't raise - this is a background task


async def trigger_instant_survey(integration_id: str, account_id: str):
    """Legacy: Trigger instant survey after OAuth completes."""
    try:
        import httpx

        async with httpx.AsyncClient() as client:
            response = await client.post(
                "http://localhost:8000/api/v2/surveys/trigger-after-oauth",
                json={"integration_id": integration_id},
                timeout=10,
            )
            if response.is_success:
                log.info(f"Instant survey triggered for integration {integration_id}")
            else:
                log.warning(f"Failed to trigger instant survey: {response.text}")
    except Exception as e:
        log.error(f"Error triggering instant survey: {e}")
        # Don't raise - this is a background task


@router.get("/ping")
async def sf_ping(
    user_id: str = Depends(require_user),
    db: AsyncSession = Depends(get_session),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
) -> Dict[str, Any]:
    """Quick smoke test to verify Salesforce connection is working"""
    try:
        # SF gateway is injected via dependency - no need to check integration

        # Make a simple SOQL query to test the connection (Organization always exists)
        data = await sf.soql("SELECT Id, OrganizationType FROM Organization LIMIT 1")

        # Return success if query worked
        return {"ok": True, "test_query": "Organization query successful"}

    except Exception as e:
        log.error(f"Salesforce ping failed: {e}")
        return {"ok": False, "error": str(e)}


# Data access endpoints with RBAC
@api_router.post("/query")
async def execute_soql_query(
    request: Request,
    db: AsyncSession = Depends(get_session),
    user_id: str = Depends(require_user),
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Execute a SOQL query against Salesforce with security enforcement"""
    try:
        body = await request.json()
        soql = body.get("soql")

        if not soql:
            raise HTTPException(400, "SOQL query required")

        # Get client IP for audit logging
        client_ip = request.client.host if request.client else "unknown"

        # Check user role for admin privileges
        # SECURITY: Never accept admin_override from client request
        # Note: With Clerk auth, role checking would need to be implemented separately
        user_is_admin = False
        admin_override = False

        # Role checking disabled for now - would need to query user from DB or get role from Clerk token
        if False:  # Placeholder for role checking
            user_is_admin = False

            # Server-side determination of sensitive queries
            # Admin override only for sensitive operations, never client-controlled
            if user_is_admin:
                # Check if query is sensitive (requires admin)
                query_lower = soql.lower()
                sensitive_objects = [
                    "case",
                    "emailmessage",
                    "attachment",
                    "document",
                    "contentdocument",
                ]
                sensitive_fields = ["body", "description", "textbody", "htmlbody"]

                # Check for sensitive objects
                is_sensitive = any(obj in query_lower for obj in sensitive_objects)

                # Check for sensitive fields
                if not is_sensitive:
                    is_sensitive = any(
                        field in query_lower for field in sensitive_fields
                    )

                # Check for wide LIKE patterns
                if not is_sensitive and "like" in query_lower:
                    # Wide LIKE patterns (e.g., LIKE '%abc%')
                    is_sensitive = bool(
                        re.search(r"like\s+['\"]%.*%['\"]", query_lower)
                    )

                admin_override = is_sensitive

                # Log admin actions for sensitive queries
                if admin_override:
                    log.warning(
                        f"ADMIN_QUERY (sensitive) by {user_id} from {client_ip}: {soql[:100]}..."
                    )

        # Determine if we should apply a scoped plan allowlist for common objects
        scope = evaluate_safe_query_scope(soql)
        canonical_object = scope.get("canonical_object")
        disallowed_fields = scope.get("disallowed_fields") or []
        plan_allowlist = scope.get("plan_allowlist") or {}
        if canonical_object and disallowed_fields:
            log.warning(
                "Blocking query for %s; fields outside allowlist: %s",
                canonical_object,
                disallowed_fields,
            )
            raise HTTPException(
                status_code=403,
                detail=(
                    "Query contains disallowed fields for "
                    + str(canonical_object)
                    + ": "
                    + ", ".join(disallowed_fields)
                ),
            )

        # Execute query through gateway with security checks
        result = await gateway.soql(
            soql,
            admin_override=admin_override,  # Server-controlled only
            user=user_id,
            ip=client_ip,
            user_is_admin=user_is_admin,
            plan_allowlist=plan_allowlist,
        )

        log.info(
            f"Query by {user_id} returned {len(result.get('records', []))} records"
        )
        return result

    except HTTPException:
        raise
    except Exception as e:
        log.error(f"SOQL query failed for {user_id}: {e}")
        # Don't expose internal errors to client
        if "SecurityViolation" in str(e.__class__.__name__):
            raise HTTPException(403, "Query blocked by security policy")
        raise HTTPException(500, "Query failed")


@api_router.post("/query/next")
async def fetch_next_query_page(
    payload: dict = Body(...),
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Fetch the next page of a SOQL query using nextRecordsUrl."""
    next_url = (
        (payload or {}).get("next_url")
        or (payload or {}).get("nextUrl")
        or (payload or {}).get("nextRecordsUrl")
    )
    if not next_url:
        raise HTTPException(400, "next_url is required")
    try:
        return await gateway.fetch_next(next_url)
    except HTTPException:
        raise
    except Exception as exc:
        log.error(f"SOQL query pagination failed: {exc}")
        raise HTTPException(500, "Failed to fetch next page")


@api_router.get("/admin/security/metrics")
async def get_security_metrics(
    hours: int = Query(24, description="Hours to look back"),
    _: AdminContext = Depends(require_admin),
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Get security metrics for dashboard (admin only)"""
    try:
        metrics = await gateway.get_security_metrics(hours)
        return metrics
    except Exception as e:
        log.error(f"Failed to get security metrics: {e}")
        raise HTTPException(500, "Failed to retrieve security metrics")


@api_router.post("/admin/emergency/revoke-all")
async def emergency_revoke_all_tokens(
    reason: str = Query(..., description="Reason for emergency revocation"),
    admin: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
):
    """Emergency token revocation - kills all Salesforce connections"""
    # Super-admin only in production: this endpoint affects *all* tenants.
    if settings.ENV.lower() not in ("dev", "development"):
        admin_email = (admin.email or "").lower().strip()
        if not admin_email or admin_email not in ADMIN_EMAILS:
            raise HTTPException(403, "Super admin access required for emergency actions")

    try:
        # Log the emergency action
        log.critical(
            "EMERGENCY TOKEN REVOCATION initiated by user=%s email=%s: %s",
            admin.user_id,
            admin.email,
            reason,
        )

        # Get all Salesforce integrations
        result = await db.execute(
            select(Integration).where(
                Integration.provider == IntegrationProvider.SALESFORCE,
                Integration.status == IntegrationStatus.CONNECTED,
            )
        )
        integrations = result.scalars().all()

        revoked_count = 0
        failed_tenants = []

        for integration in integrations:
            try:
                token_host = _resolve_salesforce_auth_host(integration)
                tokens_to_revoke: List[str] = []
                if integration.refresh_token:
                    refresh_token = (
                        decrypt_str(integration.refresh_token)
                        if settings.ENCRYPTION_KEY
                        else integration.refresh_token
                    )
                    if refresh_token:
                        tokens_to_revoke.append(str(refresh_token))
                if integration.access_token:
                    access_token = (
                        decrypt_str(integration.access_token)
                        if settings.ENCRYPTION_KEY
                        else integration.access_token
                    )
                    if access_token:
                        tokens_to_revoke.append(str(access_token))
                for token in dict.fromkeys(tokens_to_revoke):
                    try:
                        await _revoke_salesforce_token(token, token_host)
                    except Exception as revoke_exc:
                        log.warning(
                            "Best-effort revoke failed for account %s: %s",
                            integration.account_id,
                            revoke_exc,
                        )

                # Clear tokens
                integration.access_token = None
                integration.refresh_token = None
                integration.status = IntegrationStatus.NEEDS_RECONNECT
                integration.last_error = f"Emergency revocation: {reason}"
                integration.updated_at = datetime.utcnow()

                revoked_count += 1

                # Clear from gateway cache if exists (with lock)
                tenant_id = integration.account_id
                if hasattr(SalesforceGateway, "_instances") and hasattr(
                    SalesforceGateway, "_global_lock"
                ):
                    async with SalesforceGateway._global_lock:
                        SalesforceGateway._instances.pop(tenant_id, None)

            except Exception as e:
                log.error(f"Failed to revoke tokens for {integration.account_id}: {e}")
                failed_tenants.append(integration.account_id)

        # Commit all changes
        await db.commit()

        # Alert all admins
        alert_message = {
            "event": "EMERGENCY_TOKEN_REVOCATION",
            "initiated_by": admin.user_id,
            "initiated_by_email": admin.email,
            "reason": reason,
            "timestamp": datetime.utcnow().isoformat(),
            "revoked_count": revoked_count,
            "failed_tenants": failed_tenants,
        }

        # Send alert to webhook if configured
        from ... import settings as app_settings

        webhook_url = app_settings.SECURITY_ALERT_WEBHOOK
        if webhook_url:
            async with httpx.AsyncClient() as client:
                await client.post(
                    webhook_url,
                    json={
                        "text": f"EMERGENCY: All Salesforce tokens revoked by {admin.user_id}\\nReason: {reason}\\nAffected: {revoked_count} tenants"
                    },
                    timeout=5.0,
                )

        log.critical(f"Emergency revocation complete: {json.dumps(alert_message)}")

        return {
            "status": "success",
            "revoked_count": revoked_count,
            "failed_tenants": failed_tenants,
            "reason": reason,
            "initiated_by": admin.user_id,
            "initiated_by_email": admin.email,
            "timestamp": datetime.utcnow().isoformat(),
        }

    except Exception as e:
        log.error(f"Emergency revocation failed: {e}")
        raise HTTPException(
            status_code=500,
            detail="Emergency procedure failed. Check server logs for details.",
        )


@api_router.post("/admin/emergency/test-environment")
async def create_pen_test_environment(
    admin: AdminContext = Depends(require_admin),
    db: AsyncSession = Depends(get_session),
):
    """Create isolated test environment for penetration testing"""
    try:
        # Create test tenant with restricted access
        test_config = {
            "tenant_id": f"pentest_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}",
            "salesforce_sandbox": "test-pentest",
            "allowed_ips": [os.getenv("PENTEST_IP", "127.0.0.1")],
            "data_scope": "synthetic_only",
            "expires": (datetime.utcnow() + timedelta(days=7)).isoformat(),
            "created_by": admin.user_id,
            "restrictions": {
                "max_queries": 1000,
                "max_records": 100,
                "blocked_objects": ["Case", "EmailMessage", "Attachment"],
                "read_only": True,
            },
        }

        log.info(f"Pen test environment created: {json.dumps(test_config)}")

        return test_config

    except Exception as e:
        log.error(f"Failed to create pen test environment: {e}")
        raise HTTPException(500, "Failed to create test environment")


@api_router.get("/schema/{object_type}")
async def get_object_schema(
    object_type: str,
    db: AsyncSession = Depends(get_session),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
    soft: bool = Query(
        True, description="Return exists:false instead of 500 for missing objects"
    ),
):
    """Get the schema/field definitions for a Salesforce object"""
    try:
        # Check for mock mode first
        use_mock = os.getenv("USE_MOCK_SALESFORCE", "").lower() == "true"

        if use_mock:
            log.info(f"Using mock schema for {object_type} (USE_MOCK_SALESFORCE=true)")
            # Include exists flag in mock for consistency
            mock = get_mock_schema(object_type)
            if isinstance(mock, dict):
                mock["exists"] = True
            return mock

        # Use real Salesforce connection via gateway
        log.info(f"Fetching real schema for {object_type}")

        # Get object description
        schema = await sf.describe(object_type)

        # Filter to return only useful fields for the UI
        filtered_fields = []
        for field in schema.get("fields", []):
            # Only include fields that are useful for filtering/searching
            if (
                field.get("filterable", False)
                and not field.get("calculated", False)
                and field.get("type")
                in [
                    "string",
                    "email",
                    "phone",
                    "url",
                    "picklist",
                    "multipicklist",
                    "date",
                    "datetime",
                    "boolean",
                    "reference",
                    "double",
                    "int",
                    "currency",
                    "percent",
                ]
            ):
                filtered_fields.append(
                    {
                        "name": field["name"],
                        "label": field["label"],
                        "type": field["type"],
                        "length": field.get("length", 0),
                        "custom": field.get("custom", False),
                        "referenceTo": field.get("referenceTo", []),
                        "picklistValues": field.get("picklistValues", []),
                        "filterable": field.get("filterable", False),
                    }
                )

        return {
            "exists": True,
            "name": schema.get("name"),
            "label": schema.get("label"),
            "fields": filtered_fields,
        }

    except Exception as e:
        # Soft-fail on missing objects so the UI can degrade gracefully
        msg = str(e) if e else ""
        status = None
        try:
            status = getattr(
                getattr(e, "response", None), "status_code", None
            ) or getattr(e, "status_code", None)
        except Exception:
            status = None
        is_missing = (
            status == 404
            or ("INVALID_TYPE" in msg)
            or ("NOT_FOUND" in msg)
            or ("404" in msg)
        )
        if soft and is_missing:
            return {
                "exists": False,
                "name": object_type,
                "label": object_type,
                "fields": [],
            }
        log.error(f"Schema retrieval failed for {object_type}: {e}")
        raise HTTPException(500, "Schema retrieval failed")


def get_mock_query_results(soql: str) -> dict:
    """Generate mock Salesforce query results for development mode"""
    import random
    import re

    # Parse the SOQL to understand what's being requested
    soql_upper = soql.upper()

    # Extract object type from FROM clause
    from_match = re.search(r"FROM\s+(\w+)", soql_upper)
    if not from_match:
        return {"records": [], "totalSize": 0, "done": True}

    object_type = from_match.group(1)

    # Extract IDs from WHERE IN clause
    id_match = re.search(r"WHERE\s+ID\s+IN\s*\(([^)]+)\)", soql_upper)
    if id_match:
        # Parse IDs from the IN clause
        ids_str = id_match.group(1)
        ids = re.findall(r"'([A-Za-z0-9]+)'", ids_str)

        # Generate mock records for requested IDs
        records = []
        for rec_id in ids:
            if object_type == "LEAD":
                records.append(
                    {
                        "Id": rec_id,
                        "Name": f"Test Lead {rec_id[:4]}",
                        "FirstName": f"John{rec_id[:2]}",
                        "LastName": f"Smith{rec_id[2:4]}",
                        "Company": f"Test Corp {rec_id[:3]}",
                        "Email": f"test{rec_id[:4]}@example.com",
                        "Phone": f"555-{rec_id[:4]}",
                        "City": random.choice(["San Francisco", "New York", "Austin"]),
                        "State": random.choice(["CA", "NY", "TX"]),
                        "LastModifiedDate": f"2024-{random.randint(1,12):02d}-{random.randint(1,28):02d}T12:00:00.000Z",
                    }
                )
            elif object_type == "CONTACT":
                records.append(
                    {
                        "Id": rec_id,
                        "Name": f"Contact {rec_id[:4]}",
                        "FirstName": f"Jane{rec_id[:2]}",
                        "LastName": f"Doe{rec_id[2:4]}",
                        "Email": f"contact{rec_id[:4]}@example.com",
                        "AccountId": f"001{rec_id[:12]}",
                        "LastModifiedDate": f"2024-{random.randint(1,12):02d}-{random.randint(1,28):02d}T12:00:00.000Z",
                    }
                )
            elif object_type == "ACCOUNT":
                records.append(
                    {
                        "Id": rec_id,
                        "Name": f"Account {rec_id[:4]} Inc",
                        "Website": f"https://www.company{rec_id[:4]}.com",
                        "BillingCity": random.choice(
                            ["San Francisco", "Seattle", "Boston"]
                        ),
                        "BillingState": random.choice(["CA", "WA", "MA"]),
                        "ParentId": None,
                        "LastModifiedDate": f"2024-{random.randint(1,12):02d}-{random.randint(1,28):02d}T12:00:00.000Z",
                    }
                )

        return {"records": records, "totalSize": len(records), "done": True}

    # Default mock response for other queries
    return {"records": [], "totalSize": 0, "done": True}


def get_mock_schema(object_type: str):
    """Return mock schema for development/testing"""

    # Common field types for all objects
    common_fields = [
        {
            "name": "Id",
            "label": "Record ID",
            "type": "id",
            "custom": False,
            "filterable": True,
        },
        {
            "name": "Name",
            "label": "Name",
            "type": "string",
            "custom": False,
            "filterable": True,
        },
        {
            "name": "CreatedDate",
            "label": "Created Date",
            "type": "datetime",
            "custom": False,
            "filterable": True,
        },
        {
            "name": "LastModifiedDate",
            "label": "Last Modified Date",
            "type": "datetime",
            "custom": False,
            "filterable": True,
        },
        {
            "name": "OwnerId",
            "label": "Owner",
            "type": "reference",
            "custom": False,
            "filterable": True,
            "referenceTo": ["User"],
        },
    ]

    # Object-specific fields
    if object_type == "Lead":
        fields = common_fields + [
            {
                "name": "Company",
                "label": "Company",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Email",
                "label": "Email",
                "type": "email",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Phone",
                "label": "Phone",
                "type": "phone",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Status",
                "label": "Lead Status",
                "type": "picklist",
                "custom": False,
                "filterable": True,
                "picklistValues": [
                    {"label": "Open", "value": "Open"},
                    {"label": "Qualified", "value": "Qualified"},
                ],
            },
            {
                "name": "LeadSource",
                "label": "Lead Source",
                "type": "picklist",
                "custom": False,
                "filterable": True,
                "picklistValues": [
                    {"label": "Web", "value": "Web"},
                    {"label": "Email", "value": "Email"},
                ],
            },
            {
                "name": "Industry",
                "label": "Industry",
                "type": "picklist",
                "custom": False,
                "filterable": True,
                "picklistValues": [
                    {"label": "Technology", "value": "Technology"},
                    {"label": "Finance", "value": "Finance"},
                ],
            },
            {
                "name": "City",
                "label": "City",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "State",
                "label": "State/Province",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Website",
                "label": "Website",
                "type": "url",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "NumberOfEmployees",
                "label": "Employees",
                "type": "int",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "AnnualRevenue",
                "label": "Annual Revenue",
                "type": "currency",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Rating",
                "label": "Rating",
                "type": "picklist",
                "custom": False,
                "filterable": True,
                "picklistValues": [
                    {"label": "Hot", "value": "Hot"},
                    {"label": "Warm", "value": "Warm"},
                    {"label": "Cold", "value": "Cold"},
                ],
            },
        ]
    elif object_type == "Account":
        fields = common_fields + [
            {
                "name": "Website",
                "label": "Website",
                "type": "url",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Phone",
                "label": "Phone",
                "type": "phone",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Type",
                "label": "Account Type",
                "type": "picklist",
                "custom": False,
                "filterable": True,
                "picklistValues": [
                    {"label": "Customer", "value": "Customer"},
                    {"label": "Prospect", "value": "Prospect"},
                ],
            },
            {
                "name": "Industry",
                "label": "Industry",
                "type": "picklist",
                "custom": False,
                "filterable": True,
                "picklistValues": [
                    {"label": "Technology", "value": "Technology"},
                    {"label": "Finance", "value": "Finance"},
                ],
            },
            {
                "name": "BillingCity",
                "label": "Billing City",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "BillingState",
                "label": "Billing State",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "BillingCountry",
                "label": "Billing Country",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "NumberOfEmployees",
                "label": "Employees",
                "type": "int",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "AnnualRevenue",
                "label": "Annual Revenue",
                "type": "currency",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "ParentId",
                "label": "Parent Account",
                "type": "reference",
                "custom": False,
                "filterable": True,
                "referenceTo": ["Account"],
            },
        ]
    elif object_type == "Contact":
        fields = common_fields + [
            {
                "name": "FirstName",
                "label": "First Name",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "LastName",
                "label": "Last Name",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Email",
                "label": "Email",
                "type": "email",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Phone",
                "label": "Phone",
                "type": "phone",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Title",
                "label": "Title",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "Department",
                "label": "Department",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "AccountId",
                "label": "Account",
                "type": "reference",
                "custom": False,
                "filterable": True,
                "referenceTo": ["Account"],
            },
            {
                "name": "MailingCity",
                "label": "Mailing City",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "MailingState",
                "label": "Mailing State",
                "type": "string",
                "custom": False,
                "filterable": True,
            },
            {
                "name": "LeadSource",
                "label": "Lead Source",
                "type": "picklist",
                "custom": False,
                "filterable": True,
                "picklistValues": [
                    {"label": "Web", "value": "Web"},
                    {"label": "Email", "value": "Email"},
                ],
            },
        ]
    else:
        fields = common_fields

    return {"name": object_type, "label": object_type, "fields": fields}


@api_router.get("/account-user-fields")
async def get_account_user_fields(
    db: AsyncSession = Depends(get_session),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
) -> list:
    """Get Account fields that are User lookups for owner assignment"""
    try:
        # Use gateway to describe Account object
        desc = await sf.describe("Account")

        # Filter to User lookup fields only
        user_fields = []
        for field in desc.get("fields", []):
            if (
                field.get("type") == "reference"
                and "User" in field.get("referenceTo", [])
                and field["name"]
                not in [
                    "CreatedById",
                    "LastModifiedById",
                    "LastViewedById",
                    "LastReferencedById",
                ]
            ):
                user_fields.append({"value": field["name"], "label": field["label"]})

        # Sort common fields first
        priority = ["OwnerId", "SDR__c", "BDR__c", "CSM__c", "AE__c"]
        user_fields.sort(
            key=lambda x: (
                priority.index(x["value"]) if x["value"] in priority else 999,
                x["label"],
            )
        )

        return user_fields

    except Exception as e:
        log.error(f"Failed to get account user fields: {e}")
        return []


@router.get("/metrics")
async def get_oauth_metrics(
    account_id: Optional[str] = Query(
        None, description="Filter metrics for specific account"
    ),
    x_account_id: Optional[str] = Header(None),
) -> Dict[str, Any]:
    """Get OAuth performance metrics

    Returns metrics for:
    - Wait-until performance (success rate, latency percentiles)
    - Connection attempts (success/failure rates)
    - Circuit breaker events
    - Overall health score
    """
    from ...monitoring.oauth_metrics import oauth_metrics

    # If account_id provided, get account-specific stats
    if account_id:
        return oauth_metrics.get_account_stats(account_id)

    # Otherwise return global stats
    return oauth_metrics.get_stats()


@router.post("/schema-index/quick")
async def trigger_schema_index_quick(
    account_id: str = Depends(require_account),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    """Manually trigger a quick schema index build."""
    try:
        gateway = await SalesforceGateway.for_tenant(str(account_id), db)
        indexer = SalesforceSchemaIndexer(db)
        schema_hash = await indexer.ensure_quick_index(
            org_id=str(account_id), gateway=gateway
        )
        status = await SalesforceSchemaIndexRepo(db).status(str(account_id))
        await db.commit()
        return {"ok": True, "schema_hash": schema_hash, "status": status}
    except Exception as e:
        import traceback

        error_msg = f"{type(e).__name__}: {str(e)}"
        log.error(f"Schema indexing failed for {account_id}: {error_msg}")
        log.error(f"Traceback: {traceback.format_exc()}")
        raise HTTPException(
            500, detail={"error": error_msg, "traceback": traceback.format_exc()}
        )


@router.post("/schema-index/full")
async def trigger_schema_index_full(
    account_id: str = Depends(require_account),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    """Manually trigger a full schema index build (includes usage signals)."""
    gateway = await SalesforceGateway.for_tenant(str(account_id), db)
    indexer = SalesforceSchemaIndexer(db)
    result = await indexer.ensure_full_index(org_id=str(account_id), gateway=gateway)
    status = await SalesforceSchemaIndexRepo(db).status(str(account_id))
    await db.commit()
    return {
        "ok": True,
        "schema_hash": result.get("schema_hash"),
        "stats": result,
        "status": status,
    }


@router.get("/schema-index/status")
async def get_schema_index_status(
    account_id: str = Depends(require_account),
    db: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    """Fetch schema index status and counts."""
    repo = SalesforceSchemaIndexRepo(db)
    return await repo.status(str(account_id))


@router.get("/describe/{sobject}")
async def describe_salesforce_object(
    sobject: str,
    gateway: SalesforceGateway = Depends(get_salesforce_gateway),
) -> Dict[str, Any]:
    """Describe a Salesforce object and return its fields metadata.

    Args:
        sobject: The Salesforce object name (e.g., 'Account', 'Lead', 'Contact')
        account_id: The account ID from the API key
        db: Database session

    Returns:
        Dictionary containing object metadata including fields
    """
    try:
        result = await gateway.describe(sobject)
        return result
    except HTTPException:
        raise
    except Exception as e:
        log.exception(f"Failed to describe Salesforce object {sobject}: {e}")
        raise HTTPException(status_code=500, detail="Failed to describe object")


@router.get("/metrics/health")
async def get_oauth_health(
    x_account_id: Optional[str] = Header(None),
) -> Dict[str, Any]:
    """Get simplified OAuth health status

    Returns:
    - health_score: 0-100 overall health
    - status: 'healthy', 'degraded', or 'unhealthy'
    - issues: List of current issues if any
    """
    from ...monitoring.oauth_metrics import oauth_metrics

    stats = oauth_metrics.get_stats()
    health_score = stats.get("health_score", 100)

    # Determine status based on score
    if health_score >= 90:
        status = "healthy"
    elif health_score >= 70:
        status = "degraded"
    else:
        status = "unhealthy"

    # Identify issues
    issues = []
    if stats["wait_until"]["timeout_count"] > 5:
        issues.append(
            f"High wait-until timeout rate: {stats['wait_until']['timeout_count']} timeouts"
        )

    if stats["connections"]["success_rate"] < 0.95:
        issues.append(
            f"Low connection success rate: {stats['connections']['success_rate']:.1%}"
        )

    if stats["circuit_breaker"]["current_state"] == "open":
        issues.append("Circuit breaker is OPEN - connections blocked")

    return {
        "health_score": health_score,
        "status": status,
        "issues": issues,
        "timestamp": datetime.utcnow().isoformat(),
    }


# ---------------------------------------------------------------------------
# Helpers for Sheets enrich
# ---------------------------------------------------------------------------

def _salesforce_error_response(
    status_code: int,
    error: str,
    message: str,
    request: Request | None,
    extra: Optional[Dict[str, Any]] = None,
) -> JSONResponse:
    content: Dict[str, Any] = {"error": error, "message": message}
    if extra:
        for key, value in extra.items():
            if value is not None:
                content[key] = value
    req_id = _request_id(request)
    if req_id:
        content["request_id"] = req_id
    return JSONResponse(status_code=status_code, content=content)


def _row_limit_for_tier(tier: str) -> int:
    limits = {
        "pro": 5000,
        "scale": 15000,
        "unleashed": 50000,
    }
    return limits.get(tier, 2000)


def _rows_from_grid(rows_grid: List[List[Any]], headers: List[str]) -> List[Dict[str, Any]]:
    rows: List[Dict[str, Any]] = []
    for row in rows_grid or []:
        record: Dict[str, Any] = {}
        for idx, header in enumerate(headers):
            record[header] = row[idx] if idx < len(row) else ""
        rows.append(record)
    return rows


def _resolve_match_column(payload: SalesforceEnrichRequest, headers: List[str]) -> Optional[str]:
    provided = (payload.id_column or "").strip()
    mode = (payload.match_mode or "id").lower()
    if mode == "id":
        if provided:
            return provided
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_CONFIG", "message": "id_column is required for ID match."},
        )

    if provided:
        return provided

    headers_lower = [h.lower().strip() for h in headers]
    if mode == "email":
        candidates = ["email", "work_email", "personal_email"]
    elif mode == "domain":
        candidates = ["domain", "website", "web", "url"]
    elif mode == "name":
        candidates = ["name", "company_name", "account_name"]
    else:
        candidates = []

    for candidate in candidates:
        for idx, header in enumerate(headers_lower):
            if candidate in header and headers[idx]:
                return headers[idx]
    return None


def _extract_match_values(rows: List[Dict[str, Any]], column: str, match_mode: str) -> List[str]:
    seen: Set[str] = set()
    values: List[str] = []
    for row in rows:
        key = _normalize_match_value(row.get(column), match_mode)
        if not key or key in seen:
            continue
        seen.add(key)
        values.append(key)
    return values


def _normalize_client_values(values: Optional[List[List[Any]]]) -> List[List[Any]]:
    normalized: List[List[Any]] = []
    if not values:
        return normalized
    for row in values:
        if row is None:
            normalized.append([])
        elif isinstance(row, list):
            normalized.append(list(row))
        else:
            normalized.append([row])
    return normalized


def _normalize_match_value(value: Any, match_mode: str) -> str:
    if value is None:
        return ""
    text = str(value).strip()
    if not text:
        return ""

    mode = (match_mode or "id").lower()
    if mode == "id":
        return text[:15].upper()
    if mode == "email":
        return text.lower()
    if mode == "domain":
        lowered = text.lower()
        if "@" in lowered:
            lowered = lowered.split("@", 1)[1]
        return lowered
    if mode == "name":
        return text.lower()
    return text


def _normalize_sfdc_fields(fields: List[str]) -> List[str]:
    normalized: List[str] = []
    seen: Set[str] = set()
    for field in fields or []:
        value = (field or "").strip()
        if not value or value in seen:
            continue
        seen.add(value)
        normalized.append(value)
    return normalized


def _resolve_match_range(payload: MatchAccountsRequest, sheet_name: str) -> str:
    if payload.range:
        raw = payload.range.strip()
        if "!" in raw:
            return raw
        return f"{sheet_name}!{raw}"
    return f"{sheet_name}!A1:Z"


async def _read_bulk_progress(redis, job_id: str) -> Optional[Dict[str, Any]]:
    if not redis:
        return None
    raw = await redis.get(SFDC_BULK_PROGRESS_KEY.format(job_id=job_id))
    if not raw:
        return None
    try:
        return json.loads(raw.decode("utf-8") if isinstance(raw, (bytes, bytearray)) else raw)
    except Exception:
        return None


def _resolve_integration_id(gateway: SalesforceGateway) -> Optional[str]:
    integration = getattr(gateway, "_integration", None)
    if integration and getattr(integration, "id", None):
        return str(integration.id)
    return getattr(gateway, "tenant_id", None)


async def _extract_bulk_rows(
    payload: SfdcBulkMatchRequest,
    service: Any,
) -> Tuple[List[str], List[Dict[str, Any]], int]:
    """Load grid + headers for bulk match requests."""
    values_grid: List[List[Any]] = []
    sheet_name: Optional[str] = payload.sheet_name
    resolved_range: Optional[str] = None
    if payload.values:
        values_grid = _normalize_client_values(payload.values)
        sheet_name = payload.sheet_name or "Sheet1"
        resolved_range = payload.range or f"{sheet_name}!A1:Z{len(values_grid)}"
    else:
        if service is None:
            raise HTTPException(
                status_code=400,
                detail={
                    "error": "INVALID_CONFIG",
                    "message": "values payload required when Sheets token is missing.",
                },
            )
        try:
            sheet_titles = _list_sheet_titles(service, payload.spreadsheet_id)
        except Exception as err:
            raise _http_error(err, "Unable to list sheets") from err

        sheet_name = sheet_name or (sheet_titles[0] if sheet_titles else "")
        if not sheet_name:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_RANGE", "message": "Sheet name could not be resolved."},
            )
        if payload.range:
            resolved_range = payload.range if "!" in payload.range else f"{sheet_name}!{payload.range}"
        else:
            resolved_range = f"{sheet_name}!A1:Z"
        try:
            values_grid, _ = _fetch_sheet_grids(service, payload.spreadsheet_id, resolved_range)
        except Exception as err:
            raise _http_error(err, "Unable to read sheet") from err

    if not values_grid:
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_RANGE", "message": "Sheet range is empty."},
        )

    headers = [str(h) if h is not None else "" for h in (values_grid[0] or [])]
    if not headers or not any(headers):
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_HEADER", "message": "Header row is empty."},
        )
    if payload.company_column not in headers:
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_CONFIG", "message": "Company column not found in headers."},
        )
    if payload.domain_column and payload.domain_column not in headers:
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_CONFIG", "message": "Domain column not found in headers."},
        )

    rows = _rows_from_grid(values_grid[1:], headers)
    try:
        parsed_range = parse_a1_range(resolved_range or f"{sheet_name}!A1:Z")
        data_start_row = parsed_range.start_row
    except Exception:
        data_start_row = 1
    return headers, rows, data_start_row


def _normalize_threshold_value(value: Any) -> float:
    try:
        threshold = float(value)
    except (TypeError, ValueError):
        threshold = 0.85
    if threshold > 1.0:
        threshold = threshold / 100.0
    return max(0.0, min(threshold, 1.0))


def _normalize_match_score(value: Any) -> Optional[float]:
    if value is None:
        return None
    try:
        score = float(value)
    except (TypeError, ValueError):
        return None
    if score > 1.0:
        score = score / 100.0
    return max(0.0, min(score, 1.0))


def _normalize_match_dataframe(matches: Optional[pd.DataFrame]) -> pd.DataFrame:
    if matches is None:
        return pd.DataFrame()
    df = matches.copy()
    if "score" not in df.columns and "match_score" in df.columns:
        df["score"] = df["match_score"]
    if "source_id" not in df.columns and "src_id" in df.columns:
        df["source_id"] = df["src_id"]
    if "source_id" not in df.columns and "source" in df.columns:
        df["source_id"] = df["source"]
    if "target_id" not in df.columns and "ref_id" in df.columns:
        df["target_id"] = df["ref_id"]
    if "target_id" not in df.columns and "reference_id" in df.columns:
        df["target_id"] = df["reference_id"]
    return df


def _best_match_by_source(matches: Optional[pd.DataFrame]) -> Dict[str, Dict[str, Any]]:
    df = _normalize_match_dataframe(matches)
    if df is None or df.empty:
        return {}
    df = df.sort_values(by=["score"], ascending=False, na_position="last")
    winners: Dict[str, Dict[str, Any]] = {}
    for _, row in df.iterrows():
        src_id = row.get("source_id")
        if src_id is None:
            continue
        key = str(src_id)
        if key in winners:
            continue
        winners[key] = row.to_dict()
    return winners


def _count_matches_by_source(matches: Optional[pd.DataFrame]) -> Dict[str, int]:
    df = _normalize_match_dataframe(matches)
    if df is None or df.empty or "source_id" not in df.columns:
        return {}
    try:
        counts = df["source_id"].astype(str).value_counts()
        return counts.to_dict()
    except Exception:
        return {}


def _normalize_domain_value(value: Any) -> str:
    if value is None:
        return ""
    text = str(value).strip().lower()
    if not text:
        return ""
    if "@" in text:
        text = text.split("@", 1)[1]
    text = re.sub(r"^https?://", "", text)
    text = text.split("/", 1)[0]
    text = text.split(":", 1)[0]
    if text.startswith("www."):
        text = text[4:]
    return text


def _infer_match_method(source_domain: Any, matched_account: Optional[Dict[str, Any]], status: str) -> Optional[str]:
    if status == "no_match":
        return "no_match"
    if matched_account:
        account_domain = _normalize_domain_value(
            matched_account.get("Website") or matched_account.get("website")
        )
        source_domain_norm = _normalize_domain_value(source_domain)
        if source_domain_norm and account_domain and source_domain_norm == account_domain:
            return "domain"
    if status == "low_confidence":
        return "name_low_confidence"
    if status == "matched":
        return "name"
    return status or None


async def _fetch_salesforce_records(
    sf: SalesforceGateway,
    sobject: str,
    match_mode: str,
    match_column: str,
    match_values: List[str],
    fields: List[str],
) -> Dict[str, Dict[str, Any]]:
    match_field = _match_field_for_mode(sobject, match_mode)
    select_fields = ["Id"]
    for field in fields:
        if field.lower() != "id" and field not in select_fields:
            select_fields.append(field)

    results: Dict[str, Dict[str, Any]] = {}
    chunk_size = 150
    for idx in range(0, len(match_values), chunk_size):
        chunk = match_values[idx : idx + chunk_size]
        values_clause = ", ".join(f"'{_escape_soql_literal(v)}'" for v in chunk)
        soql = f"SELECT {', '.join(select_fields)} FROM {sobject} WHERE {match_field} IN ({values_clause})"
        query_result = await sf.soql(soql)
        records = query_result.get("records") if isinstance(query_result, dict) else query_result
        for record in records or []:
            raw_key = record.get(match_field) if match_field != "Id" else record.get("Id")
            key = _normalize_match_value(raw_key, match_mode)
            if key:
                results[key] = record
    return results


async def _fetch_salesforce_accounts(sf: SalesforceGateway) -> List[Dict[str, Any]]:
    soql = "SELECT Id, Name, Website FROM Account"
    records: List[Dict[str, Any]] = []
    try:
        async for batch in sf.paginate_soql(soql):
            items = batch if isinstance(batch, list) else batch.get("records") if isinstance(batch, dict) else []
            for rec in items or []:
                if isinstance(rec, dict):
                    records.append(rec)
    except AttributeError:
        # paginate_soql not available; fall back to single SOQL
        query_result = await sf.soql(soql)
        records = query_result.get("records") if isinstance(query_result, dict) else query_result or []
    return records


def _match_field_for_mode(sobject: str, match_mode: str) -> str:
    mode = (match_mode or "id").lower()
    if mode == "id":
        return "Id"
    if mode == "email":
        if sobject not in {"Contact", "Lead"}:
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "Email match is supported for Contact or Lead."},
            )
        return "Email"
    if mode == "domain":
        if sobject != "Account":
            raise HTTPException(
                status_code=400,
                detail={"error": "INVALID_CONFIG", "message": "Domain match is supported for Account."},
            )
        return "Website"
    if mode == "name":
        return "Name"
    raise HTTPException(
        status_code=400,
        detail={"error": "INVALID_CONFIG", "message": f"Unsupported match mode '{match_mode}'."},
    )


def _escape_soql_literal(value: str) -> str:
    return str(value).replace("\\", "\\\\").replace("'", "\\'")


def _extract_field_value(record: Optional[Dict[str, Any]], field_path: str) -> Any:
    if not record:
        return ""
    current: Any = record
    for part in field_path.split("."):
        if current is None:
            return ""
        if isinstance(current, dict):
            current = current.get(part)
        else:
            current = getattr(current, part, None)
    return "" if current is None else current


def _build_enrich_write_range(resolved_range: str, existing_columns: int, new_columns: int, row_count: int) -> str:
    parsed = parse_a1_range(resolved_range)
    start_col = parsed.start_col + max(existing_columns, 0)
    end_col = start_col + max(new_columns, 1) - 1
    start_row = parsed.start_row
    end_row = parsed.start_row + max(row_count, 0)
    return f"{parsed.sheet_name}!{col_index_to_letter(start_col)}{start_row}:{col_index_to_letter(end_col)}{end_row}"


def _write_enriched_columns(service, spreadsheet_id: str, write_range: str, rows: List[List[Any]]) -> None:
    service.spreadsheets().values().update(
        spreadsheetId=spreadsheet_id,
        range=write_range,
        valueInputOption="RAW",
        body={"values": rows},
    ).execute()

# Export both routers
__all__ = ["router", "api_router"]
