"""
FoundryMatch ?????FastAPI entry???point
----------------------------------
??? PostgreSQL / SQLite support via SQLModel
??? RQ + Redis for background jobs
??? Secure WebSocket progress streaming
"""

from __future__ import annotations

# Fix Windows console encoding for emoji/unicode in logs
import sys
import os

if sys.platform == "win32" and hasattr(sys.stdout, "reconfigure"):
    try:
        sys.stdout.reconfigure(encoding="utf-8")
        sys.stderr.reconfigure(encoding="utf-8")
    except Exception:
        pass

# Load environment variables from .env file (avoid clobbering test overrides)
from dotenv import load_dotenv
_override_env = os.getenv("FM_DISABLE_DB") != "1" and os.getenv("FM_UNIT_TEST") != "1"
load_dotenv(override=_override_env)
if os.path.exists(".env.dev"):
    load_dotenv(".env.dev", override=True)
TEST_MODE = os.getenv("FM_TEST_MODE") == "1" or os.getenv("FM_UNIT_TEST") == "1"

# ????????????????????????????????????????????????????????????????????????????????? Imports ????????????????????????????????????????????????????????????????????????????????????
import asyncio
import logging
import logging.config
import signal
import time
import json
from pathlib import Path
from typing import Any, Dict, Final, List, Literal, Optional, Union, Tuple
from uuid import UUID, uuid4
import io
from urllib.parse import urlparse
import re

import redis
import redis.asyncio as aioredis
import rq
import jwt
from fastapi import (
    Depends,
    FastAPI,
    File,
    Form,
    Header,
    HTTPException,
    Query,
    Request,
    Response,
    UploadFile,
    WebSocket,
    WebSocketDisconnect,
    WebSocketException,
    status,
)
from fastapi.concurrency import run_in_threadpool
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.middleware.gzip import GZipMiddleware
from .middleware.webhook_auth import WebhookAuthMiddleware
from pydantic import BaseModel, Field as PydanticField, field_validator
from sqlmodel import (
    Field,
    SQLModel,
    select,
    create_engine,
)
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from datetime import datetime, timedelta, timezone

import pandas as pd
# NOTE: duckdb import removed - was unused and slowed startup
import numpy as np
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from fmatch.core.engine import (
    MatchConfig,
    DedupeConfig,
    MatchAlgorithm,
)  # MatchAlgorithm is now centralised in engine
from fmatch.utils.data_parser import parse_data_source
from fmatch.utils.auto_detect import auto_detect_configuration
from .repositories.job_repo import JobRepository
from .models import Job, Account, APIKey
from .auth import get_current_account as real_get_current_account
from . import settings
# Backwards-compatible alias for router gating.
saas_settings = settings
from .observability.sentry_config import init_sentry
from .ml.smart_detection import SmartAutoDetector
from .billing.tracker import SimpleUsageTracker
from .services.salesforce_gateway import SalesforceGateway
from .models import Project
from .observability.context import set_request_id, set_trace_id, clear_context

# ?????????????????????????????????????????????????????????????????????????????? Logging ???????????????????????????????????????????????????????????????????????????????????????
# Configure logging using Python config (no YAML dependency)
init_sentry()
log = logging.getLogger("fmatch.api")
if not TEST_MODE:
    try:
        from .logging_config import configure_logging

        configure_logging(log_level=settings.LOG_LEVEL)
        log.info("Logging configured successfully")
    except ImportError:
        # Fallback: Basic configuration if logging_config.py is not available
        logging.basicConfig(
            level=logging.INFO,
            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
        )
        # Suppress noisy loggers manually
        logging.getLogger("multipart.multipart").setLevel(logging.INFO)
        logging.getLogger("multipart").setLevel(logging.INFO)
        logging.getLogger("uvicorn.access").setLevel(logging.INFO)
        logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
        logging.getLogger("watchfiles").setLevel(logging.WARNING)
        logging.getLogger("httpx").setLevel(logging.WARNING)
        logging.getLogger("httpcore").setLevel(logging.WARNING)
        log.info("Using fallback logging configuration")
else:
    root_logger = logging.getLogger()
    if os.getenv("FM_ENABLE_TEST_LOGGING") != "1":
        root_logger.handlers.clear()
        root_logger.addHandler(logging.NullHandler())
    root_logger.setLevel(
        getattr(logging, os.getenv("LOG_LEVEL", "ERROR").upper(), logging.ERROR)
    )

# Startup banner to kill env/process drift
import socket

logger = logging.getLogger("fmatch.api")
logger.info(
    "[BOOT] env=%s commit=%s pid=%s host=%s",
    os.getenv("APP_ENV", "dev"),
    os.getenv("COMMIT_SHA", "unknown"),
    os.getpid(),
    socket.gethostname(),
)

# ????????????????????????????????????????????????????????? Graceful Shutdown ????????????????????????????????????????????????????????????
shutdown_event = asyncio.Event()


def signal_handler(sig, frame):
    """Handle shutdown signals"""
    log.info(f"Received signal {sig}, initiating graceful shutdown...")
    shutdown_event.set()


# Register signal handlers
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)


# ?????????????????????????????????????????????????????????????????????????????? Custom Errors ???????????????????????????????????????????????????????????????????????????????????????
class iPaaSError(HTTPException):
    """User-friendly errors for iPaaS platforms."""

    def __init__(
        self,
        message: str,
        details: Optional[Dict[str, Any]] = None,
        status_code: int = 400,
    ):
        super().__init__(
            status_code=status_code,
            detail={
                "error": message,
                "details": details or {},
                "help": f"See documentation at {settings.DOCS_URL}",
                "support": f"Contact {settings.SUPPORT_EMAIL} for assistance",
            },
        )


# ---------------------------------------------------------------
# Public constants required by test-suite and other helper modules
INPUT_ID_COLUMN: Final[str] = "id"
REF_ID_COLUMN: Final[str] = "id"
# ---------------------------------------------------------------

# ?????????????????????????????????????????????????????????????????????????????? Redis / RQ ??????????????????????????????????????????????????????????????????????????????
# Import from centralized settings
from .settings import REDIS_URL, RQ_QUEUE_NAME


def _redact_redis_url(url: str) -> str:
    """Hide Redis credentials in logs."""
    try:
        parsed = urlparse(url)
        if parsed.password:
            netloc = parsed.netloc.replace(parsed.password, "*****")
            return parsed._replace(netloc=netloc).geturl()
    except Exception:
        pass
    return url

# Log Redis URL on startup (with IPv4/IPv6 detection)
parsed_redis = urlparse(REDIS_URL)
redis_host = parsed_redis.hostname or "127.0.0.1"
log.info("[STARTUP] Initializing Redis connection...")
log.info("[STARTUP] REDIS_URL = %s", _redact_redis_url(REDIS_URL))
log.info("[STARTUP] Redis host = %s (IPv4 enforced)", redis_host)

# Initialize Redis connections with explicit IPv4
try:
    blocking_redis_conn = redis.from_url(REDIS_URL)
    # Test the connection
    blocking_redis_conn.ping()
    log.info("[STARTUP] Redis connection successful (blocking client)")
except Exception as e:
    log.error("[STARTUP] Redis connection failed: %s", str(e))
    log.error("[STARTUP] Make sure Redis is running on %s", redis_host)
    blocking_redis_conn = redis.from_url(REDIS_URL)  # Still create it for later retry

async_redis_conn = aioredis.from_url(REDIS_URL, decode_responses=True)
q = rq.Queue(RQ_QUEUE_NAME, connection=blocking_redis_conn)

log.info(
    "[API] Redis/RQ initialized: redis=%s queue=%s",
    _redact_redis_url(REDIS_URL),
    RQ_QUEUE_NAME,
)

# ???????????????????????????????????????????????????????????? Database / SQLModel setup ???????????????????????????????????????????????????
# Single shared engine & session factory — avoids duplicate connection pools.
# Pool tuning is configured in db.py via DATABASE_POOL_SIZE / DATABASE_MAX_OVERFLOW env vars.
from .db import engine as async_engine, AsyncSessionLocal, get_session  # noqa: E402

DATABASE_URL = os.getenv("DATABASE_URL")
if not DATABASE_URL:
    raise RuntimeError(
        "DATABASE_URL is not configured. Set the environment variable to a PostgreSQL DSN."
    )


# Ensure schema exists (safe in dev) before app starts serving
def _to_sync_url(url: str) -> str:
    if url.startswith("postgresql+asyncpg://"):
        return url.replace("postgresql+asyncpg://", "postgresql://", 1)
    if url.startswith("sqlite+aiosqlite://"):
        return url.replace("sqlite+aiosqlite://", "sqlite://", 1)
    return url


def ensure_schema_once() -> None:
    # Skip schema sync in dev to speed up startup (cloud DB already has schema)
    if os.getenv("FM_SKIP_SCHEMA_SYNC", "false").lower() == "true":
        log.info("[STARTUP] Skipping schema sync (FM_SKIP_SCHEMA_SYNC=true)")
        return
    try:
        log.info("[STARTUP] Running schema sync (set FM_SKIP_SCHEMA_SYNC=true to skip)...")
        sync_url = _to_sync_url(DATABASE_URL)
        eng = create_engine(sync_url, future=True)
        # Create all SQLModel tables (idempotent)
        SQLModel.metadata.create_all(eng)
        eng.dispose()
        log.info("[STARTUP] Schema sync complete")
    except Exception as e:
        log.warning(f"ensure_schema_once: {e!r}")


ensure_schema_once()


# Minimal fetch_job helper for status/results
async def fetch_job(job_id: str) -> Optional[Dict[str, Any]]:
    async with AsyncSessionLocal() as db:
        # Prefer repository but fall back to direct select if needed
        try:
            repo = JobRepository(db)
            job = await repo.get_job(job_id)
        except Exception:
            result = await db.execute(select(Job).where(Job.id == job_id))
            job = result.scalar_one_or_none()

        if not job:
            return None

        # Extract mode from job_type or config
        mode_val = None
        try:
            mode_val = getattr(job, "job_type", None)
            if hasattr(mode_val, "value"):
                mode_val = mode_val.value
        except Exception:
            mode_val = None
        if not mode_val:
            cfg = getattr(job, "config", {}) or {}
            mode_val = cfg.get("mode")

        project_id_val = getattr(job, "project_id", None)
        account_id_val = getattr(job, "account_id", None)
        return {
            "id": str(getattr(job, "id", job_id)),
            "project_id": str(project_id_val) if project_id_val else None,
            "account_id": str(account_id_val) if account_id_val else None,
            "status": getattr(
                getattr(job, "status", None), "value", getattr(job, "status", None)
            )
            or "unknown",
            "created_at": getattr(job, "created_at", None),
            "updated_at": getattr(job, "updated_at", None),
            "mode": mode_val,
            "progress": getattr(job, "progress", 0) or 0,
            "results_path": getattr(job, "results_path", None),
            "matches_found": getattr(job, "matches_found", 0) or 0,
            "config": getattr(job, "config", {}) or {},
        }


# Thin async helper that persists a Job row
async def write_job_row(
    job_id: str,
    project_id: str,
    *,
    account_id: Optional[str] = None,
    status: str = "queued",
    mode: Optional[str] = None,
    ruleset: Optional[str] = None,
    filename: Optional[str] = None,
    webhook_url: Optional[str] = None,
    config: Optional[Dict[str, Any]] = None,
) -> None:
    async with AsyncSessionLocal() as db:
        repo = JobRepository(db)
        # Merge aux fields into config for traceability
        aux_meta = {
            k: v
            for k, v in {
                "ruleset": ruleset,
                "filename": filename,
                "webhook_url": webhook_url,
            }.items()
            if v is not None
        }
        await repo.create_job(
            job_id=job_id,
            project_id=project_id,
            account_id=account_id,
            status=status,
            mode=mode,
            config={**(config or {}), **aux_meta},
        )


# ????????????????????????????????????????????????????????????????????? Pydantic helpers ??????????????????????????????????????????????????????????????????
class CustomRule(BaseModel):
    condition: str
    score_adjustment: int


class ProjectConfigUpdate(BaseModel):
    match_algorithm: MatchAlgorithm | str | None = None
    match_threshold: int | None = PydanticField(default=None, ge=1, le=100)
    preprocessing_rules: list[str] | None = None
    custom_match_rules: list[CustomRule] | None = None

    @field_validator("match_algorithm", mode="before")
    @classmethod
    def _coerce_algorithm(cls, v):
        if v is None or isinstance(v, MatchAlgorithm):
            return v
        try:
            return MatchAlgorithm(v)
        except ValueError:
            # allow matching by member *name* as well
            try:
                return MatchAlgorithm[v]  # noqa: SLF001  (enum lookup by name)
            except KeyError:
                raise ValueError(
                    f"Unsupported match_algorithm '{v}'. "
                    f"Allowed: {[m.value for m in MatchAlgorithm]}"
                ) from None


class ProjectRead(SQLModel):
    id: UUID
    account_id: Optional[str] = None
    name: str
    description: Optional[str] = None
    source_file_key: Optional[str] = None
    reference_file_key: Optional[str] = None
    created_at: Optional[datetime] = None
    updated_at: Optional[datetime] = None


# ???????????????????????????????????????????????????????????????????????? Domain model ???????????????????????????????????????????????????????????????????????????
# TODO: The Project model below has been temporarily commented out to avoid
# duplicate table definition error with models.py. This extended version has
# additional fields (match_algorithm, match_threshold, etc.) that need to be
# merged into the canonical Project model in models.py
"""
class Project(SQLModel, table=True):
    id: UUID = Field(default_factory=uuid4, primary_key=True, index=True)
    __table_args__ = {"extend_existing": True}
    account_id: UUID | None = Field(default=None, index=True)
    name: str
    description: str | None = None
    source_file_key: str | None = None
    reference_file_key: str | None = None

    match_algorithm: MatchAlgorithm = Field(
        sa_column=Column(SAEnum(MatchAlgorithm), default=MatchAlgorithm.wratio, server_default=MatchAlgorithm.wratio.value, nullable=False)
    )
    match_threshold: int = Field(
        default=80,
        sa_column=Column(Integer, CheckConstraint("match_threshold BETWEEN 0 AND 100"))
    )
    match_ruleset: str = Field(default="default", nullable=False)

    # --- ARCHITECT FEEDBACK: Use the universal JSON type ---
    preprocessing_rules: List[str] = Field(
        default_factory=list,
        sa_column=Column(MutableList.as_mutable(JSON), nullable=False, server_default="[]")
    )
    custom_match_rules: List[dict] = Field(
        default_factory=list,
        sa_column=Column(MutableList.as_mutable(JSON), nullable=False, server_default="[]")
    )
    # --- END FIX ---

    created_at: float | None = None
    updated_at: float | None = None
"""


class ProjectCreate(SQLModel):
    name: str
    description: Optional[str] = None


class JobAccepted(BaseModel):
    job_id: str
    status: str = "queued"


# ????????????????????????????????????????????????????????? Universal Match Models ????????????????????????????????????????????????????????????
class DataSource(BaseModel):
    """Universal data source that works for any integration"""

    data: Union[List[Dict[str, Any]], str, bytes]
    format: Literal["json", "csv", "parquet"] = "json"
    id_column: Optional[str] = None
    column_types: Optional[Dict[str, str]] = None


class UniversalMatchRequest(BaseModel):
    """Universal request format for all integrations"""

    mode: Literal["dedupe", "match"]
    source: DataSource
    reference: Optional[DataSource] = None

    # Optional overrides (all auto-detected if not provided)
    algorithm: Optional[MatchAlgorithm] = None
    threshold: Optional[int] = PydanticField(None, ge=0, le=100)
    blocking_strategy: Optional[str] = None

    # Response preferences
    webhook_url: Optional[str] = None
    include_non_matches: bool = False
    max_results: Optional[int] = None


class UniversalMatchResponse(BaseModel):
    job_id: str
    status: Literal["completed", "processing", "failed"]
    results: Optional[Union[List[Dict], str]] = None  # For sync responses
    results_url: Optional[str] = None  # For async responses
    auto_detected_config: Dict[str, Any]
    estimated_time_seconds: Optional[int]
    metadata: Dict[str, Any] = Field(default_factory=dict)


class GoogleSheetSource(BaseModel):
    """Google Sheet data source configuration"""

    sheet_id: str
    range: str = "A:Z"
    api_key: Optional[str] = None


class GoogleSheetsRequest(BaseModel):
    """Request model for Google Sheets integration"""

    mode: Literal["dedupe", "match"]
    source: GoogleSheetSource
    reference: Optional[GoogleSheetSource] = None
    threshold: Optional[int] = PydanticField(None, ge=0, le=100)
    algorithm: Optional[MatchAlgorithm] = None
    webhook_url: Optional[str] = None


# ??????????????????????????????????????????????????????????????? Authentication ???????????????????????????????????????????????????????????????
async def get_current_account(
    credentials: str = Depends(real_get_current_account),
) -> UUID:
    """Now uses real JWT authentication"""
    # Convert string account ID to UUID
    try:
        return UUID(credentials)
    except ValueError:
        # If credentials is not a valid UUID, create a deterministic UUID from it
        import hashlib

        hash_object = hashlib.md5(credentials.encode())
        return UUID(hash_object.hexdigest())


async def get_current_account_ws(websocket: WebSocket) -> UUID:
    """WebSocket authentication using JWT"""
    auth_header = websocket.headers.get("authorization")
    if not auth_header or not auth_header.lower().startswith("bearer "):
        raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION)

    # Extract token and validate using the real auth
    try:
        from fastapi.security import HTTPAuthorizationCredentials

        token = auth_header.split(" ")[1]
        credentials = HTTPAuthorizationCredentials(scheme="Bearer", credentials=token)

        # Call the real auth function directly
        account_str = await real_get_current_account(credentials)
        return (
            UUID(account_str)
            if account_str
            else UUID("00000000-0000-0000-0000-000000000001")
        )
    except Exception as e:
        log.error(f"WebSocket auth failed: {e}")
        raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION)


def generate_api_key() -> Tuple[str, str, str]:
    """
    Generate a secure API key.
    Returns: (full_key, key_prefix, key_hash)
    """
    import secrets
    import bcrypt

    # Generate 32-byte random key
    full_key = secrets.token_urlsafe(32)
    key_prefix = full_key[:8]

    # Hash the key
    key_hash = bcrypt.hashpw(full_key.encode(), bcrypt.gensalt()).decode()

    return full_key, key_prefix, key_hash


async def validate_api_key(api_key: str, db: AsyncSession) -> Optional[APIKey]:
    """
    Validate API key against database.
    Returns APIKey object if valid, None otherwise.
    """
    try:
        import bcrypt

        # Extract prefix (first 8 chars)
        if len(api_key) < 32:
            return None

        key_prefix = api_key[:8]

        # Look up by prefix
        stmt = select(APIKey).where(
            APIKey.key_prefix == key_prefix, APIKey.is_active == True
        )
        result = await db.execute(stmt)
        api_key_obj = result.scalar_one_or_none()

        if not api_key_obj:
            return None

        # Check expiration
        if api_key_obj.expires_at and api_key_obj.expires_at < datetime.utcnow():
            log.warning(f"API key {key_prefix}... has expired")
            return None

        # Verify the full key
        if not bcrypt.checkpw(api_key.encode(), api_key_obj.key_hash.encode()):
            log.warning(f"Invalid API key attempted: {key_prefix}...")
            return None

        # Update last used timestamp
        api_key_obj.last_used_at = datetime.utcnow()
        db.add(api_key_obj)
        await db.commit()

        return api_key_obj

    except Exception as e:
        log.error(f"Error validating API key: {e}")
        return None


async def get_account_from_api_key_or_header(
    api_key: Optional[str] = None,
    authorization: Optional[str] = None,
    db: Optional[AsyncSession] = None,
) -> UUID:
    """
    Flexible authentication for iPaaS endpoints.
    Supports both API keys and JWT tokens.
    """
    # Dev bypass (explicit opt-in only)
    from . import settings as saas_settings
    import os

    env_name = getattr(saas_settings, "ENV", os.getenv("ENV", "")).lower()
    dev_bypass_enabled = (
        os.getenv("FM_TEST_MODE") == "1"
        or (
            env_name in {"development", "dev", "test"}
            and os.getenv("AUTH_DEV_BYPASS_ENABLED", "false").strip().lower()
            in {"1", "true", "yes", "on"}
        )
    )
    if dev_bypass_enabled:
        log.debug(
            "DEV AUTH: Returning dev account from get_account_from_api_key_or_header"
        )
        # Return the dev account ID as a string (not UUID since it's not a valid UUID format)
        return saas_settings.DEV_ACCOUNT_ID

    if api_key and db:
        # Validate API key against database
        api_key_obj = await validate_api_key(api_key, db)
        if api_key_obj:
            return api_key_obj.account_id
        else:
            raise HTTPException(401, "Invalid API key")

    elif api_key and not db:
        # Environment-gated fallback (explicit dev/test opt-in only)
        if (
            os.getenv("ALLOW_INSECURE_API_KEY_FALLBACK", "false").lower() == "true"
            and env_name in {"development", "dev", "test"}
        ):
            log.warning(
                "API key validation without database access - using insecure fallback (DEV MODE ONLY)"
            )
            import hashlib

            hash_object = hashlib.sha256(api_key.encode())
            return UUID(hash_object.hexdigest()[:32])
        else:
            raise HTTPException(401, "API key validation requires database access")

    elif authorization:
        # Use JWT authentication
        from fastapi.security import HTTPAuthorizationCredentials

        if not authorization.lower().startswith("bearer "):
            raise HTTPException(401, "Invalid authorization header format")

        token = authorization.split(" ")[1]
        credentials = HTTPAuthorizationCredentials(scheme="Bearer", credentials=token)

        try:
            account_str = await real_get_current_account(credentials)
            return (
                UUID(account_str)
                if account_str
                else UUID("00000000-0000-0000-0000-000000000001")
            )
        except Exception as e:
            raise HTTPException(401, f"Invalid token: {str(e)}")

    else:
        raise HTTPException(401, "Authentication required (API key or Bearer token)")


async def require_admin(
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> Account:
    """
    Require admin role for accessing admin endpoints.
    """
    # Get the account from database
    account = await db.get(Account, account_id)
    if not account:
        raise HTTPException(404, "Account not found")

    # Check if account has admin role
    if account.role != "admin":
        raise HTTPException(403, "Admin access required")

    return account


# ?????????????????????????????????????????????????????????????????????????????? Billing Helpers ???????????????????????????????????????????????????????????????????????????????????????
def calculate_credits(request: UniversalMatchRequest) -> int:
    """Calculate credits based on request size"""
    # Simple pricing: 1 credit per 1000 rows
    if isinstance(request.source.data, list):
        row_count = len(request.source.data)
    else:
        # For CSV/JSON strings, estimate
        row_count = request.source.data.count("\n")

    # Add reference data if matching
    if request.reference and isinstance(request.reference.data, list):
        row_count += len(request.reference.data)

    return max(1, row_count // 1000)  # Minimum 1 credit


# ?????????????????????????????????????????????????????????????????????????????? Rate Limiting ???????????????????????????????????????????????????????????????????????????????????????
# Check if rate limiting should be enabled
ENABLE_RATE_LIMITING = os.getenv("ENABLE_RATE_LIMITING", "true").lower() == "true"

if ENABLE_RATE_LIMITING:
    try:
        from slowapi import Limiter, _rate_limit_exceeded_handler
        from slowapi.util import get_remote_address
        from slowapi.errors import RateLimitExceeded

        # Create limiter with custom key function for API keys
        def get_rate_limit_key(request: Request) -> str:
            """Get rate limit key from API key or IP address."""
            # Check for API key in various places
            api_key = (
                request.headers.get("X-API-Key")
                or request.headers.get("x-api-key")
                or request.query_params.get("api_key")
                or request.headers.get("Authorization", "").replace("Bearer ", "")[
                    :8
                ]  # Use key prefix
            )

            if api_key and len(api_key) >= 8:
                return f"api_key:{api_key[:8]}"  # Use prefix for rate limiting

            return get_remote_address(request)

        limiter = Limiter(key_func=get_rate_limit_key)
        RATE_LIMITING_ENABLED = True
        log.info("Rate limiting enabled (ENABLE_RATE_LIMITING=true)")

    except ImportError:
        # Only warn in production; silence in development
        if settings.ENV not in ["development", "dev"]:
            log.warning(
                "slowapi not installed but ENABLE_RATE_LIMITING=true. Install with: pip install slowapi"
            )
        limiter = None
        RATE_LIMITING_ENABLED = False
else:
    # Rate limiting explicitly disabled
    log.info("Rate limiting disabled (ENABLE_RATE_LIMITING=false)")
    limiter = None
    RATE_LIMITING_ENABLED = False


# ????????????????????????????????????????????????????????????????????????????????? Route Validation ????????????????????????????????????????????????????????????????????????
def _assert_unique_routes(app: FastAPI):
    """Verify no duplicate routes exist"""
    seen = set()
    dups = []
    for r in app.router.routes:
        path = getattr(r, "path_format", None) or getattr(r, "path", None) or repr(r)
        methods = getattr(r, "methods", None) or {"ANY"}
        for m in methods:
            key = (m.upper(), path.rstrip("/"))
            if key in seen:
                dups.append(key)
            else:
                seen.add(key)
    if dups:
        raise RuntimeError(f"Duplicate routes detected: {sorted(dups)}")


# ????????????????????????????????????????????????????????????????????????????????? FastAPI app ????????????????????????????????????????????????????????????????????????
from contextlib import asynccontextmanager


@asynccontextmanager
async def lifespan(app: FastAPI):
    # Startup validation - fail fast on configuration issues
    from .startup_validator import validate_startup

    minimal_mode = os.getenv("FM_MINIMAL_ROUTERS", "false").lower() == "true"
    skip_validation = (
        os.getenv("FM_SKIP_STARTUP_VALIDATION", "false").lower() == "true"
    )
    if minimal_mode or skip_validation:
        log.warning(
            "Skipping startup validation (minimal_mode=%s, skip_flag=%s)",
            minimal_mode,
            skip_validation,
        )
    else:
        validate_startup()

    # Startup
    log.info("Starting FoundryMatch API...")
    yield
    # Shutdown
    log.info("Shutting down...")


class RequestContextMiddleware(BaseHTTPMiddleware):
    """Unified middleware for request context: request_id, trace_id, response headers.

    Replaces the old add_request_id HTTP middleware and consolidates all
    request-scoped context setup into one place.
    """

    async def dispatch(self, request, call_next):
        # trace_id: from GCP Cloud Trace header (Cloud Run injects this)
        trace_header = request.headers.get("X-Cloud-Trace-Context", "")
        trace_id = trace_header.split("/")[0] if trace_header else None
        if trace_id:
            set_trace_id(trace_id)

        # request_id: caller-supplied or fresh UUID (always present)
        request_id = request.headers.get("X-Request-ID") or str(uuid4())
        set_request_id(request_id)
        request.state.request_id = request_id  # backward compat

        try:
            response = await call_next(request)
            response.headers["X-Request-Id"] = request_id
            return response
        finally:
            clear_context()


def create_app() -> FastAPI:
    app = FastAPI(title="FoundryMatch API", version="2.1.0", lifespan=lifespan)

    app.add_middleware(RequestContextMiddleware)

    # Enable gzip for large responses
    try:
        app.add_middleware(GZipMiddleware, minimum_size=1024)
    except Exception:
        pass

    # -------- CORS (place early so it processes preflights) --------
    try:
        raw_origins = settings.CORS_ORIGINS.split(",") if settings.CORS_ORIGINS else []
        default_origins = [
            "http://localhost:3000",
            "http://127.0.0.1:3000",
            "http://localhost:3004",
            "http://127.0.0.1:3004",
        ]
        allowed_origins = [
            o.strip()
            for o in (raw_origins or default_origins)
            if o.strip() and o.strip() != "*"
        ]

        # Always include production dashboard domains even if env var omits them
        fallback_origins = [
            "https://foundryops-site.pages.dev",
            "https://foundryops.io",
            "https://foundrymatch-admin.pages.dev",  # Admin UI (legacy)
            "https://foundryops-admin.pages.dev",  # Admin UI (new location)
            # Local dev origins (ensure these are always allowed)
            "http://localhost",
            "http://127.0.0.1",
            "http://localhost:8000",
            "http://127.0.0.1:8000",
        ]
        for origin in fallback_origins:
            if origin not in allowed_origins:
                allowed_origins.append(origin)

        cors_regex_patterns = [
            r"https?://localhost(:\d+)?",  # http or https, any port
            r"https?://127\.0\.0\.1(:\d+)?",  # http or https, any port
            r"https://([a-zA-Z0-9-]+\.)?foundryops-site\.pages\.dev",
            r"https://([a-zA-Z0-9-]+\.)?foundryops\.io",
            r"https://([a-zA-Z0-9-]+\.)?foundrymatch-admin\.pages\.dev",  # Admin UI (legacy)
            r"https://([a-zA-Z0-9-]+\.)?foundryops-admin\.pages\.dev",  # Admin UI (new)
            r"https://([a-zA-Z0-9-]+\.)?github\.io",  # GitHub Pages (launch site)
        ]
        allow_origin_regex = (
            rf"^(?:{'|'.join(f'(?:{pattern})' for pattern in cors_regex_patterns)})$"
        )

        log.info("[CORS] Allowed origins (final): %s", allowed_origins)

        _cors_allowed_origins = set(allowed_origins)
        _cors_allowed_origin_regexes = [
            re.compile(pattern, flags=re.IGNORECASE) for pattern in cors_regex_patterns
        ]
        _cors_allowed_methods = "GET,POST,PUT,PATCH,DELETE,OPTIONS"
        _cors_allowed_headers = "Authorization, Content-Type, Accept, ngrok-skip-browser-warning, Idempotency-Key, X-Requested-With, X-Account-Id, X-API-Key, X-Webhook-Signature, X-Webhook-Signature-256"

        app.add_middleware(
            CORSMiddleware,
            allow_origins=allowed_origins,
            allow_origin_regex=allow_origin_regex,
            allow_methods=["*"],
            allow_headers=[
                "Authorization",
                "Content-Type",
                "Accept",
                "ngrok-skip-browser-warning",
                "Idempotency-Key",
                "X-Requested-With",
                "X-Account-Id",
                "X-API-Key",
                "X-Webhook-Signature",
                "X-Webhook-Signature-256",
            ],
            allow_credentials=True,
            expose_headers=["Content-Disposition"],
        )

        @app.middleware("http")
        async def _ensure_cors_headers(request: Request, call_next):
            """
            Ensure Access-Control headers are always present for approved origins.

            Starlette's CORS middleware sometimes skips attaching headers when multiple
            middlewares short-circuit (e.g. auth failures). This guard guarantees the
            browser sees the right headers for every response and preflight.
            """
            origin = request.headers.get("origin")
            log.debug(
                "[CORS] Incoming request: %s %s origin=%s",
                request.method,
                request.url.path,
                origin,
            )
            is_allowed_origin = False
            if origin:
                if origin in _cors_allowed_origins:
                    is_allowed_origin = True
                else:
                    is_allowed_origin = any(
                        pattern.fullmatch(origin)
                        for pattern in _cors_allowed_origin_regexes
                    )
            log.debug("[CORS] is_allowed_origin=%s", is_allowed_origin)

            if request.method == "OPTIONS" and is_allowed_origin:
                log.debug(
                    "[CORS] Short-circuiting OPTIONS preflight for origin=%s", origin
                )
                response = Response(status_code=200)
            else:
                response = await call_next(request)

            if is_allowed_origin:
                log.debug("[CORS] Setting CORS headers for origin=%s", origin)
                response.headers["Access-Control-Allow-Origin"] = origin
                response.headers["Access-Control-Allow-Credentials"] = "true"
                response.headers["Access-Control-Allow-Headers"] = _cors_allowed_headers
                response.headers["Access-Control-Allow-Methods"] = _cors_allowed_methods
                log.debug(
                    "[CORS] Response headers now: %s",
                    {
                        k: v
                        for k, v in response.headers.items()
                        if k.lower().startswith("access-control")
                    },
                )
            else:
                log.debug(
                    "[CORS] Origin %s not allowed or missing; skipping header injection",
                    origin,
                )

            return response
    except Exception as _cors_err:
        log.warning(f"CORS middleware not initialized: {_cors_err}")

    # -------- Compression --------
    app.add_middleware(GZipMiddleware, minimum_size=1024)

    # -------- Webhook Security (AFTER CORS so preflights bypass auth) --------
    try:
        app.add_middleware(
            WebhookAuthMiddleware,
            rate_limit_enabled=True,
            signature_verification_enabled=True,
            ip_allowlist_enabled=False,
            global_rate_limit=settings.WEBHOOK_RATE_LIMIT_GLOBAL,
            per_key_rate_limit=settings.WEBHOOK_RATE_LIMIT_PER_MINUTE,
        )
    except Exception as _wh_err:
        log.warning(f"WebhookAuth middleware not initialized: {_wh_err}")

    # -------- Dev helper: inject X-Account-Id if missing in dev --------
    @app.middleware("http")
    async def _inject_dev_account_id(request: Request, call_next):
        import os as _os

        if _os.getenv("ENV", "").lower() in ("dev", "development"):
            has_account_id = any(
                h[0].lower() == b"x-account-id"
                for h in request.scope.get("headers", [])
            )
            if not has_account_id:
                request.scope["headers"].append(
                    (b"x-account-id", settings.DEV_ACCOUNT_ID.encode())
                )
        # Always allow OPTIONS through quickly (preflight)
        if request.method == "OPTIONS":
            return await call_next(request)
        return await call_next(request)

    # -------- Register routers --------
    _register_routers(app)

    return app


def _register_routers(app: FastAPI):
    """Register all routers with the app instance"""
    from . import settings as saas_settings

    minimal_mode = os.getenv("FM_MINIMAL_ROUTERS", "false").lower() == "true"
    sheets_only = os.getenv("FM_SHEETS_ONLY", "false").lower() == "true"

    # ========================================================================
    # ALWAYS-ON ROUTERS (required for basic operation)
    # ========================================================================
    from .api import jobs, utils
    from .api.v2 import jobs as jobs_v2
    from .api.v2 import ai_match_config
    from .api.v1 import agent_public as agent_public_v1

    app.include_router(jobs.router)
    app.include_router(utils.router)
    app.include_router(jobs_v2.router)
    app.include_router(ai_match_config.router)
    app.include_router(agent_public_v1.router)

    # ========================================================================
    # SHEETS-REQUIRED ROUTERS (Google Sheets add-on support)
    # ========================================================================
    if sheets_only or not minimal_mode:
        # Import Sheets-required routers
        from .api.v2 import uploads as uploads_v2
        from .api.v2 import telemetry as telemetry_v2
        from .api.v2 import integrations as integrations_v2
        from .api.v2 import salesforce as salesforce_v2
        from .api.v2 import hubspot as hubspot_v2
        from .api.v2 import zoominfo as zoominfo_v2
        from .api.v2 import outreach_oauth as outreach_oauth_v2
        from .api.v2 import flow_builder as flow_builder_v2
        from .api.v2 import id_hunter
        from .api.v1 import pdl as pdl_v1
        from .api.v1 import auth as auth_v1
        from .api.v1 import transform as transform_v1
        from .api.v1 import company_domains as company_domains_v1
        from .api.v1 import sheets_results as sheets_results_v1
        from .api.v1 import scheduling as scheduling_v1
        from .api.v1 import quota as quota_v1
        from .api.v1 import credits as credits_v1
        from .api.v1 import locker as locker_v1  # BYO Provider Locker (Unleashed tier)
        from .api.v1 import enrichment as enrichment_v1  # BYO Enrichment jobs (Unleashed tier)
        from .api.v1 import scraper as scraper_v1  # BYO Apify scraper jobs (Unleashed tier)
        from .api.v1 import oauth_apollo as oauth_apollo_v1  # Apollo OAuth (Sheets/CLI)
        from .api.v1 import oauth_google as oauth_google_v1  # Google OAuth (Sheets writes)
        from .api.v1 import oauth_outreach as oauth_outreach_v1  # Outreach OAuth bridge
        from .api.v1 import telemetry as telemetry_v1
        from .api.v2 import account_overview
        from .api.v2 import dedupe as dedupe_v2  # Dedupe for Sheets
        from .api.v2 import intelligence as intelligence_v2  # Auto-config & preview
        from .api.v2 import gremlin as gremlin_v2  # g-gremlin Sheets integration
        from .api.v2 import formula_fixer as formula_fixer_v2
        from .api.v2 import model_explainer as model_explainer_v2
        from .api.v2 import gremlin_triage as gremlin_triage_v2  # Gremlin triage
        from .api.v2 import sequencer as sequencer_v2
        from .api.v2 import justcall as justcall_v2
        from .api.v2 import heyreach as heyreach_v2
        from .api.v2 import support as support_v2
        from .api.v2 import sheet_webhooks as sheet_webhooks_v2

        # Mount Sheets-required routers
        app.include_router(jobs_v2.manual_router)  # Manual match UI
        app.include_router(uploads_v2.router)  # File uploads
        app.include_router(auth_v1.router, prefix="/api/v1")  # Authentication
        app.include_router(salesforce_v2.router)  # Salesforce integration (core PLG)
        app.include_router(salesforce_v2.api_router)  # Salesforce API routes
        app.include_router(hubspot_v2.router)  # HubSpot OAuth integration
        app.include_router(zoominfo_v2.router)  # ZoomInfo OAuth integration
        app.include_router(outreach_oauth_v2.router)  # Outreach OAuth integration
        app.include_router(flow_builder_v2.router)  # Flow Builder
        app.include_router(telemetry_v1.router, prefix="/api/v1")  # Telemetry V1
        app.include_router(telemetry_v2.router)  # Telemetry V2
        app.include_router(transform_v1.router)  # Data transforms
        app.include_router(company_domains_v1.router)  # Domain promotion
        app.include_router(sheets_results_v1.router, prefix="/api/v1")  # Sheets results
        app.include_router(scheduling_v1.router, prefix="/api/v1")  # Scheduled results
        app.include_router(quota_v1.router, prefix="/api/v1")  # Quota checking
        app.include_router(credits_v1.router, prefix="/api/v1")  # Credit management
        app.include_router(locker_v1.router, prefix="/api/v1")  # BYO Provider Locker
        app.include_router(enrichment_v1.router, prefix="/api/v1")  # BYO Enrichment
        app.include_router(scraper_v1.router, prefix="/api/v1")  # BYO Apify Scraper
        app.include_router(oauth_apollo_v1.router, prefix="/api/v1")  # Apollo OAuth
        app.include_router(oauth_google_v1.router, prefix="/api/v1")  # Google OAuth
        app.include_router(oauth_outreach_v1.router, prefix="/api/v1")  # Outreach OAuth bridge
        app.include_router(pdl_v1.router)  # PDL enrichment (CRITICAL for Sheets)
        app.include_router(integrations_v2.router)  # Integrations
        app.include_router(account_overview.router)  # Account info
        app.include_router(dedupe_v2.router)  # Dedupe (required for Sheets QA)
        app.include_router(intelligence_v2.router)  # Auto-config for Sheets
        app.include_router(id_hunter.router)  # Salesforce ID Hunter lookup
        app.include_router(gremlin_v2.router)  # g-gremlin API for Sheets add-on
        if saas_settings.GREMLIN_V2_ENABLED and saas_settings.GREMLIN_V2_FORMULA_FIXER:
            app.include_router(formula_fixer_v2.router)
        if saas_settings.GREMLIN_V2_ENABLED and saas_settings.GREMLIN_V2_EXPLAINER:
            app.include_router(model_explainer_v2.router)
        app.include_router(sequencer_v2.router)
        app.include_router(justcall_v2.router)
        app.include_router(heyreach_v2.router)
        app.include_router(sheet_webhooks_v2.router)
        if not any(getattr(r, "path", "") == "/api/v2/gremlin/triage" for r in app.router.routes):
            app.include_router(gremlin_triage_v2.router)
        app.include_router(support_v2.router, prefix="/api/v2")

        # Optional Salesforce push router
        try:
            from .api.v1 import salesforce_push as salesforce_push_v1
            app.include_router(salesforce_push_v1.router)
            log.info("Salesforce push router mounted")
        except Exception as _sf_push_err:
            log.warning(f"salesforce_push v1 not registered: {_sf_push_err}")

        log.info("Sheets-required routers mounted (includes PDL enrichment)")
        if sheets_only:
            log.info("FM_SHEETS_ONLY enabled; skipping full SaaS router set")
            return

    # If sheets-only mode, stop here
    if sheets_only:
        log.info("FM_SHEETS_ONLY enabled; skipping SaaS web app routers")
        return

    # If minimal mode, stop here (backward compatibility)
    if minimal_mode:
        log.info("FM_MINIMAL_ROUTERS enabled; skipping optional routers")
        return

    # ========================================================================
    # FULL SAAS WEB APP ROUTERS (L2A, C2A, routing, admin, etc.)
    # ========================================================================
    from .api.v2 import intelligence_learning as intelligence_learning_v2
    from .api.v2 import surveys as surveys_v2
    from .api.v2 import findings as findings_v2
    from .api.v2 import investigations as investigations_v2
    from .api.v2 import discovery as discovery_v2
    from .api.v2 import l2a as l2a_v2
    from .api.v2 import c2a as c2a_v2
    # dedupe_v2 already imported in sheets-only section above
    from .api.v2 import match_explain as match_explain_v2
    from .api.v2 import match
    from .api.v2 import slack
    from .api.v2 import slack_interactive
    from .api.v2 import notifications
    from .api.v2 import column_mappings
    from .api.v2 import webhooks
    from .api import webhooks_stripe  # Stripe billing webhooks
    from .api.v2 import quality
    from .api.v2 import rate_limits
    from .api.v2 import tradeshow
    from .api.v2 import feedback
    from .api.v2 import review
    from .api.v2 import crm_append, crm_gap_analysis
    from .api.v2 import approvals as approvals_v2
    from .api.v2 import autofill as autofill_v2
    from .api.v2 import api_keys as api_keys_v2
    from .api.v2 import admin_customers
    from .api.v2 import admin_analytics
    from .api.v2 import admin_usage
    from .api.v2 import admin_foundrygraph
    from .api.v2 import admin_activations
    from .api.v2 import admin_plg
    from .api.v2 import admin_trials
    from .api.v2 import admin_trial_emails
    from .api.v2 import beta_signup
    from .api.v2 import support as support_v2
    try:
        from .api.v2 import admin_wikidata
    except Exception as exc:  # pragma: no cover
        log.warning("Admin Wikidata router disabled: %s", exc)
        admin_wikidata = None
    from .api.v2 import seats as seats_v2
    from .api.v2 import enrichment_buckets as enrichment_buckets_v2
    from .api.v2 import enrichment_archive as enrichment_archive_v2
    from .api.v2 import enrichment_field_mappings as enrichment_field_mappings_v2
    from .api.v2 import privacy as privacy_v2
    from .api.internal import metrics as internal_metrics
    from .api.v1 import public as public_v1
    from .api.v1 import gateway_metrics as gateway_metrics_v1
    from .api.v1 import mappings as mappings_v1
    from .api.v1 import tradeshow as tradeshow_v1
    from .api.v1 import soql_builder as soql_builder_v1
    from .api.v1 import salesforce_ai_reports as salesforce_ai_reports_v1
    from .api.v1 import sync_tier as sync_tier_v1
    from .api.v1 import checkout as checkout_v1
    from .api.v1 import admin_telemetry as admin_telemetry_v1
    from .api.v1 import admin_insights as admin_insights_v1
    from .api.v1 import portal as portal_v1

    app.include_router(intelligence_learning_v2.router)
    app.include_router(surveys_v2.router)
    app.include_router(findings_v2.router)
    app.include_router(investigations_v2.router)
    app.include_router(discovery_v2.router)
    app.include_router(l2a_v2.router)
    app.include_router(c2a_v2.router)
    # dedupe_v2.router already included in sheets-only section above
    app.include_router(match_explain_v2.router)
    app.include_router(match.router)
    app.include_router(slack.router)
    app.include_router(slack_interactive.router)
    app.include_router(notifications.router)
    app.include_router(column_mappings.router)
    app.include_router(webhooks.router)
    app.include_router(webhooks_stripe.router)  # Stripe billing webhooks
    app.include_router(webhooks_stripe.legacy_router)  # /api/v1/webhooks/stripe
    app.include_router(quality.router)
    app.include_router(rate_limits.router)
    app.include_router(crm_append.router)
    app.include_router(crm_gap_analysis.router)
    app.include_router(tradeshow.router)
    app.include_router(feedback.router)

    # Local imports for modules not yet imported at module level
    from .api.v2 import forecast

    app.include_router(forecast.router, prefix="/api/v2", tags=["forecast"])
    app.include_router(review.router)
    app.include_router(approvals_v2.router)
    app.include_router(autofill_v2.router)
    app.include_router(api_keys_v2.router)
    app.include_router(seats_v2.router)
    app.include_router(enrichment_buckets_v2.router)
    app.include_router(enrichment_archive_v2.router, prefix="/api/v2")
    app.include_router(enrichment_field_mappings_v2.router)
    app.include_router(privacy_v2.router, prefix="/api/v2")
    app.include_router(admin_customers.router, prefix="/api/v2")
    app.include_router(admin_analytics.router, prefix="/api/v2")
    app.include_router(admin_usage.router, prefix="/api/v2")
    app.include_router(admin_foundrygraph.router, prefix="/api/v2/admin/foundrygraph")
    app.include_router(admin_activations.router, prefix="/api/v2/admin/activations")
    app.include_router(admin_plg.router, prefix="/api/v2")
    app.include_router(admin_trials.router, prefix="/api/v2")
    app.include_router(admin_trial_emails.router, prefix="/api/v2")
    app.include_router(beta_signup.public_router, prefix="/api/v2")
    app.include_router(beta_signup.admin_router, prefix="/api/v2")

    # ── Static brand assets (used in emails) ────────────────────────────────
    @app.get("/static/logo-hex.png", include_in_schema=False)
    async def _serve_logo():
        import pathlib
        from fastapi.responses import FileResponse
        logo = pathlib.Path(__file__).parent / "static" / "logo-hex.png"
        return FileResponse(
            logo,
            media_type="image/png",
            headers={"Cache-Control": "public, max-age=31536000, immutable"},
        )

    app.include_router(admin_telemetry_v1.router, prefix="/api/v1")
    app.include_router(admin_insights_v1.router, prefix="/api/v1")
    app.include_router(support_v2.admin_router, prefix="/api/v2")
    if admin_wikidata:
        app.include_router(admin_wikidata.router, prefix="/api/v2/admin/wikidata")
    app.include_router(internal_metrics.router)
    app.include_router(public_v1.router)
    app.include_router(gateway_metrics_v1.router)
    app.include_router(mappings_v1.router)
    app.include_router(tradeshow_v1.router)
    app.include_router(soql_builder_v1.router, prefix="/api/v1")
    app.include_router(salesforce_ai_reports_v1.router, prefix="/api/v1")
    app.include_router(sync_tier_v1.router)  # Already has /api/v1/admin prefix
    app.include_router(checkout_v1.router)
    app.include_router(portal_v1.router)

    # Root endpoint to prevent 404 on homepage
    @app.get("/")
    async def root():
        return {
            "service": "FoundryOps API",
            "version": "2.0",
            "status": "operational",
            "docs": "/docs",
            "health": "/api/v1/health",
        }

    # Optional middleware
    try:
        from .api.v1.middleware import install_v1_middleware

        install_v1_middleware(app)
    except Exception as _mw_err:
        log.warning(f"v1 middleware not installed: {_mw_err}")

    # Health check routes
    try:
        from .api import health

        app.include_router(health.router)
        log.info("Health router mounted")
    except Exception as e:
        log.warning(f"Health router not mounted: {e}")

    # Legacy auth routes
    try:
        from .api import auth as _auth

        app.include_router(_auth.router)
        log.info("Auth router mounted at /api/auth")
    except Exception as e:
        log.warning(f"Auth router not mounted: {e}")

    # Import routing and action routers
    from .routing.api import (
        router as routing_router,
        admin_router as routing_admin_router,
    )
    from .api.v2 import actions as actions_v2
    from .api.v2 import audit as audit_v2
    from .api.v2 import policies as policies_v2
    from .api.v2 import rollback as rollback_v2

    app.include_router(routing_router)
    app.include_router(routing_admin_router)
    app.include_router(actions_v2.router)
    app.include_router(audit_v2.router)
    app.include_router(policies_v2.router)
    app.include_router(rollback_v2.router)

    # Admin routers
    from .api.v2 import admin_domain
    from .api.v2 import admin_domain_families
    from .api.v2 import admin_search
    from .api.v2 import admin_users
    from .api.v2 import admin_manual
    from .api.v2 import admin as admin_v2

    app.include_router(admin_domain.router)
    app.include_router(admin_domain_families.router)
    app.include_router(admin_search.router)
    app.include_router(admin_users.router)
    app.include_router(admin_manual.router)
    app.include_router(admin_v2.router)

    # Public lookup endpoints (no auth required)
    from .api.v2 import domain_lookup, wikidata_lookup

    app.include_router(domain_lookup.router)
    app.include_router(wikidata_lookup.router)
    log.info("Public domain and wikidata lookup endpoints mounted")

    # FoundryGraph authenticated endpoints
    from .api.v2 import foundrygraph_lookup
    from .api.v2.company_list_builder import router as list_builder_router

    app.include_router(foundrygraph_lookup.router)
    app.include_router(list_builder_router)
    log.info("FoundryGraph lookup endpoints mounted")

    # Reporting and enrichment
    from .api.v2 import reporting
    from .api.v2 import enrichment as enrichment_v2
    from .api.v2 import contact_enrichment as contact_enrichment_v2
    from .api.v2.admin import enrichment_monitor, enrichment_billing

    app.include_router(reporting.router)
    app.include_router(enrichment_v2.router)
    app.include_router(contact_enrichment_v2.router)
    app.include_router(
        enrichment_monitor.router, prefix="/api/v2/admin/enrichment/monitor"
    )
    app.include_router(
        enrichment_billing.router, prefix="/api/v2/admin/enrichment/billing"
    )

    # Dev uploads (development only)
    if os.getenv("FM_DEV_UPLOADS") == "1":
        try:
            from .api import dev_uploads

            app.include_router(dev_uploads.router)
            log.info("Dev uploads enabled at /api/v1/uploads")
        except Exception as _dev_err:
            log.warning(f"Dev uploads not registered: {_dev_err}")

    # Sheets API (if enabled)
    if os.getenv("ENABLE_SHEETS_API", "false").lower() == "true":
        try:
            from .api import sheets as sheets_api

            app.include_router(sheets_api.router, prefix="/api/v1/sheets")
            log.info("Sheets API enabled at /api/v1/sheets")
        except Exception as _sheets_err:
            log.warning(f"Sheets API not registered: {_sheets_err}")


# Create the app instance
app = create_app()


# --- Ollama warm-up on startup ---
@app.on_event("startup")
async def warmup_ollama():
    """Warm up Ollama models on startup to avoid cold start delays"""
    if not settings.AI_REPORTS_ENABLED:
        return

    import httpx

    ollama_host = settings.AI_REPORTS_HOST
    planner_model = settings.AI_REPORTS_PLANNER_MODEL

    log.info(f"[WARMUP] Warming up Ollama model: {planner_model}")
    try:
        async with httpx.AsyncClient(timeout=30.0) as client:
            response = await client.post(
                f"{ollama_host}/api/chat",
                json={
                    "model": planner_model,
                    "messages": [{"role": "user", "content": "ping"}],
                    "stream": False,
                },
            )
            if response.status_code == 200:
                data = response.json()
                load_time = (
                    data.get("load_duration", 0) / 1_000_000_000
                )  # nanoseconds to seconds
                log.info(
                    f"[WARMUP] ✅ Model {planner_model} loaded successfully (load time: {load_time:.2f}s)"
                )
            else:
                log.warning(
                    f"[WARMUP] Model warmup returned {response.status_code}: {response.text[:200]}"
                )
    except Exception as e:
        log.warning(f"[WARMUP] Failed to warm up model (non-critical): {e}")


# --- DEV 422 logger (always enabled for debugging) ---
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse


@app.exception_handler(RequestValidationError)
async def _dev_validation_handler(request: Request, exc):
    try:
        body = await request.body()
    except Exception:
        body = b""
    body_str = body.decode("utf-8", "ignore")[:2048]

    # Log the validation error details
    logger.error(f"[422 DEBUG] Validation error on {request.url.path}")
    raw_errors = exc.errors()
    sanitized_errors = []
    for err in raw_errors:
        ctx = err.get("ctx")
        if isinstance(ctx, dict):
            ctx = {
                k: (str(v) if isinstance(v, Exception) else v) for k, v in ctx.items()
            }
            err = {**err, "ctx": ctx}
        sanitized_errors.append(err)

    logger.error(f"[422 DEBUG] Errors: {sanitized_errors}")
    logger.error(f"[422 DEBUG] Body received: {body_str[:500]}")

    return JSONResponse(
        status_code=422,
        content={
            "ok": False,
            "errors": sanitized_errors,
            "path": str(request.url),
            "method": request.method,
            "query": dict(request.query_params),
            "body": body_str,
        },
    )


# Add SecurityViolation exception handler for clean 403s with CORS
from .services.salesforce_gateway import SecurityViolation


@app.exception_handler(SecurityViolation)
async def security_violation_handler(request, exc: SecurityViolation):
    """Return clean 403 with CORS headers for security blocks"""
    return JSONResponse(
        status_code=403, content={"error": "blocked_by_policy", "message": str(exc)}
    )


# Global handler: DB connection pool exhaustion → 503 (retryable)
from sqlalchemy.exc import (  # noqa: E402
    OperationalError as SAOperationalError,
    DBAPIError as SADBAPIError,
    TimeoutError as SATimeoutError,
)
from asyncpg.exceptions import TooManyConnectionsError  # noqa: E402


def _is_pool_exhaustion_error(exc: BaseException) -> bool:
    markers = (
        "queuepool limit",
        "remaining connection slots",
        "remaining connection slots are reserved",
        "toomanyconnectionserror",
        "too many clients",
        "too many connections",
        "connection pool",
        "timed out waiting for connection",
        "sorry, too many clients already",
    )

    seen = set()
    stack = [exc]
    while stack:
        current = stack.pop()
        if current is None:
            continue

        marker = id(current)
        if marker in seen:
            continue
        seen.add(marker)

        if isinstance(current, TooManyConnectionsError):
            return True

        text = str(current).lower()
        if any(candidate in text for candidate in markers):
            return True

        stack.append(getattr(current, "orig", None))
        stack.append(getattr(current, "__cause__", None))
        stack.append(getattr(current, "__context__", None))

    return False


@app.exception_handler(SAOperationalError)
@app.exception_handler(SADBAPIError)
@app.exception_handler(SATimeoutError)
@app.exception_handler(TooManyConnectionsError)
async def _db_operational_error_handler(request: Request, exc: Exception):
    """Map DB pool timeouts and connection exhaustion to 503 Service Unavailable."""
    if _is_pool_exhaustion_error(exc):
        logger.warning(
            "DB pool exhaustion on %s %s: %s",
            request.method,
            request.url.path,
            str(exc)[:200],
        )
        return JSONResponse(
            status_code=503,
            content={"error": "service_overloaded", "message": "Please retry shortly"},
            headers={"Retry-After": "2"},
        )
    # Re-raise non-pool DB errors so they surface as normal 500s
    raise exc


# Initialize scheduler
scheduler = AsyncIOScheduler(timezone="UTC")

# Build a specific allowlist; do NOT use "*" with allow_credentials=True.
raw_origins = settings.CORS_ORIGINS.split(",") if settings.CORS_ORIGINS else []
default_origins = [
    "http://localhost:3000",
    "http://127.0.0.1:3000",
    "http://localhost:3004",
    "http://127.0.0.1:3004",
    # Additional localhost variants for local dev
    "http://localhost",
    "http://127.0.0.1",
    "http://localhost:8000",
    "http://127.0.0.1:8000",
]
allowed_origins = [
    o.strip()
    for o in (raw_origins or default_origins)
    if o.strip() and o.strip() != "*"
]

# Always include production domains even if env var omits them
_production_origins = [
    "https://foundryops-site.pages.dev",
    "https://foundryops.io",
    "https://foundrymatch-admin.pages.dev",
    "https://foundryops-admin.pages.dev",
]
for _po in _production_origins:
    if _po not in allowed_origins:
        allowed_origins.append(_po)

# CORS middleware will be added after all other middleware for proper ordering

# ????????????????????????????????????????????????????????????????????????????????? Router Setup ????????????????????????????????????????????????????????????????????????
if os.getenv("FM_MINIMAL_ROUTERS", "false").lower() == "true":
    log.info("FM_MINIMAL_ROUTERS enabled; skipping legacy module-level router imports")
elif os.getenv("FM_SHEETS_ONLY", "false").lower() == "true":
    log.info("FM_SHEETS_ONLY enabled; skipping legacy module-level router imports")
else:
    # Import routers from api modules
    # TODO: Fix circular import - billing imports from main which imports billing
    # from .api import billing, health, dashboard, admin, reports, auth
    # from .api import projects, jobs, telemetry  # TODO: implement
    # from .billing.alerts import CreditAlertService
    # from .billing.invoices import InvoiceGenerator

    # Include routers
    # Frontend router removed - using new unified API instead

    # Import and include the new routers
    pass

# Feature flag for sheets API (currently disabled until fully implemented)
ENABLE_SHEETS_API = os.getenv("ENABLE_SHEETS_API", "false").lower() == "true"


# --- Place permissive status route BEFORE router includes ---
@app.get(
    "/api/v1/jobs/{job_id:path}/status",
    tags=["iPaaS"],
    dependencies=[Depends(limiter.limit(settings.RATE_LIMIT_STATUS))]
    if RATE_LIMITING_ENABLED
    else [],
)
async def get_job_status_ipaas(
    request: Request,
    response: Response,
    job_id: str,
    include_partial: bool = Query(
        False, description="Include partial results if available"
    ),
    format: Literal["json", "csv"] = Query("json", description="Response format"),
) -> Dict[str, Any]:
    """
    iPaaS-friendly status endpoint. Registered early and permissive on job_id.
    """
    job = await fetch_job(job_id)
    if not job:
        raise HTTPException(404, "Job not found")

    payload: Dict[str, Any] = {
        "job_id": job_id,
        "status": job.get("status", "unknown"),
        "created_at": job.get("created_at"),
        "updated_at": job.get("updated_at"),
        "progress": job.get("progress", 0),
    }

    if "mode" in job:
        payload["mode"] = job["mode"]

    if job["status"] == "completed":
        payload["completed_at"] = job.get("completed_at", job.get("updated_at"))
        payload["results_url"] = f"/api/v1/jobs/{job_id}/results"
        payload["total_matches"] = job.get("matches_found", 0)

        if "results_path" in job:
            try:
                results_df = pd.read_parquet(job["results_path"])  # type: ignore[arg-type]
                payload["total_matches"] = len(results_df)
                if include_partial and len(results_df) > 0:
                    preview = results_df.head(10)
                    if format == "csv":
                        payload["preview"] = preview.to_csv(index=False)
                    else:
                        payload["preview"] = preview.to_dict(orient="records")
            except Exception as e:  # pragma: no cover - best effort preview
                log.warning(f"Could not read results for preview: {e}")
    elif job["status"] == "failed":
        payload["error"] = job.get("error", job.get("message", "Unknown error"))
        payload["failed_at"] = job.get("updated_at")
    elif job["status"] in ["queued", "processing"]:
        created_at = job.get("created_at")
        rows = job.get("rows_to_process")
        if isinstance(created_at, datetime) and isinstance(rows, int):
            elapsed = (datetime.now(timezone.utc) - created_at).total_seconds()
            estimated_total = estimate_processing_time(rows)
            payload["estimated_completion_seconds"] = max(0, estimated_total - elapsed)
    response.headers["Cache-Control"] = "no-store"
    return payload


# Router registrations moved to _register_routers function

# Duplicate router includes removed - already included above (lines 929-933)
# app.include_router(results.router)
# app.include_router(job_management.router)
# app.include_router(progress.router)

# --- DEV: force auth dependency overrides so requests reach handlers ---
import os
from fastapi.routing import APIRoute


# ========== Development Schema Health Check ==========
import logging


def _sqlite_path_from_url(url: str) -> str | None:
    if not url:
        return None
    m = re.match(r"^sqlite(?:\+aiosqlite)?:/{2,3}(.*)$", url)
    if m:
        return m.group(1)
    return None


@app.on_event("startup")
async def schema_health_check() -> None:
    """On startup, verify critical columns exist in dev to avoid 500/CORS loops."""
    try:
        from . import settings as _settings

        if _settings.ENV.lower() not in ("dev", "development"):
            return

        db_url = _settings.DATABASE_URL
        db_path = _sqlite_path_from_url(db_url)
        if not db_path:
            # Non-sqlite (e.g., Postgres) ??? skip simple check
            return

        import sqlite3

        conn = sqlite3.connect(db_path)
        cur = conn.cursor()

        def table_exists(name: str) -> bool:
            cur.execute(
                "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)
            )
            return cur.fetchone() is not None

        def cols(name: str) -> set[str]:
            cur.execute(f"PRAGMA table_info({name})")
            return {r[1] for r in cur.fetchall()}

        expectations = {
            "l2a_suggestions": {
                "job_id",
                "applied_field",
                "previous_value",
                "applied_value",
            },
            "l2a_manual_views": {"scope"},
            "l2a_bulk_jobs": {
                "scope",
                "approval_status",
                "requested_by",
                "approved_by",
                "started_at",
                "ended_at",
                "last_cursor",
                "last_page_size",
                "eta_sec",
                "blocked_if_empty",
                "blocked_never",
            },
        }

        problems = []
        for t, need in expectations.items():
            if not table_exists(t):
                problems.append(f"Missing table {t}")
                continue
            have = cols(t)
            miss = sorted(list(need - have))
            if miss:
                problems.append(f"{t}: missing columns {miss}")

        conn.close()

        if problems:
            logging.getLogger("fmatch.schema").warning(
                "Schema health check detected issues: %s. Run scripts/ensure_l2a_schema.py or alembic upgrade heads.",
                "; ".join(problems),
            )
        else:
            logging.getLogger("fmatch.schema").info("Schema health check OK")
    except Exception as e:
        logging.getLogger("fmatch.schema").warning(f"Schema health check skipped: {e}")


@app.on_event("startup")
async def ensure_c2a_schema_dev() -> None:
    """Dev-only: ensure C2A columns exist in local SQLite to avoid 500s.

    Production should use Alembic migrations. This is a safety net for local dev.
    """
    try:
        from . import settings as _settings

        if _settings.ENV.lower() not in ("dev", "development"):
            return

        db_url = _settings.DATABASE_URL
        db_path = _sqlite_path_from_url(db_url)
        if not db_path:
            return

        import sqlite3

        conn = sqlite3.connect(db_path)
        cur = conn.cursor()

        def has_col(table: str, col: str) -> bool:
            try:
                cur.execute(f"PRAGMA table_info({table})")
                return any(r[1] == col for r in cur.fetchall())
            except Exception:
                return False

        def add_col(table: str, col_sql: str):
            try:
                cur.execute(f"ALTER TABLE {table} ADD COLUMN {col_sql}")
            except Exception:
                pass

        # c2a_suggestions ??? backfill columns commonly added by later migrations
        if not has_col("c2a_suggestions", "relation_id"):
            add_col("c2a_suggestions", "relation_id TEXT")
        if not has_col("c2a_suggestions", "job_id"):
            add_col("c2a_suggestions", "job_id TEXT")
        if not has_col("c2a_suggestions", "error_message"):
            add_col("c2a_suggestions", "error_message TEXT")
        if not has_col("c2a_suggestions", "tier"):
            add_col("c2a_suggestions", "tier TEXT")
        if not has_col("c2a_suggestions", "reasons"):
            add_col("c2a_suggestions", "reasons JSON")
        if not has_col("c2a_suggestions", "field_score_details"):
            add_col("c2a_suggestions", "field_score_details JSON")
        if not has_col("c2a_suggestions", "incumbent_field_score_details"):
            add_col("c2a_suggestions", "incumbent_field_score_details TEXT")
        if not has_col("c2a_suggestions", "decided_at"):
            add_col("c2a_suggestions", "decided_at TIMESTAMP")
        if not has_col("c2a_suggestions", "decided_by"):
            add_col("c2a_suggestions", "decided_by TEXT")
        if not has_col("c2a_suggestions", "registry_version"):
            add_col("c2a_suggestions", "registry_version TEXT")
        if not has_col("c2a_suggestions", "gain"):
            add_col("c2a_suggestions", "gain REAL")
        if not has_col("c2a_suggestions", "last_verified_at"):
            add_col("c2a_suggestions", "last_verified_at TIMESTAMP")
        if not has_col("c2a_suggestions", "score_gap"):
            add_col("c2a_suggestions", "score_gap REAL")
        try:
            cur.execute(
                "CREATE INDEX IF NOT EXISTS ix_c2a_suggestions_job_id ON c2a_suggestions (job_id)"
            )
        except Exception:
            pass

        # c2a_jobs
        if not has_col("c2a_jobs", "skipped_b2c"):
            add_col("c2a_jobs", "skipped_b2c INTEGER DEFAULT 0")
        if not has_col("c2a_jobs", "conflicts"):
            add_col("c2a_jobs", "conflicts INTEGER DEFAULT 0")
        if not has_col("c2a_jobs", "error_count"):
            add_col("c2a_jobs", "error_count INTEGER DEFAULT 0")

        conn.commit()
        conn.close()
        logging.getLogger("fmatch.schema").info("C2A dev schema ensured")
    except Exception as e:
        logging.getLogger("fmatch.schema").warning(f"C2A schema ensure skipped: {e}")


@app.on_event("startup")
async def _override_auth_in_dev():
    from . import settings
    from .auth_security import require_account as _real_require_account
    from .services.salesforce_gateway import SalesforceGateway
    from .models import User

    env = os.getenv("ENV", "production").lower()
    if env not in ("dev", "development"):
        log.info("DEV OVERRIDES: skipped (ENV != dev/development)")
        return

    DEV_ACCT = settings.DEV_ACCOUNT_ID  # "dev-account-id"
    log.info(f"DEV OVERRIDES: using DEV_ACCOUNT_ID={DEV_ACCT}")

    # Simple overrides that all return the same account ID
    async def _dev_require_account():
        return DEV_ACCT

    async def _dev_current_account():
        return DEV_ACCT

    async def _dev_user():
        """Return a mock user for development"""
        return User(
            id="dev-user-id",
            email="dev@example.com",
            account_id=DEV_ACCT,
            is_admin=True,
        )

    # Override require_account (the main one we use)
    app.dependency_overrides[_real_require_account] = _dev_require_account

    # Try to import and override get_current_account
    try:
        from .auth import get_current_account as _real_get_current_account

        app.dependency_overrides[_real_get_current_account] = _dev_current_account
    except ImportError:
        pass

    # Try to import and override get_current_user for API auth
    try:
        from .api.auth import get_current_user as _real_get_current_user

        app.dependency_overrides[_real_get_current_user] = _dev_user
    except ImportError:
        pass

    # Override the local get_account_from_api_key_or_header if it exists
    global get_account_from_api_key_or_header
    if "get_account_from_api_key_or_header" in globals():
        app.dependency_overrides[get_account_from_api_key_or_header] = (
            _dev_current_account
        )

    # Clear any stale gateways created before overrides applied
    SalesforceGateway._instances.clear()

    log.info("  - Auth dependencies overridden for development")


# Storage endpoints for pre-signed URLs
try:
    from .api import storage_endpoints

    app.include_router(storage_endpoints.router)
    log.info("Storage endpoints loaded successfully")
except ImportError as e:
    log.warning(f"Storage endpoints not available: {e}")

# Dev uploads and sheets API moved to _register_routers function

# from .api.v2 import metrics as metrics_v2  # TODO: Create metrics.py in api/v2 directory
# app.include_router(metrics_v2.router)  # TODO: Enable when metrics.py is created

# TODO: Fix these missing routers
# app.include_router(auth.router)  # Auth router first for login endpoint
# app.include_router(billing.router)
# app.include_router(health.router)
# app.include_router(dashboard.router)
# app.include_router(admin.router)
# app.include_router(reports.router)

# Validate no duplicate routes exist
try:
    _assert_unique_routes(app)
    log.info("Route uniqueness validation passed")
except RuntimeError as e:
    log.error(f"Route validation failed: {e}")
    # In development, fail fast. In production, log but continue
    if settings.ENV in ["development", "dev"]:
        raise

# ============ L2A DEBUGGING CODE ============
# Middleware to log all L2A requests (instead of ASGI wrapper)
class L2ATapMiddleware(BaseHTTPMiddleware):
    async def dispatch(self, request, call_next):
        path = request.url.path
        if path.startswith("/api/v2/l2a"):
            log.info(f"[TAP IN] {request.method} {path}")
            response = await call_next(request)
            log.info(f"[TAP OUT] {request.method} {path} -> {response.status_code}")
            return response
        return await call_next(request)


# Add middleware AFTER CORS so we see final status codes
app.add_middleware(L2ATapMiddleware)

# Route dependency dump
import inspect


@app.on_event("startup")
async def _dump_l2a_routes_full():
    for r in app.routes:
        if isinstance(r, APIRoute) and "/l2a" in r.path:
            deps = [
                getattr(getattr(d, "call", None), "__name__", repr(d))
                for d in r.dependant.dependencies
            ]
            log.info(
                "L2A %s %-28s deps=%s security=%s endpoint=%s:%s",
                list(r.methods)[0] if r.methods else "-",
                r.path,
                deps,
                getattr(r.dependant, "security_schemes", None),
                inspect.getmodule(r.endpoint).__name__,
                r.endpoint.__name__,
            )


# Route "winner" probe
from starlette.routing import Match


@app.on_event("startup")
async def _probe_match():
    scope = {
        "type": "http",
        "path": "/api/v2/l2a/writeback/options",
        "method": "GET",
        "root_path": "",
        "query_string": b"",
        "headers": [],
    }
    winners = []
    for r in app.routes:
        if isinstance(r, APIRoute):
            m, _ = r.matches(scope)
            if m == Match.FULL:
                winners.append((r.path, r.endpoint.__name__))
    log.info("FULL MATCHES for /api/v2/l2a/writeback/options -> %s", winners)


# Removed duplicate override - all handled in the main startup function above

# ============ END L2A DEBUGGING CODE ============

# Initialize collectors on startup
app.state.billing_collector = None
app.state.telemetry_collector = None
app.state.redis_client = None
app.state.smart_detector = None

# Scheduled job for credit alerts - DISABLED
# @scheduler.scheduled_job('interval', hours=1)
# async def check_credit_alerts():
#     """Check and send credit alerts hourly"""
#     try:
#         async with AsyncSessionLocal() as db:
#             # alert_service = CreditAlertService(db)
#             # alerts_sent = await alert_service.check_and_send_alerts()
#             # log.info(f"Credit alert check complete: {alerts_sent} alerts sent")
#             pass  # Disabled for now
#     except Exception as e:
#         log.error(f"Error checking credit alerts: {e}", exc_info=True)


# Scheduled job for investigations
@scheduler.scheduled_job("interval", hours=24)
async def cleanup_old_findings():
    """Clean up findings older than 60 days to minimize storage and PII"""
    from .models import Finding
    from sqlalchemy import delete
    from datetime import datetime

    try:
        cutoff_date = datetime.utcnow() - timedelta(days=60)
        async with AsyncSessionLocal() as db:
            stmt = delete(Finding).where(Finding.created_at < cutoff_date)
            result = await db.execute(stmt)
            await db.commit()
            if result.rowcount > 0:
                log.info(f"Cleaned up {result.rowcount} findings older than 60 days")
    except Exception as e:
        log.error(f"Error cleaning up old findings: {e}", exc_info=True)


@scheduler.scheduled_job("interval", hours=24)
async def cleanup_sfdc_import_chunks():
    """Clean up expired Salesforce import chunk idempotency records."""
    from .services import sfdc_import_idempotency as sfdc_idem

    try:
        async with AsyncSessionLocal() as db:
            expired_count = await sfdc_idem.cleanup_expired_import_chunks(db)
            if expired_count:
                log.info(
                    "Cleaned up %s expired Salesforce import chunks", expired_count
                )
    except Exception as exc:
        log.error(
            "Error cleaning up Salesforce import chunks: %s", exc, exc_info=True
        )


@scheduler.scheduled_job("interval", minutes=1)
async def check_scheduled_investigations():
    """Check for due investigations with policy application"""
    from sqlalchemy.exc import OperationalError

    # Skip in development mode or if table doesn't exist
    if settings.ENV == "development":
        return  # Skip scheduled investigations in dev mode

    try:
        # Check if investigations table exists first
        async with AsyncSessionLocal() as db:
            try:
                # Check for table existence (works for both SQLite and PostgreSQL)
                check_result = await db.execute(
                    text(
                        "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'investigations'"
                    )
                )
                if check_result.scalar() == 0:
                    # Table doesn't exist, skip
                    return
            except OperationalError:
                # SQLite doesn't have information_schema, try a different approach
                try:
                    await db.execute(text("SELECT 1 FROM investigations LIMIT 0"))
                except OperationalError:
                    # Table doesn't exist
                    log.debug(
                        "Investigations table not found, skipping scheduled check"
                    )
                    return
    except Exception as e:
        log.debug(f"Skipping scheduled investigations: {e}")
        return

    # Now import and run the actual investigation logic
    from .services.investigation_service import InvestigationService
    from .services.policy_service import PolicyService
    from .models import Investigation, InvestigationStatus
    from sqlalchemy import select
    from croniter import croniter
    from datetime import datetime

    # Get Redis connection for mutex
    redis_client = None
    try:
        redis_client = aioredis.from_url(settings.REDIS_URL, decode_responses=False)

        async with AsyncSessionLocal() as db:
            # Get all scheduled investigations
            stmt = select(Investigation).where(
                Investigation.schedule_cron.isnot(None),
                Investigation.status != InvestigationStatus.RUNNING,
            )
            result = await db.execute(stmt)

            for investigation in result.scalars():
                # Check if due
                cron = croniter(investigation.schedule_cron, investigation.created_at)
                next_run = cron.get_next(datetime)

                if datetime.utcnow() >= next_run:
                    # Try to acquire lock with Redis SET NX
                    lock_key = f"investigation_lock:{investigation.id}"
                    lock_acquired = await redis_client.set(
                        lock_key,
                        "1",
                        nx=True,
                        ex=300,  # 5 minute lock
                    )

                    if lock_acquired:
                        log.info(f"Running scheduled investigation {investigation.id}")

                        # Get active policies for this account
                        policy_service = PolicyService(db)
                        policies = await policy_service.get_active_policies(
                            investigation.account_id
                        )

                        # Pass policies to investigation
                        if policies:
                            investigation.params["policies"] = {
                                p.salesforce_object: p.policy_hash for p in policies
                            }
                            await db.commit()  # Save updated params

                        # Run investigation
                        service = InvestigationService(db)
                        asyncio.create_task(service.run_investigation(investigation.id))
                    else:
                        log.debug(
                            f"Investigation {investigation.id} already being processed by another worker"
                        )
    except Exception as e:
        log.error(f"Error checking scheduled investigations: {e}", exc_info=True)
    finally:
        if redis_client:
            await redis_client.close()


@app.on_event("startup")
async def startup_event():
    """Initialize async resources on startup."""
    # Register SQLite UUID adapter first (before any DB operations)
    try:
        import sqlite3
        import uuid

        sqlite3.register_adapter(uuid.UUID, lambda u: str(u))
        sqlite3.register_converter("UUID", lambda b: uuid.UUID(b.decode("ascii")))
        log.info("SQLite UUID adapter registered successfully")
    except ImportError:
        pass  # sqlite3 not available
    except Exception as e:
        log.warning(f"Could not register SQLite UUID adapter: {e}")

    # Validate tier limits configuration loads
    try:
        from .config.tier_limits import TIER_CONFIG

        if TIER_CONFIG is None:
            raise ValueError(
                "Tier limits config failed to load. Check config/tier_limits.yaml"
            )

        # Log loaded config summary
        tier_names = list(TIER_CONFIG.tiers.keys())
        pack_names = list(TIER_CONFIG.enrichment_packs.keys())
        log.info(
            f"✅ Tier limits config loaded: "
            f"{len(tier_names)} tiers {tier_names}, "
            f"{len(pack_names)} packs {pack_names}"
        )

        # Validate margin meets threshold
        if not TIER_CONFIG.validate_margin():
            log.warning(
                f"⚠️  Margin ({TIER_CONFIG.pricing.margin_percentage:.1f}%) "
                f"below alert threshold ({TIER_CONFIG.features.margin_alert_threshold * 100:.1f}%)"
            )

    except FileNotFoundError as e:
        log.error(
            f"❌ CRITICAL: Tier limits config not found: {e}\n"
            f"Create config/tier_limits.yaml in repo root or pricing will be broken!"
        )
        if os.getenv("ENVIRONMENT") == "production":
            raise  # Fail fast in production
    except Exception as e:
        log.error(f"❌ Failed to load tier limits config: {e}")
        if os.getenv("ENVIRONMENT") == "production":
            raise  # Fail fast in production

    # Initialize domain family registry using new get_registry function
    from .services.domain_family import get_registry

    # The get_registry function handles DuckDB vs YAML automatically
    registry = get_registry()
    app.state.domain_families = registry

    # Log which implementation is being used
    try:
        from .services.domain_family_duckdb import DuckDBDomainRegistry

        registry_type = (
            "DuckDB" if isinstance(registry, DuckDBDomainRegistry) else "YAML"
        )
    except ImportError:
        registry_type = "YAML"

    log.info(
        f"Domain family registry initialized ({registry_type}): {registry.to_dict()['total_families']} families, {registry.to_dict()['total_domains']} domains"
    )

    # Create database tables in development
    if settings.ENV == "development":
        try:
            from .db import engine, Base

            # Ensure all models are imported and registered before creating tables
            from .model_defs import l2a_models  # noqa: F401
            from .model_defs import c2a_models  # noqa: F401
            from .model_defs import dedupe_models  # noqa: F401
            from .model_defs import reporting_models  # noqa: F401
            from .model_defs import routing_models  # noqa: F401
            from .model_defs import sheet_webhook_models  # noqa: F401

            async with engine.begin() as conn:
                await conn.run_sync(Base.metadata.create_all)
            log.info("Database tables created (development mode)")
        except Exception as e:
            log.warning(f"Could not create database tables: {e}")

    # Schedule SaaS event maintenance tasks
    try:
        from .services.saas_event_tasks import (
            purge_old_saas_events,
            refresh_saas_event_rollup,
        )
        from .services.webhook_retention import purge_expired_webhook_history

        scheduler.add_job(
            refresh_saas_event_rollup,
            trigger="cron",
            hour=3,
            minute=0,
            id="saas_event_refresh_mv",
            max_instances=1,
            replace_existing=True,
            misfire_grace_time=900,
        )
        scheduler.add_job(
            purge_old_saas_events,
            trigger="cron",
            day=1,
            hour=4,
            minute=0,
            id="saas_event_retention",
            max_instances=1,
            replace_existing=True,
            misfire_grace_time=3600,
        )
        scheduler.add_job(
            purge_expired_webhook_history,
            trigger="cron",
            hour=3,
            minute=30,
            id="webhook_history_retention",
            max_instances=1,
            replace_existing=True,
            misfire_grace_time=3600,
        )
        # Trial drip email scan — every 4 hours.
        async def _run_trial_drip():
            try:
                from .services.trial_drip_service import run_trial_drip_scan
                await run_trial_drip_scan()
            except Exception as exc:
                log.error("Trial drip scan failed: %s", exc, exc_info=True)

        scheduler.add_job(
            _run_trial_drip,
            trigger="cron",
            hour="*/4",
            minute=15,
            id="trial_drip_scan",
            max_instances=1,
            replace_existing=True,
            misfire_grace_time=3600,
        )
    except Exception as sched_err:
        log.warning("Failed to register SaaS event maintenance jobs: %s", sched_err)

    if TEST_MODE:
        log.info("APScheduler startup skipped in test mode")
        return

    # Start the scheduler
    scheduler.start()
    log.info("APScheduler started for credit alerts")

    # Start schedule manager worker (leader elected via DB lock).
    try:
        from .services.scheduler_service import get_scheduler

        schedule_manager_scheduler = await get_scheduler()
        await schedule_manager_scheduler.start()
        app.state.schedule_manager_scheduler = schedule_manager_scheduler
        log.info("Schedule manager scheduler initialized")
    except Exception as sched_err:
        log.error(
            "Failed to initialize schedule manager scheduler: %s",
            sched_err,
            exc_info=True,
        )

    try:
        from .events.transport import RedisStreamTransport
        from .events.collectors import BillingCollector, TelemetryCollector

        # Create Redis client instance
        redis_client = await aioredis.from_url(REDIS_URL, decode_responses=True)
        app.state.redis_client = redis_client

        # Initialize transport with client
        redis_transport = RedisStreamTransport(redis_client)

        # Initialize collectors
        app.state.billing_collector = BillingCollector(redis_transport)
        app.state.telemetry_collector = TelemetryCollector(redis_transport)

        log.info("Event collectors initialized successfully")
    except ImportError as e:
        log.warning(
            f"Event collectors not available - billing/telemetry features disabled: {e}"
        )
    except Exception as e:
        log.error(f"Failed to initialize Redis connection: {e}")

    # Debug: show current ENV setting
    from . import settings as saas_settings

    log.info(f"Running with ENV={saas_settings.ENV}")

    # Check L2A schema health
    try:
        import sqlite3
        from pathlib import Path

        def _path_exists(p) -> bool:
            """Check if path exists, handling both str and Path objects."""
            try:
                return Path(p).exists()
            except Exception:
                return False

        # Find the database
        db_locations = [
            r"C:\Users\Mikeh\Python\Fuzzy_Matcher\src\fmatch\saas\fmatch.db",
            Path.home() / ".foundrymatch" / "fmatch.db",
            Path("src/fmatch/saas/fmatch.db"),
            Path("fmatch.db"),
        ]

        db_path = None
        for path in db_locations:
            if _path_exists(path):
                db_path = str(path)
                break

        if db_path:
            conn = sqlite3.connect(db_path)
            cursor = conn.cursor()

            # Check for L2A tables
            cursor.execute(
                "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('l2a_policies', 'l2a_suggestions')"
            )
            tables = {row[0] for row in cursor.fetchall()}

            if "l2a_policies" in tables:
                # Check for required columns
                cursor.execute("PRAGMA table_info('l2a_policies');")
                columns = {row[1] for row in cursor.fetchall()}

                required_columns = {
                    "upgrade_min_gain",
                    "upgrade_min_score",
                    "upgrade_anchor_required",
                    "upgrade_enablement",
                    "last_run_at",
                }

                missing = required_columns - columns
                if missing:
                    log.warning(
                        f"[WARNING] L2A schema missing columns in l2a_policies: {sorted(missing)}. "
                        f"Run 'python fix_l2a_schema_async.py' to fix."
                    )
                else:
                    log.info("[OK] L2A policy schema check passed")

            if "l2a_suggestions" in tables:
                cursor.execute("PRAGMA table_info('l2a_suggestions');")
                sug_columns = {row[1] for row in cursor.fetchall()}

                required_sug_columns = {
                    "incumbent_account_id",
                    "incumbent_score",
                    "gain",
                    "reason_code",
                }

                missing_sug = required_sug_columns - sug_columns
                if missing_sug:
                    log.warning(
                        f"[WARNING] L2A schema missing columns in l2a_suggestions: {sorted(missing_sug)}. "
                        f"Run 'python fix_l2a_schema_async.py' to fix."
                    )
                else:
                    log.info("[OK] L2A suggestions schema check passed")

            conn.close()
        else:
            log.info("L2A schema check skipped - database not found")

    except Exception as e:
        log.warning(f"L2A schema health check failed: {e}")

    # Initialize ML services if models are available
    try:
        # Use MLFLOW_TRACKING_URI from settings if available
        mlflow_uri = getattr(settings, "MLFLOW_TRACKING_URI", None)

        # Skip MLflow initialization if not configured or in development
        if not mlflow_uri or settings.ENV == "development":
            log.info(
                "MLflow not configured or in development mode - skipping SmartAutoDetector initialization"
            )
            app.state.smart_detector = None
        else:
            detector = SmartAutoDetector(mlflow_uri)

            # Load models in a background thread to avoid blocking startup
            # Add timeout to prevent hanging
            import asyncio

            try:
                await asyncio.wait_for(
                    run_in_threadpool(detector.load_models),
                    timeout=5.0,  # 5 second timeout
                )
                app.state.smart_detector = detector
                log.info("SmartAutoDetector ML models loaded successfully.")
            except asyncio.TimeoutError:
                log.warning(
                    "MLflow model loading timed out - continuing without ML models"
                )
                app.state.smart_detector = None

    except Exception as e:
        log.warning(f"Could not initialize SmartAutoDetector: {e}")
        app.state.smart_detector = None


@app.on_event("shutdown")
async def handle_shutdown():
    """Graceful shutdown"""
    log.info("Shutting down gracefully...")

    # Stop accepting new jobs
    shutdown_event.set()

    # Wait for in-flight requests to complete (max 30 seconds)
    await asyncio.sleep(0.5)

    # Stop the scheduler
    scheduler.shutdown()
    log.info("APScheduler stopped")

    # Stop schedule manager worker
    try:
        schedule_manager_scheduler = getattr(
            app.state, "schedule_manager_scheduler", None
        )
        if schedule_manager_scheduler:
            await schedule_manager_scheduler.stop()
            log.info("Schedule manager scheduler stopped")
    except Exception as e:
        log.error("Error stopping schedule manager scheduler: %s", e, exc_info=True)

    # Shutdown engine bridge thread pool
    try:
        from fmatch.saas.engine_bridge import shutdown_executor

        shutdown_executor()
        log.info("Engine bridge thread pool shut down")
    except Exception as e:
        log.error(f"Error shutting down engine bridge: {e}")

    # Close connections
    if hasattr(app.state, "redis_client"):
        await app.state.redis_client.close()
        log.info("Redis connection closed")

    log.info("Shutdown complete")


# Debug: Print all registered routes
log.info("=== Registered Routes ===")
for route in app.routes:
    if hasattr(route, "path") and hasattr(route, "methods"):
        methods = ", ".join(route.methods) if route.methods else "ANY"
        log.info(f"  {methods:7} {route.path}")
log.info("=========================")


# Ensure dedupe schema is up to date
@app.on_event("startup")
async def ensure_dedupe_schema_startup():
    """Ensure dedupe_suggestions table has all required columns."""
    from .api.v2.dedupe import ensure_dedupe_schema
    from .db import AsyncSessionLocal

    try:
        async with AsyncSessionLocal() as db:
            await ensure_dedupe_schema(db)
            log.info("Dedupe schema check completed")
    except Exception as e:
        log.error(f"Failed to ensure dedupe schema: {e}")
        # Don't crash the server startup for this


# Production audit gate - fail fast if L2A routes are unprotected
@app.on_event("startup")
async def _assert_l2a_secured():
    """Verify all L2A routes are properly secured in production"""
    from . import settings as saas_settings

    if saas_settings.ENV.lower() != "production":
        log.info("L2A security audit: skipped (not in production)")
        return

    from fastapi.routing import APIRoute

    unsafe = []
    for r in app.routes:
        if isinstance(r, APIRoute) and r.path.startswith("/api/v2/l2a"):
            dep_names = [
                getattr(getattr(d, "call", None), "__name__", repr(d))
                for d in r.dependant.dependencies
            ]
            # Allow test-no-auth endpoint to be unprotected
            # Accept either require_account or require_user as valid auth
            if (
                "require_account" not in dep_names and "require_user" not in dep_names
            ) and r.path != "/api/v2/l2a/test-no-auth":
                unsafe.append(f"{list(r.methods)[0] if r.methods else '-'} {r.path}")

    if unsafe:
        raise RuntimeError(f"L2A routes missing auth in production: {unsafe}")
    log.info(
        f"L2A security audit: passed ({len([r for r in app.routes if isinstance(r, APIRoute) and r.path.startswith('/api/v2/l2a')])} routes secured)"
    )


# Debug L2A routes security
@app.on_event("startup")
async def _dump_l2a_routes_full():
    from fastapi.routing import APIRoute

    log.info("=== L2A Route Security Audit ===")
    for r in app.routes:
        if isinstance(r, APIRoute) and "/l2a" in r.path:
            deps = [
                getattr(getattr(d, "call", None), "__name__", repr(d))
                for d in r.dependant.dependencies
            ]
            has_auth = (
                "require_account" in deps
                or "require_user" in deps
                or "require_admin" in deps
            )
            status = (
                "[SECURED]"
                if has_auth or r.path == "/api/v2/l2a/test-no-auth"
                else "[UNPROTECTED]"
            )
            log.info(
                "L2A %s %-28s %s deps=%s",
                list(r.methods)[0] if r.methods else "-",
                r.path,
                status,
                deps,
            )


# L2A debug logging (request_id tracking handled by RequestContextMiddleware)
@app.middleware("http")
async def log_l2a_requests(request: Request, call_next):
    """Debug logging for L2A routes."""
    if "/l2a/" in str(request.url):
        log.info(
            f"L2A Request: {request.method} {request.url.path} Headers: {dict(request.headers)}"
        )
    return await call_next(request)


# Dev helper: inject X-Account-Id if missing in development
@app.middleware("http")
async def inject_dev_account_id(request: Request, call_next):
    """In development, ensure X-Account-Id header is always present."""
    import os

    if os.getenv("ENV", "").lower() in ("dev", "development"):
        # Check if X-Account-Id is missing (case-insensitive)
        has_account_id = any(
            h[0].lower() == b"x-account-id" for h in request.scope.get("headers", [])
        )

        if not has_account_id:
            # Inject the dev account ID
            from . import settings

            request.scope["headers"].append(
                (b"x-account-id", settings.DEV_ACCOUNT_ID.encode())
            )
            log.debug(
                f"Injected X-Account-Id: {settings.DEV_ACCOUNT_ID} for {request.url.path}"
            )

    return await call_next(request)


# Add rate limit exceeded handler if available
if RATE_LIMITING_ENABLED:
    app.state.limiter = limiter
    app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)

# Webhook security middleware (auth, signatures, rate limits)
app.add_middleware(
    WebhookAuthMiddleware,
    rate_limit_enabled=True,
    signature_verification_enabled=True,
    ip_allowlist_enabled=False,
    global_rate_limit=settings.WEBHOOK_RATE_LIMIT_GLOBAL,
    per_key_rate_limit=settings.WEBHOOK_RATE_LIMIT_PER_MINUTE,
)

# Add CORS middleware LAST so it runs FIRST (middleware runs in reverse order)
# This ensures CORS headers are added to ALL responses including errors
app.add_middleware(
    CORSMiddleware,
    allow_origins=allowed_origins,
    # Also accept any localhost port during local dev (http or https)
    allow_origin_regex=r"^(?:https?://(localhost|127\.0\.0\.1)(:\d+)?|https://([a-zA-Z0-9-]+\.)?foundryops\.io|https://([a-zA-Z0-9-]+\.)?foundryops-site\.pages\.dev|https://([a-zA-Z0-9-]+\.)?foundryops-admin\.pages\.dev|https://([a-zA-Z0-9-]+\.)?foundrymatch-admin\.pages\.dev|https://([a-zA-Z0-9-]+\.)?github\.io)$",
    allow_methods=["*"],
    allow_headers=[
        "Authorization",
        "Content-Type",
        "Accept",
        "ngrok-skip-browser-warning",
        "Idempotency-Key",
        "X-Requested-With",
        "X-Account-Id",
        "X-API-Key",
        "X-Webhook-Signature",
        "X-Webhook-Signature-256",
    ],
    allow_credentials=True,
    expose_headers=["Content-Disposition"],  # For file downloads
)


# ??????????????????????????????????????????????????????????????????????????? API endpoints ????????????????????????????????????????????????????????????????????????
@app.post("/api/projects/", response_model=ProjectRead, status_code=201)
async def create_project(
    payload: ProjectCreate,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> Project:
    """Create a new project shell for matching jobs."""
    project = Project(
        name=payload.name,
        description=payload.description,
        account_id=str(account_id),
        created_at=datetime.utcnow(),
        updated_at=datetime.utcnow(),
    )
    db.add(project)
    await db.commit()
    await db.refresh(project)
    return project


@app.patch("/api/projects/{project_id}/config", response_model=ProjectRead)
async def update_project_config(
    project_id: UUID,
    payload: ProjectConfigUpdate,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> Project:
    """Updates the configuration of a specific project.

    This endpoint allows for partial updates to a project's configuration fields
    like the matching algorithm, threshold, and various rule sets.

    Args:
        project_id: The UUID of the project to update.
        payload: The configuration data to update.
        db: The database session dependency.
        account_id: The authenticated user's account ID.

    Returns:
        The updated Project object.

    Raises:
        HTTPException: If the project is not found for the given account.
    """
    project_id_str = str(project_id)
    account_id_str = str(account_id)
    stmt = select(Project).where(
        Project.id == project_id_str, Project.account_id == account_id_str
    )
    project = (await db.execute(stmt)).scalar_one_or_none()
    if not project:
        raise HTTPException(404, "Project not found")

    update_data = payload.dict(exclude_unset=True)

    if "match_algorithm" in update_data and isinstance(
        update_data["match_algorithm"], MatchAlgorithm
    ):
        #  ??????  store the *member name* in snake???case
        update_data["match_algorithm"] = update_data["match_algorithm"].name.lower()

    for key, value in update_data.items():
        setattr(project, key, value)

    project.updated_at = datetime.utcnow()
    db.add(project)
    await db.commit()
    await db.refresh(project)
    return project


@app.post(
    "/api/projects/{project_id}/jobs/{mode}",
    response_model=JobAccepted,
    status_code=202,
)
async def upload_file_and_enqueue_job(
    project_id: UUID,
    mode: Literal["duplicate", "match"],
    file: UploadFile = File(...),
    ruleset: Literal["default"] = "default",
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> JobAccepted:
    # Normalize mode: "duplicate" -> "dedupe"
    if mode == "duplicate":
        mode = "dedupe"
    # 1????????validate project
    project_id_str = str(project_id)
    account_id_str = str(account_id)
    stmt = select(Project).where(
        Project.id == project_id_str, Project.account_id == account_id_str
    )
    project = (await db.execute(stmt)).scalar_one_or_none()
    if not project:
        raise HTTPException(404, "Project not found")

    algorithm = getattr(project, "match_algorithm", MatchAlgorithm.wratio)
    threshold = getattr(project, "match_threshold", 80)
    preprocessing_rules = getattr(project, "preprocessing_rules", []) or []
    custom_match_rules = getattr(project, "custom_match_rules", []) or []

    # 2????????persist upload to tmp
    tmp_dir = Path(os.getenv("TMPDIR", Path.cwd()))
    tmp_dir.mkdir(parents=True, exist_ok=True)
    upload_path = tmp_dir / file.filename
    upload_path.write_bytes(await file.read())

    # 3????????register job
    job_id = str(uuid4())
    await write_job_row(
        job_id,
        project_id=str(project_id),
        account_id=str(account_id),
        status="queued",
        mode=mode,
        ruleset=ruleset,
        filename=file.filename,
        # Persist worker config so RQ job only needs job_id
        config={
            "mode": mode,
            "source_url": str(upload_path),
            "reference_url": str(upload_path) if mode == "match" else None,
            "algorithm": algorithm,
            "threshold": threshold,
            "maps": custom_match_rules,
            "src_id_col": None,
            "ref_id_col": None,
            "webhook_url": None,
        },
    )

    # 4????????build config dataclass
    if mode == "match":
        cfg = MatchConfig(
            job_id=job_id,
            src_path=str(upload_path),
            ref_path=str(upload_path),  # TODO: real reference path
            ruleset_name=ruleset,
            algorithm=algorithm,
            threshold=threshold,
            preprocessing_rules=preprocessing_rules,
            custom_match_rules=custom_match_rules,
        )
    else:
        cfg = DedupeConfig(
            job_id=job_id,
            input_path=str(upload_path),
            rec_id_col=INPUT_ID_COLUMN,
            ruleset_name=ruleset,
            algorithm=algorithm,
            threshold=threshold,
            dup_block_limit=settings.DEFAULT_DUP_BLOCK_LIMIT,
            block_key_length=settings.DEFAULT_BLOCK_KEY_LENGTH,
            preprocessing_rules=preprocessing_rules,
            custom_match_rules=custom_match_rules,
        )

    # 5????????enqueue using **safe serialisation**
    from fmatch.saas import worker as fm_worker  # Lazy import: new path after refactor

    # On Windows, job_timeout uses SIGALRM which doesn't exist, so skip it
    if sys.platform.startswith("win"):
        job = q.enqueue(
            fm_worker.run_matching_job,
            job_id,
        )
    else:
        job = q.enqueue(
            fm_worker.run_matching_job,
            job_id,
            job_timeout="1h",
        )
    try:
        log.info(
            "ENQ job_id=%s rq_id=%s queue=%s redis=%s",
            job_id,
            getattr(job, "id", "?"),
            q.name,
            _redact_redis_url(REDIS_URL),
        )
    except Exception:
        log.info(
            "Enqueued job %s to queue=%s redis=%s for project %s",
            job_id,
            q.name,
            _redact_redis_url(REDIS_URL),
            project_id,
        )

    # Optional: Process in-app for debugging (bypasses queue)
    if os.getenv("FM_PROCESS_IN_APP") == "1":
        log.info("FM_PROCESS_IN_APP=1 detected, running job synchronously")
        fm_worker.run_matching_job(job_id)

    return JobAccepted(job_id=job_id)


@app.get("/api/jobs/{job_id}")
async def get_job(job_id: str) -> Dict[str, Any]:
    job = await fetch_job(job_id)
    if not job:
        raise HTTPException(404, "Job not found")
    return job


@app.get("/api/v1/jobs/{job_id}/mappings")
async def get_job_mappings(
    job_id: str, threshold: int = Query(75), enable_multi_algo: bool = Query(True)
):
    """Get mapping suggestions using cached profile data"""

    if job_id.startswith("temp-"):
        # Get cached file profiles
        redis_conn = redis.from_url(REDIS_URL)

        # Get the latest source and reference file hashes
        source_hash = redis_conn.get("latest_source_file")
        ref_hash = redis_conn.get("latest_reference_file")

        if not source_hash:
            # For dedupe mode, only source is needed
            return {
                "mappings": [],
                "message": "Please profile files first",
                "source_id_suggestion": None,
                "ref_id_suggestion": None,
            }

        # Get the cached profile data
        source_data = json.loads(
            redis_conn.get(f"file_profile:{source_hash.decode()}") or "{}"
        )
        ref_data = (
            json.loads(redis_conn.get(f"file_profile:{ref_hash.decode()}") or "{}")
            if ref_hash
            else None
        )

        if not source_data:
            return {
                "mappings": [],
                "message": "File profile data not found. Please re-upload files.",
                "source_id_suggestion": None,
                "ref_id_suggestion": None,
            }

        # For dedupe mode, use source columns for both
        if not ref_data:
            ref_data = source_data

        # Generate mappings using fuzzy matching
        from rapidfuzz import fuzz

        mappings = []
        source_columns = source_data.get("columns", [])
        ref_columns = ref_data.get("columns", [])
        mapped_refs = set()

        # Simple fuzzy matching logic
        for src_col in source_columns:
            src_name = src_col["name"]
            best_match = None
            best_score = 0

            for ref_col in ref_columns:
                ref_name = ref_col["name"]

                # Skip if already mapped
                if ref_name in mapped_refs and ref_data != source_data:
                    continue

                # Use rapidfuzz for similarity scoring
                score = fuzz.token_set_ratio(
                    src_name.lower().replace("_", " ").replace("-", " "),
                    ref_name.lower().replace("_", " ").replace("-", " "),
                )

                if score > best_score and score >= threshold:
                    best_score = score
                    best_match = ref_name

            if best_match:
                mappings.append(
                    {
                        "source": src_name,
                        "ref": best_match,
                        "weight": best_score / 100.0,  # Normalize to 0-1
                        "preferred_algo": "WRatio",
                        "algo_confidence": best_score / 100.0,
                    }
                )
                mapped_refs.add(best_match)

        return {
            "mappings": mappings,
            "source_id_suggestion": source_data.get("id_column"),
            "ref_id_suggestion": ref_data.get("id_column") if ref_data else None,
            "confidence_scores": {m["source"]: m["weight"] for m in mappings},
            "message": f"Found {len(mappings)} potential mappings"
            if mappings
            else "No automatic mappings found",
        }

    # For real jobs
    raise HTTPException(404, f"Job {job_id} not found")


@app.websocket("/ws/jobs/{job_id}/progress")
async def websocket_progress(
    websocket: WebSocket,
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account_ws),
):
    # --- authz ---
    details = await fetch_job(job_id)
    if not details:
        raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION)

    project_id = details.get("project_id")
    if not project_id:
        raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION)
    stmt = select(Project.id).where(
        Project.id == project_id, Project.account_id == str(account_id)
    )
    if (await db.execute(stmt)).scalar_one_or_none() is None:
        raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION)

    await websocket.accept()

    pubsub = async_redis_conn.pubsub()
    channel = f"job:{job_id}:progress"
    await pubsub.subscribe(channel)

    try:
        async for msg in pubsub.listen():
            if msg["type"] == "message":
                await websocket.send_text(msg["data"])
    except WebSocketDisconnect:
        pass
    finally:
        await pubsub.unsubscribe(channel)
        await pubsub.close()


# ????????????????????????????????????????????????????????? Universal Match Helpers ????????????????????????????????????????????????????????????
async def get_or_create_project_for_request(
    db: AsyncSession,
    account_id: UUID,
    request: UniversalMatchRequest,
    detected_config: Dict[str, Any],
) -> Project:
    """Get or create a project for the universal match request."""
    account_id_str = str(account_id)
    # Try to find existing project with similar config
    stmt = (
        select(Project)
        .where(
            Project.account_id == account_id_str,
            Project.name.like(f"Universal Match - {request.mode}%"),
        )
        .order_by(Project.created_at.desc())
        .limit(1)
    )

    result = await db.execute(stmt)
    project = result.scalar_one_or_none()

    if not project:
        # Create new project
        project = Project(
            name=f"Universal Match - {request.mode}",
            description="Auto-created for universal match API",
            account_id=account_id_str,
            match_algorithm=MatchAlgorithm(detected_config["algorithm"]),
            match_threshold=detected_config["threshold"],
            match_ruleset=detected_config.get("ruleset_name", "default"),
            preprocessing_rules=detected_config.get("preprocessing_rules", []),
            custom_match_rules=detected_config.get("custom_match_rules", []),
            created_at=time.time(),
            updated_at=time.time(),
        )
        db.add(project)
        await db.commit()
        await db.refresh(project)

    return project


async def save_dataframe_to_temp(df: pd.DataFrame, filename: str) -> str:
    """Save DataFrame to temporary file."""
    tmp_dir = Path(os.getenv("TMPDIR", Path.cwd()))
    tmp_dir.mkdir(parents=True, exist_ok=True)
    filepath = tmp_dir / filename
    df.to_csv(filepath, index=False)
    return str(filepath)


async def run_sync_match(
    source_df: pd.DataFrame,
    ref_df: Optional[pd.DataFrame],
    config: Dict[str, Any],
    mode: str,
) -> List[Dict[str, Any]]:
    """Run synchronous matching for small datasets."""
    from fmatch.core.engine import process_dataframe, MatchConfig, DedupeConfig

    # Hard sync cap to prevent resource exhaustion
    HARD_SYNC_LIMIT = 50_000  # Absolute max
    total_rows = len(source_df) + (len(ref_df) if ref_df is not None else 0)
    if total_rows > HARD_SYNC_LIMIT:
        raise HTTPException(
            413,
            f"Dataset too large for sync processing ({total_rows} rows > {HARD_SYNC_LIMIT} max). Please use async processing.",
        )

    if mode == "match" and ref_df is not None:
        # Create proper config object
        match_config = MatchConfig(
            job_id=str(uuid4()),
            src_path=source_df,  # Can accept DataFrame directly
            ref_path=ref_df,
            algorithm=config.get("algorithm", "WRatio"),
            threshold=config.get("threshold", 80),
            maps=config.get("maps", []),
            ruleset_name=config.get("ruleset_name", "default"),
            block_key_length=3,
            apply_blocking=True,
        )

        # Call process_dataframe
        results_df = await run_in_threadpool(
            process_dataframe, source_df, ref_df, match_config
        )
    else:  # dedupe mode
        dedupe_config = DedupeConfig(
            job_id=str(uuid4()),
            input_path=source_df,
            rec_id_col=config.get("source_id_column", "id"),
            algorithm=config.get("algorithm", "WRatio"),
            threshold=config.get("threshold", 80),
            maps=config.get("maps", []),
            ruleset_name=config.get("ruleset_name", "default"),
            dup_block_limit=2000,
            block_key_length=3,
        )

        results_df = await run_in_threadpool(
            process_dataframe,
            source_df,
            None,  # No ref_df for dedupe
            dedupe_config,
        )

    # Convert DataFrame to list of dicts
    if results_df is not None and not results_df.empty:
        return results_df.to_dict(orient="records")
    return []


def format_results(results: List[Dict], max_results: Optional[int]) -> List[Dict]:
    """Format and limit results."""
    if max_results and len(results) > max_results:
        return results[:max_results]
    return results


def estimate_processing_time(total_rows: int) -> int:
    """Estimate processing time based on row count."""
    # Simple estimation: ~1 second per 1000 rows
    return max(5, total_rows // 1000)


# Import shared webhook validation
from .webhook_safe import validate_webhook_url


async def read_google_sheet(sheet_source: GoogleSheetSource) -> pd.DataFrame:
    """
    Read data from Google Sheets.
    In production, this would use the Google Sheets API.
    """
    # For demo purposes, return sample data
    # In production: use googleapiclient.discovery to fetch real data

    # Construct Google Sheets URL
    sheet_url = f"https://docs.google.com/spreadsheets/d/{sheet_source.sheet_id}/export?format=csv&range={sheet_source.range}"

    try:
        # In production, you'd use proper Google Sheets API
        # For now, try to read as CSV export (works for public sheets)
        df = pd.read_csv(sheet_url)
        return df
    except Exception as e:
        # Return demo data for testing
        log.warning(f"Could not fetch Google Sheet {sheet_source.sheet_id}: {e}")

        # Demo data
        if "customer" in sheet_source.range.lower():
            return pd.DataFrame(
                {
                    "id": ["1", "2", "3"],
                    "name": ["John Smith", "Jane Doe", "Bob Wilson"],
                    "email": [
                        "john@example.com",
                        "jane@example.com",
                        "bob@example.com",
                    ],
                    "company": ["Acme Corp", "Tech Inc", "Sales Co"],
                }
            )
        else:
            return pd.DataFrame(
                {
                    "id": ["A", "B", "C"],
                    "name": ["John Smyth", "Jane Doe", "Robert Wilson"],
                    "email": [
                        "j.smith@example.com",
                        "jane@example.com",
                        "bobby@example.com",
                    ],
                    "company": ["Acme Corporation", "Tech Inc", "Sales Company"],
                }
            )


# ???????????????????????????????????????????????????????????? Universal Match Endpoint ???????????????????????????????????????????????????
@app.post("/api/v1/match/universal", response_model=UniversalMatchResponse)
async def universal_match(
    request: UniversalMatchRequest,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> UniversalMatchResponse:
    """
    Universal matching endpoint that auto-detects everything.
    Works for Google Sheets, Clay.com, Zapier, and any integration.
    """

    # Initialize tracker
    tracker = SimpleUsageTracker(db)

    # Calculate credits needed
    credits_needed = calculate_credits(request)

    # Check credit limit
    has_credits = await tracker.check_credit_limit(account_id, credits_needed)
    if not has_credits:
        raise HTTPException(
            status_code=402,  # Payment Required
            detail={
                "error": "Insufficient credits",
                "credits_needed": credits_needed,
                "message": "Please purchase additional credits to continue",
            },
        )

    # 0. Validate webhook URL if provided
    if request.webhook_url and not validate_webhook_url(
        request.webhook_url, require_https=settings.REQUIRE_HTTPS_WEBHOOKS
    ):
        raise HTTPException(400, "Invalid webhook URL")

    # 1. Parse data from any source
    source_df = await parse_data_source(request.source)
    ref_df = await parse_data_source(request.reference) if request.reference else None

    # 2. Auto-detect configuration using your existing intelligence
    detected_config = await auto_detect_configuration(
        source_df=source_df,
        ref_df=ref_df,
        mode=request.mode,
        user_overrides={
            "algorithm": request.algorithm,
            "threshold": request.threshold,
            "blocking_strategy": request.blocking_strategy,
        },
    )

    # 3. Determine if sync or async based on size and memory
    total_rows = len(source_df) + (len(ref_df) if ref_df is not None else 0)

    # Estimate memory usage (rough: 1KB per row)
    estimated_memory_mb = total_rows / 1000

    # Force async for large datasets or if webhook requested
    is_sync = (
        total_rows < settings.SYNC_ROW_LIMIT
        and estimated_memory_mb < settings.MAX_SYNC_MEMORY_MB
        and not request.webhook_url
    )

    # 4. Create or use existing project
    project = await get_or_create_project_for_request(
        db, account_id, request, detected_config
    )

    if is_sync:
        # Small dataset - process immediately
        results = await run_sync_match(source_df, ref_df, detected_config, request.mode)

        # Generate job ID for tracking
        job_id = str(uuid4())

        # Track usage AFTER successful processing
        await tracker.track_usage(
            account_id=account_id,
            credits=credits_needed,
            event_type="match_job",
            metadata={
                "job_id": job_id,
                "mode": request.mode,
                "source_rows": len(source_df),
                "reference_rows": len(ref_df) if ref_df else 0,
                "endpoint": "/api/v1/match/universal",
                "sync": True,
            },
        )

        # Deduct credits from balance
        account = await db.get(Account, account_id)
        if account:
            account.credits_balance -= credits_needed
            db.add(account)
            await db.commit()

        return UniversalMatchResponse(
            job_id=job_id,
            status="completed",
            results=format_results(results, request.max_results),
            auto_detected_config=detected_config,
            estimated_time_seconds=0,
            metadata={
                "rows_processed": total_rows,
                "matches_found": len(results),
                "processing_time_ms": 0,  # Will be set by middleware
                "credits_used": credits_needed,
            },
        )

    else:
        # Large dataset - queue for async processing
        job_id = str(uuid4())

        # Save data to temp files
        source_path = await save_dataframe_to_temp(source_df, f"{job_id}_source.csv")
        ref_path = (
            await save_dataframe_to_temp(ref_df, f"{job_id}_ref.csv")
            if ref_df
            else None
        )

        # Register job and persist worker config
        await write_job_row(
            job_id,
            project_id=str(project.id),
            account_id=str(account_id),
            status="queued",
            mode=request.mode,
            webhook_url=request.webhook_url,
            ruleset=detected_config.get("ruleset_name", "default"),
            filename=f"{job_id}_source.csv",
            config={
                "mode": request.mode,
                "source_url": source_path,
                "reference_url": (ref_path or source_path)
                if request.mode == "match"
                else None,
                "algorithm": detected_config["algorithm"],
                "threshold": detected_config["threshold"],
                "maps": detected_config.get("maps", []),
                "src_id_col": detected_config.get("source_id_column"),
                "ref_id_col": detected_config.get("reference_id_column"),
                "webhook_url": request.webhook_url,
            },
        )

        # Build config
        if request.mode == "match":
            cfg = MatchConfig(
                job_id=job_id,
                src_path=source_path,
                ref_path=ref_path or source_path,  # Fallback if no reference
                ruleset_name=detected_config.get("ruleset_name", "default"),
                algorithm=MatchAlgorithm(detected_config["algorithm"]),
                threshold=detected_config["threshold"],
                preprocessing_rules=detected_config.get("preprocessing_rules", []),
                custom_match_rules=detected_config.get("custom_match_rules", []),
            )
        else:
            cfg = DedupeConfig(
                job_id=job_id,
                input_path=source_path,
                rec_id_col=detected_config["source_id_column"],
                ruleset_name=detected_config.get("ruleset_name", "default"),
                algorithm=MatchAlgorithm(detected_config["algorithm"]),
                threshold=detected_config["threshold"],
                dup_block_limit=settings.DEFAULT_DUP_BLOCK_LIMIT,
                block_key_length=settings.DEFAULT_BLOCK_KEY_LENGTH,
                preprocessing_rules=detected_config.get("preprocessing_rules", []),
                custom_match_rules=detected_config.get("custom_match_rules", []),
            )

        # Queue job with job_id only (worker loads config from DB)
        # On Windows, job_timeout uses SIGALRM which doesn't exist, so skip it
        if sys.platform.startswith("win"):
            job = q.enqueue(
                "fmatch.saas.worker.run_matching_job",
                job_id,
            )
        else:
            job = q.enqueue(
                "fmatch.saas.worker.run_matching_job",
                job_id,
                job_timeout="1h",
            )
        log.info(
            "Enqueued job %s to queue=%s redis=%s",
            job_id,
            q.name,
            _redact_redis_url(REDIS_URL),
        )

        # Optional: Process in-app for debugging (bypasses queue)
        if os.getenv("FM_PROCESS_IN_APP") == "1":
            log.info("FM_PROCESS_IN_APP=1 detected, running job synchronously")
            from fmatch.saas.worker import run_matching_job

            run_matching_job(job_id)

        # Track usage for async job (will be updated by worker on completion)
        await tracker.track_usage(
            account_id=account_id,
            credits=credits_needed,
            event_type="match_job",
            metadata={
                "job_id": job_id,
                "mode": request.mode,
                "source_rows": len(source_df),
                "reference_rows": len(ref_df) if ref_df else 0,
                "endpoint": "/api/v1/match/universal",
                "sync": False,
                "status": "pending",
            },
        )

        # Deduct credits from balance (pre-charge for async)
        account = await db.get(Account, account_id)
        if account:
            account.credits_balance -= credits_needed
            db.add(account)
            await db.commit()

        return UniversalMatchResponse(
            job_id=job_id,
            status="processing",
            results_url=f"/api/v1/jobs/{job_id}/results",
            auto_detected_config=detected_config,
            estimated_time_seconds=estimate_processing_time(total_rows),
            metadata={
                "webhook_url": request.webhook_url,
                "rows_to_process": total_rows,
                "credits_used": credits_needed,
            },
        )


# ?????????????????????????????????????????????????????? iPaaS-Specific Endpoints ?????????????????????????????????????????????????????????
@app.post(
    "/api/v1/match/simple",
    response_model=UniversalMatchResponse,
    tags=["iPaaS"],
    dependencies=[Depends(limiter.limit(settings.RATE_LIMIT_SIMPLE_MATCH))]
    if RATE_LIMITING_ENABLED
    else [],
)
async def simple_match_api(
    request: Request,
    mode: Literal["dedupe", "match"] = Form(...),
    source_data: str = Form(..., description="CSV or JSON string"),
    source_format: Literal["csv", "json"] = Form("csv"),
    reference_data: Optional[str] = Form(None),
    reference_format: Optional[Literal["csv", "json"]] = Form("csv"),
    threshold: int = Form(80, ge=0, le=100),
    webhook_url: Optional[str] = Form(None),
    api_key: Optional[str] = Form(None),
    x_api_key: Optional[str] = Header(None),
    authorization: Optional[str] = Header(None),
    db: AsyncSession = Depends(get_session),
) -> UniversalMatchResponse:
    """
    Simplified endpoint for iPaaS platforms (Zapier, Make, etc).
    Accepts form data and auto-detects everything.
    """

    # Validate webhook URL first
    if webhook_url and not validate_webhook_url(
        webhook_url, require_https=settings.REQUIRE_HTTPS_WEBHOOKS
    ):
        raise iPaaSError(
            "Invalid webhook URL",
            details={
                "webhook_url": webhook_url,
                "reason": "URL points to forbidden host or uses invalid protocol",
                "allowed_protocols": ["https", "http (dev only)"],
                "forbidden_hosts": ["localhost", "127.0.0.1", "metadata endpoints"],
            },
        )

    # Use flexible authentication (supports all iPaaS patterns)
    auth_token = api_key or x_api_key or authorization
    if not auth_token:
        raise iPaaSError(
            "Authentication required",
            details={
                "accepted_methods": [
                    "Form field: api_key",
                    "Header: X-API-Key",
                    "Header: Authorization: Bearer <token>",
                ],
                "get_api_key": "POST /api/keys (requires JWT auth)",
            },
            status_code=401,
        )

    try:
        account_id = await get_account_from_api_key_or_header(
            api_key or x_api_key, authorization, db
        )
    except HTTPException as e:
        raise iPaaSError(
            "Authentication failed",
            details={
                "error": str(e.detail),
                "provided_method": "API key"
                if (api_key or x_api_key)
                else "Bearer token",
            },
            status_code=401,
        )

    # Convert form data to universal format
    universal_request = UniversalMatchRequest(
        mode=mode,
        source=DataSource(data=source_data, format=source_format),
        reference=DataSource(data=reference_data, format=reference_format)
        if reference_data
        else None,
        threshold=threshold,
        webhook_url=webhook_url,
    )

    # Use the same universal endpoint logic with error handling
    try:
        return await universal_match(universal_request, db, account_id)
    except ValueError as e:
        raise iPaaSError(
            "Data validation error",
            details={
                "error": str(e),
                "format": source_format,
                "tips": [
                    "Ensure CSV has headers",
                    "Check JSON is valid array of objects",
                    "Verify data size is under 100MB",
                ],
            },
        )
    except Exception as e:
        log.error(f"iPaaS match error: {e}", exc_info=True)
        raise iPaaSError(
            "Processing error",
            details={"error": str(e), "type": type(e).__name__},
            status_code=500,
        )


# ???????????????????????????????????? Google Sheets Integration Endpoint ????????????????????????????????????????????????
@app.post("/api/v1/match/sheets", response_model=UniversalMatchResponse)
async def google_sheets_match(
    mode: Literal["dedupe", "match"] = Form(...),
    source_sheet_id: str = Form(..., description="Google Sheet ID"),
    source_range: str = Form("A:Z", description="Sheet range (e.g., 'Sheet1!A:F')"),
    reference_sheet_id: Optional[str] = Form(None),
    reference_range: Optional[str] = Form("A:Z"),
    threshold: int = Form(80, ge=0, le=100),
    webhook_url: Optional[str] = Form(None),
    google_api_key: Optional[str] = Form(None, description="Optional Google API key"),
    authorization: Optional[str] = Header(None),
    db: AsyncSession = Depends(get_session),
) -> UniversalMatchResponse:
    """
    Direct Google Sheets integration endpoint.
    Fetches data directly from Google Sheets and runs matching.
    """

    # Use flexible authentication (Google API key can be used as auth)
    account_id = await get_account_from_api_key_or_header(google_api_key, authorization)

    # Fetch data from Google Sheets (simplified version)
    # In production, you'd use the Google Sheets API
    try:
        # For now, return a helpful message
        # In production: source_data = await fetch_google_sheet(source_sheet_id, source_range, google_api_key)
        raise HTTPException(
            501,
            "Google Sheets direct integration coming soon. "
            "Please export your sheet to CSV and use the /api/v1/match endpoint.",
        )
    except Exception as e:
        log.error(f"Failed to fetch Google Sheets data: {e}")
        raise HTTPException(400, f"Failed to fetch sheet data: {str(e)}")


# ???????????????????????????????????? Results Streaming Endpoint ????????????????????????????????????????????????
@app.get(
    "/api/v1/jobs/{job_id:path}/results",
    tags=["iPaaS"],
    dependencies=[Depends(limiter.limit(settings.RATE_LIMIT_RESULTS))]
    if RATE_LIMITING_ENABLED
    else [],
)
async def get_job_results(
    request: Request,
    job_id: str,
    format: Literal["json", "csv", "parquet"] = Query("json"),
    limit: Optional[int] = Query(None, description="Number of results to return"),
    offset: int = Query(0, description="Number of results to skip"),
    authorization: Optional[str] = Header(None),
):
    """
    Stream results in any format.
    Supports pagination and multiple output formats.
    """

    # Optional authentication - make it more flexible for iPaaS
    if authorization:
        try:
            await get_account_from_api_key_or_header(None, authorization)
        except HTTPException:
            pass  # Allow unauthenticated access for job results

    # Fetch job details
    job = await fetch_job(job_id)
    if not job:
        raise HTTPException(404, "Job not found")

    # Check job status
    if job.get("status") != "completed":
        raise HTTPException(
            404, f"Results not available. Job status: {job.get('status', 'unknown')}"
        )

    # Get results path
    results_path = job.get("results_path")
    if not results_path or not Path(results_path).exists():
        # Try to find results in output directory
        if "config" in job and "output_dir" in job["config"]:
            output_dir = Path(job["config"]["output_dir"])
            # Look for match or dedupe results
            possible_files = list(output_dir.glob("match-*.parquet")) + list(
                output_dir.glob("dupes-*.parquet")
            )
            if possible_files:
                results_path = str(possible_files[0])
            else:
                raise HTTPException(404, "Results file not found")
        else:
            raise HTTPException(404, "Results file not found")

    # Load results
    try:
        df = pd.read_parquet(results_path)

        # Clean NaN values for JSON serialization
        if not df.empty:
            # Replace NaN with None for JSON compatibility
            df = df.replace({np.nan: None, np.inf: None, -np.inf: None})

        # Get total count before pagination
        total_results = len(df)

        # Apply pagination
        if limit:
            df = df.iloc[offset : offset + limit]
        else:
            df = df.iloc[offset:]

        # Format response based on requested format
        if format == "json":
            return {
                "job_id": job_id,
                "total_results": total_results,
                "returned_results": len(df),
                "offset": offset,
                "limit": limit,
                "results": df.to_dict(orient="records"),
            }

        elif format == "csv":
            # Stream CSV file
            csv_data = df.to_csv(index=False)
            return Response(
                content=csv_data,
                media_type="text/csv",
                headers={
                    "Content-Disposition": f"attachment; filename=results_{job_id}.csv",
                    "X-Total-Results": str(total_results),
                    "X-Returned-Results": str(len(df)),
                },
            )

        else:  # parquet
            # Stream Parquet file
            buffer = io.BytesIO()
            df.to_parquet(buffer, index=False)
            buffer.seek(0)

            return Response(
                content=buffer.getvalue(),
                media_type="application/octet-stream",
                headers={
                    "Content-Disposition": f"attachment; filename=results_{job_id}.parquet",
                    "X-Total-Results": str(total_results),
                    "X-Returned-Results": str(len(df)),
                },
            )

    except Exception as e:
        log.error(f"Failed to read results for job {job_id}: {e}")
        raise HTTPException(500, f"Failed to read results: {str(e)}")


@app.post(
    "/api/v1/integrations/google-sheets/match", response_model=UniversalMatchResponse
)
async def google_sheets_match_wrapper(
    request: GoogleSheetsRequest,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> UniversalMatchResponse:
    """
    Google Sheets integration - reads sheets and runs matching.
    Simple 5-line wrapper around the universal endpoint!
    """
    # Read sheets
    source_data = await read_google_sheet(request.source)
    reference_data = (
        await read_google_sheet(request.reference) if request.reference else None
    )

    # Call universal endpoint
    return await universal_match(
        UniversalMatchRequest(
            mode=request.mode,
            source=DataSource(
                data=source_data.to_json(orient="records"), format="json"
            ),
            reference=DataSource(
                data=reference_data.to_json(orient="records"), format="json"
            )
            if reference_data is not None
            else None,
            threshold=request.threshold,
            algorithm=request.algorithm,
            webhook_url=request.webhook_url,
        ),
        db,
        account_id,
    )


# API Keys Management routes moved to api/v2/api_keys.py
# Reload trigger: 2025-09-20 06:50


# ?????????????????????????????????????????????????????????????????? Stripe Webhook ??????????????????????????????????????????????????????????????????
@app.post("/api/billing/stripe-webhook", include_in_schema=False)
async def stripe_webhook(
    request: Request,
    db: AsyncSession = Depends(get_session),
):
    """
    Handle Stripe webhook events for automatic credit addition.
    This endpoint should be configured in your Stripe dashboard.
    """
    # Get the webhook payload and signature
    payload = await request.body()
    sig_header = request.headers.get("stripe-signature")

    if not sig_header:
        raise HTTPException(400, "Missing stripe-signature header")

    try:
        from .billing.stripe_integration import StripeWebhookHandler

        # Initialize handler
        handler = StripeWebhookHandler(db)

        # Verify and parse webhook
        event = await handler.verify_webhook(payload, sig_header)

        # Process the event
        result = await handler.handle_event(event)

        log.info(f"Stripe webhook processed: {event.type} - {result}")

        return {"received": True, "result": result}

    except ValueError as e:
        log.error(f"Invalid webhook payload: {e}")
        raise HTTPException(400, f"Invalid payload: {str(e)}")
    except Exception as e:
        log.error(f"Stripe webhook error: {e}", exc_info=True)
        # Return 200 to prevent Stripe from retrying
        # But log the error for investigation
        return {"received": True, "error": str(e)}


# @app.post("/api/billing/create-checkout", tags=["Billing"])
# async def create_checkout_session(
#     credits: int = Body(..., gt=0, description="Number of credits to purchase"),
#     success_url: str = Body(..., description="URL to redirect after successful payment"),
#     cancel_url: str = Body(..., description="URL to redirect if payment cancelled"),
#     db: AsyncSession = Depends(get_session),
#     account_id: UUID = Depends(get_current_account),
# ) -> Dict[str, Any]:
#     """
#     Create a Stripe checkout session for credit purchase.
#     Returns a checkout URL to redirect the user to.
#     """
#     try:
#         from .billing.stripe_integration import create_checkout_session as stripe_checkout
#
#         # Get account for email
#         account = await db.get(Account, account_id)
#         if not account:
#             raise HTTPException(404, "Account not found")
#
#         # Create checkout session
#         result = await stripe_checkout(
#             account_id=account_id,
#             credits=credits,
#             success_url=success_url,
#             cancel_url=cancel_url,
#             customer_email=account.name  # Using name as email for now
#         )
#
#         return result
#
#     except Exception as e:
#         log.error(f"Failed to create checkout session: {e}")
#         raise HTTPException(500, f"Failed to create checkout: {str(e)}")


# @app.post("/api/billing/create-subscription", tags=["Billing"])
# async def create_subscription(
#     price_id: str = Body(..., description="Stripe price ID for subscription plan"),
#     trial_days: int = Body(0, description="Number of trial days"),
#     db: AsyncSession = Depends(get_session),
#     account_id: UUID = Depends(get_current_account),
# ) -> Dict[str, Any]:
#     """
#     Create a subscription for recurring credit allocation.
#     Use predefined price IDs from your Stripe dashboard.
#     """
#     try:
#         from .billing.stripe_integration import create_subscription as stripe_subscription
#
#         # Get account
#         account = await db.get(Account, account_id)
#         if not account:
#             raise HTTPException(404, "Account not found")
#
#         # Create subscription
#         result = await stripe_subscription(
#             account_id=account_id,
#             price_id=price_id,
#             customer_email=account.name,  # Using name as email
#             trial_days=trial_days
#         )
#
#         # Update account with subscription ID
#         account.stripe_subscription_id = result["subscription_id"]
#         account.stripe_customer_id = result["customer_id"]
#         db.add(account)
#         await db.commit()
#
#         return result
#
#     except Exception as e:
#         log.error(f"Failed to create subscription: {e}")
#         raise HTTPException(500, f"Failed to create subscription: {str(e)}")


# ?????????????????????????????????????????????????????????????????? Billing Endpoints ??????????????????????????????????????????????????????????????????
# @app.post("/api/billing/purchase-credits", tags=["Billing"])
# async def purchase_credits(
#     amount: int = Body(..., gt=0, description="Number of credits to purchase"),
#     db: AsyncSession = Depends(get_session),
#     account_id: UUID = Depends(get_current_account)
# ) -> Dict[str, Any]:
#     """
#     Purchase credits for your account.
#     In production, this would integrate with a payment processor.
#     """
#     # Get account
#     account = await db.get(Account, account_id)
#     if not account:
#         raise HTTPException(404, "Account not found")
#
#     # Add credits
#     account.credits_balance += amount
#
#     # Track the purchase as negative usage (credit added)
#     tracker = SimpleUsageTracker(db)
#     await tracker.track_usage(
#         account_id=account_id,
#         credits=-amount,  # Negative for credits added
#         event_type="credit_purchase",
#         metadata={
#             "amount": amount,
#             "payment_method": "demo",  # In production: actual payment method
#             "transaction_id": str(uuid4())  # In production: payment processor ID
#         }
#     )
#
#     db.add(account)
#     await db.commit()
#
#     return {
#         "success": True,
#         "credits_purchased": amount,
#         "new_balance": account.credits_balance,
#         "transaction_id": str(uuid4())
#     }


# @app.get("/api/billing/usage", tags=["Billing"])
# async def get_usage_summary(
#     db: AsyncSession = Depends(get_session),
#     account_id: UUID = Depends(get_current_account)
# ) -> Dict[str, Any]:
#     """Get current month's usage summary and credit balance."""
#     tracker = SimpleUsageTracker(db)
#     account = await db.get(Account, account_id)
#
#     if not account:
#         raise HTTPException(404, "Account not found")
#
#     # Get current month usage from the continuous aggregate
#     current_usage = await tracker.get_current_usage(account_id)
#
#     # Calculate days remaining in billing period
#     now = datetime.utcnow()
#     days_in_month = (datetime(now.year, now.month % 12 + 1, 1) - timedelta(days=1)).day
#     days_remaining = days_in_month - now.day
#
#     return {
#         "current_balance": account.credits_balance,
#         "month_to_date_usage": current_usage,
#         "billing_period": now.strftime("%Y-%m"),
#         "days_remaining": days_remaining,
#         "average_daily_usage": current_usage / now.day if now.day > 0 else 0,
#         "projected_monthly_usage": (current_usage / now.day * days_in_month) if now.day > 0 else 0
#     }


# @app.get("/api/billing/history", tags=["Billing"])
# async def get_billing_history(
#     limit: int = Query(12, description="Number of months to retrieve"),
#     db: AsyncSession = Depends(get_session),
#     account_id: UUID = Depends(get_current_account)
# ) -> Dict[str, Any]:
#     """Get historical billing data by month."""
#     from sqlalchemy import text
#
#     # Query the continuous aggregate for historical data
#     stmt = text("""
#         SELECT
#             billing_month,
#             total_credits,
#             total_events,
#             unique_users,
#             popular_event_types
#         FROM monthly_billing_summary
#         WHERE account_id = :account_id
#         ORDER BY billing_month DESC
#         LIMIT :limit
#     """)
#
#     result = await db.execute(
#         stmt,
#         {"account_id": str(account_id), "limit": limit}
#     )
#
#     history = []
#     for row in result:
#         history.append({
#             "month": row.billing_month.strftime("%Y-%m"),
#             "total_credits": row.total_credits,
#             "total_events": row.total_events,
#             "unique_users": row.unique_users,
#             "event_breakdown": row.popular_event_types
#         })
#
#     return {
#         "account_id": str(account_id),
#         "history": history,
#         "months_returned": len(history)
#     }


# @app.get("/api/billing/events", tags=["Billing"])
# async def get_usage_events(
#     start_date: Optional[datetime] = Query(None, description="Start date filter"),
#     end_date: Optional[datetime] = Query(None, description="End date filter"),
#     event_type: Optional[str] = Query(None, description="Filter by event type"),
#     limit: int = Query(100, le=1000, description="Maximum events to return"),
#     offset: int = Query(0, description="Pagination offset"),
#     db: AsyncSession = Depends(get_session),
#     account_id: UUID = Depends(get_current_account)
# ) -> Dict[str, Any]:
#     """Get detailed usage events with filtering."""
#     from sqlalchemy import and_
#
#     # Build query conditions
#     conditions = [UsageEvent.account_id == account_id]
#
#     if start_date:
#         conditions.append(UsageEvent.timestamp >= start_date)
#     if end_date:
#         conditions.append(UsageEvent.timestamp <= end_date)
#     if event_type:
#         conditions.append(UsageEvent.event_type == event_type)
#
#     # Count total matching events
#     count_stmt = select(func.count()).select_from(UsageEvent).where(and_(*conditions))
#     total_count = await db.scalar(count_stmt)
#
#     # Get paginated events
#     stmt = (
#         select(UsageEvent)
#         .where(and_(*conditions))
#         .order_by(UsageEvent.timestamp.desc())
#         .offset(offset)
#         .limit(limit)
#     )
#
#     result = await db.execute(stmt)
#     events = result.scalars().all()
#
#     return {
#         "total_count": total_count,
#         "returned_count": len(events),
#         "offset": offset,
#         "limit": limit,
#         "events": [
#             {
#                 "id": str(event.id),
#                 "timestamp": event.timestamp.isoformat(),
#                 "credits": event.credits_consumed,
#                 "event_type": event.event_type,
#                 "user_id": str(event.user_id) if event.user_id else None,
#                 "metadata": event.usage_metadata
#             }
#             for event in events
#         ]
#     }


# @app.get("/api/billing/invoices/{year}/{month}", tags=["Billing"])
# async def download_invoice(
#     year: int,
#     month: int = Field(..., ge=1, le=12),
#     db: AsyncSession = Depends(get_session),
#     account_id: UUID = Depends(get_current_account)
# ) -> Response:
#     """
#     Download invoice for a specific month as PDF.
#     Year and month should be in the past or current month.
#     """
#     # Validate date is not in future
#     now = datetime.utcnow()
#     if year > now.year or (year == now.year and month > now.month):
#         raise HTTPException(400, "Cannot generate invoice for future dates")
#
#     generator = InvoiceGenerator(db)
#
#     try:
#         pdf_buffer = await generator.generate_monthly_invoice(account_id, year, month)
#
#         return Response(
#             content=pdf_buffer.getvalue(),
#             media_type="application/pdf",
#             headers={
#                 "Content-Disposition": f"attachment; filename=invoice_{year}_{month:02d}.pdf",
#                 "Content-Type": "application/pdf"
#             }
#         )
#     except ValueError as e:
#         raise HTTPException(404, str(e))
#     except Exception as e:
#         log.error(f"Failed to generate invoice: {e}", exc_info=True)
#         raise HTTPException(500, "Failed to generate invoice")


@app.get(
    "/sheets/salesforce/search", response_class=HTMLResponse, include_in_schema=False
)
async def sheets_salesforce_search(
    request: Request,
    token: Optional[str] = Query(None),
    source: Optional[str] = Query(None),
    db: AsyncSession = Depends(get_session),
):
    """Lightweight embedded UI surface for Sheets integrations."""

    claims: Dict[str, Any] = {}
    tenant_id: Optional[str] = None
    secret = settings.ENCRYPTION_KEY or settings.JWT_SECRET_KEY
    if not secret:
        env = (getattr(settings, "ENV", "") or "").strip().lower()
        if env in ("dev", "development", "local"):
            secret = "dev-secret"
        else:
            raise HTTPException(
                500,
                "Embed token signing secret not configured (ENCRYPTION_KEY/JWT_SECRET_KEY)",
            )

    if token:
        try:
            claims = jwt.decode(
                token,
                secret,
                algorithms=["HS256"],
                audience="gsheets",
                options={"require": ["exp", "aud"]},
            )
            tenant_id = claims.get("account_id")
        except jwt.ExpiredSignatureError:
            raise HTTPException(401, "Embed token expired") from None
        except jwt.InvalidTokenError:
            raise HTTPException(401, "Invalid embed token") from None
    elif settings.ALLOW_GSHEETS_EMBED_WITHOUT_TOKEN:
        tenant_id = settings.DEV_ACCOUNT_ID
    else:
        raise HTTPException(status_code=401, detail="Embed token required")

    connection_status: Dict[str, Any] = {}
    if tenant_id:
        try:
            gateway = await SalesforceGateway.for_tenant(str(tenant_id), db)
            connection_status = await gateway.get_status()
        except Exception as exc:  # pragma: no cover - defensive
            log.warning("Sheets embed status lookup failed: %s", exc)
            connection_status = {"connected": False, "error": str(exc)}

    context = json.dumps(
        {
            "claims": claims,
            "source": source,
            "connection": connection_status,
            "issued_at": datetime.utcnow().isoformat() + "Z",
        }
    )

    html_template = """<!DOCTYPE html>
<html>
<head>
  <meta charset='utf-8' />
  <title>Salesforce Search Bridge</title>
  <style>
    body { font: 14px 'Segoe UI', Arial, sans-serif; margin: 0; padding: 16px; background: #f8fafc; color: #0f172a; }
    .card { background: #fff; border: 1px solid #e2e8f0; border-radius: 10px; padding: 18px; box-shadow: 0 4px 16px rgba(15,23,42,0.08); max-width: 760px; margin: 0 auto; }
    h1 { margin-top: 0; font-size: 20px; display: flex; align-items: center; gap: 8px; }
    h1 span { font-size: 14px; font-weight: 700; padding: 2px 6px; border-radius: 4px; background: #2563eb; color: #fff; }
    .status { margin: 12px 0; padding: 12px; border-radius: 8px; background: #f1f5f9; display: flex; justify-content: space-between; align-items: center; }
    .status.ok { background: #ecfdf5; color: #047857; }
    .status.warn { background: #fef3c7; color: #b45309; }
    .status.fail { background: #fee2e2; color: #b91c1c; }
    button { border: none; background: #2563eb; color: #ffffff; padding: 10px 16px; border-radius: 6px; cursor: pointer; font-size: 13px; margin-right: 8px; }
    button.secondary { background: #e2e8f0; color: #0f172a; }
    button:disabled { opacity: 0.6; cursor: not-allowed; }
    pre { background: #0f172a; color: #e2e8f0; padding: 12px; border-radius: 6px; overflow: auto; max-height: 240px; font-size: 12px; }
    .dataset-preview { margin-top: 18px; border-top: 1px solid #e2e8f0; padding-top: 16px; }
    table { width: 100%; border-collapse: collapse; margin-top: 8px; font-size: 12px; }
    th, td { border: 1px solid #e2e8f0; padding: 6px 8px; text-align: left; }
    th { background: #f8fafc; }
  </style>
</head>
<body>
  <div class='card'>
    <h1><span>SF</span>Salesforce Embedded Search (Bridge)</h1>
    <div id='connectionStatus' class='status'>Checking Salesforce connection...</div>
    <div>
      <button id='sendSample'>Send Sample Data</button>
      <button id='sendChunk' class='secondary'>Send Chunked Sample</button>
    </div>
    <div class='dataset-preview'>
      <strong>Preview:</strong>
      <table>
        <thead><tr><th>Id</th><th>Name</th><th>Email</th><th>Company</th><th>Status</th></tr></thead>
        <tbody id='previewRows'></tbody>
      </table>
    </div>
    <details style='margin-top:16px;'><summary>Token Claims</summary><pre id='claimDump'></pre></details>
  </div>
  <script>
    const CONTEXT = __CONTEXT__;
    const SAMPLE_ROWS = [
      ["00Q8A00001", "Jane Doe", "jane@example.com", "Acme Inc", "New"],
      ["00Q8A00002", "Marco Li", "marco@example.com", "Globex", "Working"],
      ["00Q8A00003", "Priya Natarajan", "priya@example.com", "Initech", "Qualified"]
    ];
    const HEADERS = ["Id", "Name", "Email", "Company", "Status"];
    const preview = document.getElementById('previewRows');
    SAMPLE_ROWS.forEach(row => {
      const tr = document.createElement('tr');
      row.forEach(value => {
        const td = document.createElement('td');
        td.textContent = value;
        tr.appendChild(td);
      });
      preview.appendChild(tr);
    });

    const claimDump = document.getElementById('claimDump');
    claimDump.textContent = JSON.stringify(CONTEXT.claims || {}, null, 2);

    const connectionStatus = document.getElementById('connectionStatus');
    const conn = CONTEXT.connection || {};
    if (conn.connected) {
      connectionStatus.classList.add('ok');
      connectionStatus.textContent = 'Salesforce connected';
    } else if (conn.error) {
      connectionStatus.classList.add('fail');
      connectionStatus.textContent = 'Connection error: ' + conn.error;
    } else {
      connectionStatus.classList.add('warn');
      connectionStatus.textContent = 'Salesforce status unknown';
    }

    let parentOrigin = '*';
    const sendSampleButton = document.getElementById('sendSample');
    const sendChunkButton = document.getElementById('sendChunk');

    function postMessageToParent(payload) {
      window.parent.postMessage(payload, parentOrigin);
    }

    sendSampleButton.addEventListener('click', () => {
      postMessageToParent({ action: 'import-data', headers: HEADERS, rows: SAMPLE_ROWS });
    });

    sendChunkButton.addEventListener('click', () => {
      SAMPLE_ROWS.forEach((row, index) => {
        const remaining = SAMPLE_ROWS.length - index - 1;
        postMessageToParent({
          action: 'import-chunk',
          headers: HEADERS,
          data: [row],
          chunk: index,
          total: SAMPLE_ROWS.length,
          remaining: remaining,
        });
      });
    });

    window.addEventListener('message', (event) => {
      parentOrigin = event.origin || parentOrigin;
      const data = event.data || {};
      if (data.action === 'gsheets-host-ready') {
        postMessageToParent({ action: 'sheets-embed-ready', accountId: CONTEXT.claims?.account_id || null });
      } else if (data.action === 'request-sample') {
        postMessageToParent({ action: 'import-data', headers: HEADERS, rows: SAMPLE_ROWS });
      }
    });

    // Notify parent that the bridge is ready
    postMessageToParent({ action: 'sheets-embed-ready', accountId: CONTEXT.claims?.account_id || null });
  </script>
</body>
</html>
"""

    html = html_template.replace("__CONTEXT__", context)

    response = HTMLResponse(html)
    response.headers["Cache-Control"] = "no-store"
    return response


@app.get("/healthz", include_in_schema=False)
def health() -> Dict[str, str]:
    return {"status": "ok"}


@app.get("/health/llm", include_in_schema=False)
async def llm_health() -> Dict[str, Any]:
    """Health check for LLM provider with warm-up."""
    try:
        # Get the LLM provider
        from .providers import get_llm_provider

        provider = get_llm_provider()
        if not provider:
            return JSONResponse(
                {"status": "disabled", "message": "LLM provider not configured"},
                status_code=503,
            )

        # Simple JSON generation test
        test_schema = {
            "type": "object",
            "properties": {"ok": {"type": "boolean"}},
            "required": ["ok"],
        }

        result = provider.generate_json(
            '{"ok":true}', schema=test_schema, num_predict=8
        )

        # Extract metrics if available
        metrics = result.get("metrics", {})

        return {
            "status": "ok",
            "model": result.get("model"),
            "provider": result.get("provider"),
            "latency_ms": metrics.get("latency_ms"),
            "fallback_used": metrics.get("fallback_used", False),
        }

    except Exception as e:
        return JSONResponse(
            {"status": "degraded", "error": str(e)[:160]}, status_code=503
        )


@app.get("/api/v1/info", tags=["iPaaS"])
def api_info() -> Dict[str, Any]:
    """Get API information and limits."""
    return {
        "version": settings.API_VERSION,
        "rate_limits": {
            "enabled": RATE_LIMITING_ENABLED,
            "endpoints": {
                "/api/v1/match/simple": settings.RATE_LIMIT_SIMPLE_MATCH,
                "/api/v1/jobs/{job_id:path}/status": settings.RATE_LIMIT_STATUS,
                "/api/v1/jobs/{job_id}/results": settings.RATE_LIMIT_RESULTS,
            },
            "key_based": "Rate limits are per API key or IP address",
        },
        "limits": {
            "max_sync_rows": settings.SYNC_ROW_LIMIT,
            "max_file_size_mb": settings.MAX_UPLOAD_SIZE // (1024 * 1024),
            "max_string_data_mb": settings.MAX_STRING_DATA_MB,
        },
        "defaults": {
            "dedupe_threshold": settings.DEFAULT_DEDUPE_THRESHOLD,
            "match_threshold": settings.DEFAULT_MATCH_THRESHOLD,
        },
        "documentation": settings.DOCS_URL,
        "support": settings.SUPPORT_EMAIL,
    }


# ????????????????????????????????????????????????????????????????????????????????? OpenAPI Export ???????????????????????????????????????????????????????????????????????????
@app.get("/openapi.yaml", include_in_schema=False)
async def export_openapi_yaml():
    """Export OpenAPI spec as YAML for SDK generation"""
    try:
        import yaml
    except ImportError:
        raise HTTPException(
            status_code=500, detail="PyYAML not installed. Run: pip install pyyaml"
        )

    openapi_schema = app.openapi()

    # Clean up the schema for better SDK generation
    # Remove internal endpoints
    if "paths" in openapi_schema:
        paths_to_remove = ["/healthz", "/openapi.yaml", "/openapi.json"]
        for path in paths_to_remove:
            openapi_schema["paths"].pop(path, None)

    # Add additional metadata for SDK generators
    if "info" in openapi_schema:
        openapi_schema["info"]["x-logo"] = {"url": "https://foundrymatch.com/logo.png"}
        openapi_schema["info"]["contact"] = {
            "name": "FoundryMatch Support",
            "email": settings.SUPPORT_EMAIL,
            "url": "https://foundrymatch.com/support",
        }
        openapi_schema["info"]["license"] = {
            "name": "Commercial",
            "url": "https://foundrymatch.com/license",
        }

    # Add server information
    openapi_schema["servers"] = [
        {
            "url": settings.API_URL or "http://localhost:8000",
            "description": "Production API",
        },
        {"url": "http://localhost:8000", "description": "Local Development"},
    ]

    yaml_content = yaml.dump(
        openapi_schema, default_flow_style=False, sort_keys=False, allow_unicode=True
    )

    return Response(
        content=yaml_content,
        media_type="application/x-yaml",
        headers={
            "Content-Disposition": "attachment; filename=foundrymatch-openapi.yaml"
        },
    )


@app.get("/openapi.json", include_in_schema=False)
async def export_openapi_json():
    """Export OpenAPI spec as JSON (already available by default at /openapi.json)"""
    openapi_schema = app.openapi()

    # Clean up the schema
    if "paths" in openapi_schema:
        paths_to_remove = ["/healthz", "/openapi.yaml", "/openapi.json"]
        for path in paths_to_remove:
            openapi_schema["paths"].pop(path, None)

    # Add SDK generation hints
    openapi_schema["x-generator-hints"] = {
        "python": {
            "packageName": "foundrymatch_client",
            "projectName": "FoundryMatch Python SDK",
        },
        "typescript": {
            "npmName": "@foundrymatch/client",
            "npmVersion": settings.API_VERSION,
        },
        "go": {"packageName": "foundrymatch"},
    }

    return openapi_schema


@app.get("/api/v1/sdk-examples", include_in_schema=False)
async def get_sdk_examples():
    """Get example code for using the API with various SDKs"""
    return {
        "python": {
            "install": "pip install foundrymatch-client",
            "example": """
from foundrymatch_client import Client

client = Client(api_key="your-api-key")

# Simple match
result = client.match.simple(
    left_data=[{"name": "John Doe", "email": "john@example.com"}],
    right_data=[{"name": "J. Doe", "email": "john@example.com"}],
    threshold=80
)

# Create and execute job
job = client.jobs.create(
    source_file="data.csv",
    mode="dedupe"
)
config = client.intelligence.auto_configure(
    job_id=job.id,
    optimization_goal="balanced"
)
result = client.jobs.execute(job.id, config)
            """,
        },
        "typescript": {
            "install": "npm install @foundrymatch/client",
            "example": """
import { FoundryMatchClient } from '@foundrymatch/client';

const client = new FoundryMatchClient({ apiKey: 'your-api-key' });

// Simple match
const result = await client.match.simple({
  leftData: [{name: 'John Doe', email: 'john@example.com'}],
  rightData: [{name: 'J. Doe', email: 'john@example.com'}],
  threshold: 80
});

// Stream progress
const eventSource = client.progress.stream(jobId);
eventSource.onmessage = (event) => {
  const progress = JSON.parse(event.data);
  console.log(`${progress.phase}: ${progress.percent}%`);
};
            """,
        },
        "curl": {
            "example": """
# Simple match
curl -X POST https://api.foundrymatch.com/api/v1/match/simple \\
  -H "Authorization: Bearer your-api-key" \\
  -H "Content-Type: application/json" \\
  -d '{
    "left_data": [{"name": "John Doe"}],
    "right_data": [{"name": "J. Doe"}],
    "threshold": 80
  }'

# Stream progress with SSE
curl -N https://api.foundrymatch.com/api/v1/jobs/{job_id}/progress \\
  -H "Authorization: Bearer your-api-key" \\
  -H "Accept: text/event-stream"
            """
        },
    }


# Scheduler: weekly digest (runs in-process)
if not TEST_MODE:
    try:
        from .scheduler import start_digest_scheduler

        start_digest_scheduler()
    except Exception:
        pass


# ---------------------------------------------------------------------------
# Route table sanity checks (detect duplicate (METHOD, PATH) pairs)
# ---------------------------------------------------------------------------
def _assert_unique_routes(app: FastAPI) -> None:
    seen = set()
    dups = []
    for r in app.router.routes:
        path = getattr(r, "path_format", None) or getattr(r, "path", None) or repr(r)
        methods = getattr(r, "methods", None) or {"ANY"}
        for m in methods:
            key = (m.upper(), path.rstrip("/"))  # normalize trailing slash
            if key in seen:
                dups.append(key)
            else:
                seen.add(key)
    if dups:
        raise RuntimeError(f"Duplicate routes detected: {sorted(dups)}")


def _print_routes(app: FastAPI) -> None:
    rows = []
    for r in app.router.routes:
        path = getattr(r, "path_format", None) or getattr(r, "path", None) or repr(r)
        methods = ",".join(sorted((getattr(r, "methods", None) or {"ANY"})))
        name = getattr(r, "name", "")
        rows.append((methods, path, name))
    for m, p, n in sorted(rows, key=lambda x: (x[1], x[0])):
        print(f"{m:10} {p:60} {n}")


@app.on_event("startup")
async def _check_duplicate_routes_on_startup() -> None:
    # Optional quick audit while refactoring
    if os.getenv("PRINT_ROUTES", "0").lower() in ("1", "true", "yes"):  # dev-only
        try:
            _print_routes(app)
        except Exception:
            pass
    _assert_unique_routes(app)


# Dev CORS for public v1 (safe default)
try:
    from . import settings as _s

    if _s.ENV != "production":
        app.add_middleware(
            CORSMiddleware,
            allow_origins=["*"],
            allow_credentials=False,
            allow_methods=["GET", "POST", "OPTIONS"],
            allow_headers=[
                "Authorization",
                "Content-Type",
                "Accept",
                "ngrok-skip-browser-warning",
                "Idempotency-Key",
                "X-Requested-With",
                "X-Account-Id",
                "X-API-Key",
                "X-Webhook-Signature",
                "X-Webhook-Signature-256",
            ],
            expose_headers=[
                "X-Foundry-Request-Id",
                "X-RateLimit-Limit",
                "X-RateLimit-Remaining",
                "Retry-After",
            ],
        )
except Exception:
    pass
    from .services.saas_event_tasks import (
        purge_old_saas_events,
        refresh_saas_event_rollup,
    )
