"""
AutoPIL REST API
================
All /v1/* routes are tenant-scoped — the tenant is resolved from the API key.
Superadmin routes at /v1/admin/* require a key with is_superadmin=True.

Endpoints:
  POST   /v1/context/evaluate             — evaluate a context request
  GET    /v1/audit/sessions/{id}          — audit trail for a session
  GET    /v1/audit/events                 — all events for this tenant
  GET    /v1/audit/export                 — export audit events as CSV or JSON
  POST   /v1/audit/lineage               — record an agent action linked to an audit event
  GET    /v1/audit/lineage/{event_id}    — retrieve event + all downstream actions (full lineage)
  GET    /v1/audit/verify                 — verify SHA-256 hash chain integrity for this tenant
  GET    /v1/audit/retention              — active retention policy and last purge anchor for this tenant
  GET    /v1/audit/stats                  — dashboard stats for this tenant
  GET    /v1/policies                     — loaded policies (DB or YAML fallback)
  POST   /v1/policies/reload              — hot-reload policies from YAML into DB
  POST   /v1/policies                     — create a policy
  PUT    /v1/policies/{name}              — update a policy (saves version)
  DELETE /v1/policies/{name}              — delete a policy
  GET    /v1/policies/{name}/history      — version history for a policy
  POST   /v1/alerts/rules                 — create alert rule
  GET    /v1/alerts/rules                 — list alert rules
  PUT    /v1/alerts/rules/{rule_id}       — update alert rule
  DELETE /v1/alerts/rules/{rule_id}       — delete alert rule
  GET    /v1/alerts/fired                 — recent alert deliveries
  POST   /v1/keys                         — create API key for this tenant
  GET    /v1/keys                         — list keys for this tenant
  DELETE /v1/keys/{key_id}               — revoke a key
  POST   /v1/keys/{key_id}/rotate        — rotate a key (new key, same scope/expiry, old revoked)
  POST   /v1/admin/tenants               — create tenant (superadmin)
  GET    /v1/admin/tenants               — list all tenants (superadmin)
  DELETE /v1/admin/tenants/{id}          — deactivate tenant (superadmin)
  POST   /v1/admin/trial/provision      — provision a 15-day trial (superadmin)
  GET    /v1/admin/trials               — list all trials (superadmin)
  POST   /v1/trial/hubspot              — HubSpot form webhook → auto-provision trial
  GET    /v1/sessions                   — list active sessions for this tenant
  GET    /v1/sessions/{id}             — session status (active/expired/revoked)
  DELETE /v1/sessions/{id}            — revoke a session immediately
  GET    /health                         — liveness check (no auth)
"""

import csv
import io
import uuid
from datetime import datetime
from pathlib import Path
from typing import Optional

import yaml
from fastapi import Depends, FastAPI, HTTPException, Query, Request, Security
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, StreamingResponse
from fastapi.security import APIKeyHeader
from fastapi.staticfiles import StaticFiles

import hmac
import json
import os
import secrets

from autopil.alerting import AlertEngine
from autopil.db import create_catalog_store, create_stores, create_trial_store
from autopil.models import AgentAction, AuditEvent, CatalogConnection, ContextRequest, Decision
from autopil.policy_engine import PolicyEngine
from autopil import telemetry

from .schemas import (
    AgentActionResponse,
    AlertDeliveryResponse,
    AlertRuleResponse,
    AuditEventResponse,
    AuditRetentionResponse,
    AuditTrailResponse,
    CatalogClassificationResponse,
    CatalogConnectionResponse,
    CreateAlertRuleRequest,
    CreateKeyRequest,
    CreateKeyResponse,
    CreatePolicyRequest,
    CreateTenantRequest,
    CreateTenantResponse,
    EvaluateRequest,
    EvaluateResponse,
    HealthResponse,
    KeySummary,
    LineageResponse,
    RotateKeyResponse,
    PolicyDetailResponse,
    PolicyVersionResponse,
    RecordActionRequest,
    RegisterCatalogConnectionRequest,
    SessionResponse,
    SyncResultResponse,
    TenantResponse,
    TrialProvisionRequest,
    TrialProvisionResponse,
    TrialSummary,
    CatalogStatsResponse,
    ChainVerifyResponse,
    UpdateAlertRuleRequest,
    UpdateCatalogConnectionRequest,
    UpdateCatalogCredentialsRequest,
    UpdatePolicyRequest,
)

STATIC_DIR = Path(__file__).parent / "static"

AUDIT_RETENTION_DAYS = int(os.environ.get("AUDIT_RETENTION_DAYS", "365"))

_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)


def _load_yaml_policies(path: str) -> list:
    """Load policies from a single YAML file or every *.yaml in a directory tree."""
    p = Path(path)
    files = sorted(p.glob("**/*.yaml")) if p.is_dir() else [p]
    policies = []
    for f in files:
        with open(f) as fh:
            data = yaml.safe_load(fh)
        parts = f.relative_to(p).parts if p.is_dir() else ()
        industry = parts[0] if len(parts) >= 2 else None
        category = Path(parts[1]).stem if len(parts) >= 2 else (Path(parts[0]).stem if len(parts) == 1 else None)
        for pol in data.get("policies", []):
            pol.setdefault("industry", industry)
            pol.setdefault("category", category)
            policies.append(pol)
    return policies


def create_app(
    policy_path: str,
    audit_db: str = "autopil_audit.db",
    database_url: str | None = None,
    session_ttl_minutes: int | None = None,
) -> FastAPI:
    app = FastAPI(
        title="AutoPIL",
        description="Platform Intelligence Layer — context governance for autonomous agents",
        version="0.1.1",
        docs_url="/docs",
        redoc_url="/redoc",
    )

    app.add_middleware(
        CORSMiddleware,
        allow_origins=["*"],
        allow_methods=["*"],
        allow_headers=["*"],
    )

    app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
    app.mount("/assets", StaticFiles(directory=str(STATIC_DIR / "assets")), name="assets")

    telemetry.setup()

    # ── shared state ─────────────────────────────────────────────────────────
    audit_log, key_store, tenant_store, policy_store, alert_store, lineage_store, session_store = create_stores(
        database_url=database_url, db_path=audit_db
    )
    catalog_store = create_catalog_store(database_url=database_url, db_path=audit_db)
    trial_store = create_trial_store(database_url=database_url, db_path=audit_db)
    alert_engine = AlertEngine(alert_store, audit_log)
    state = {
        "policy_path":          policy_path,
        "audit_db":             database_url or audit_db,
        "engine":               PolicyEngine(policy_path),
        "log":                  audit_log,
        "keys":                 key_store,
        "tenants":              tenant_store,
        "policy_store":         policy_store,
        "alert_store":          alert_store,
        "alert_engine":         alert_engine,
        "lineage_store":        lineage_store,
        "session_store":        session_store,
        "session_ttl_minutes":  session_ttl_minutes,
        "catalog_store":        catalog_store,
        "catalog_scheduler":    None,  # initialised in startup event
        "trial_store":          trial_store,
        "last_purge_count":     0,
    }

    # ── catalog scheduler startup ─────────────────────────────────────────────

    @app.on_event("startup")
    def _start_audit_retention_worker():
        import logging
        import threading
        import time

        logger = logging.getLogger(__name__)

        def _retention_loop():
            while True:
                time.sleep(24 * 3600)
                if AUDIT_RETENTION_DAYS > 0:
                    try:
                        n = state["log"].purge_old_events(AUDIT_RETENTION_DAYS)
                        state["last_purge_count"] = n
                        if n:
                            logger.info(
                                "Audit retention: purged %d events older than %d days", n, AUDIT_RETENTION_DAYS
                            )
                    except Exception:
                        logger.exception("Audit retention purge failed")

        if AUDIT_RETENTION_DAYS > 0:
            t = threading.Thread(target=_retention_loop, daemon=True, name="audit-retention")
            t.start()

    @app.on_event("startup")
    def _start_trial_background_workers():
        try:
            from autopil.trial import start_background_workers
            start_background_workers(state["trial_store"], state["tenants"])
        except Exception:
            import logging
            logging.getLogger(__name__).warning(
                "Trial background workers failed to start", exc_info=True
            )

    @app.on_event("startup")
    def _start_catalog_scheduler():
        try:
            from autopil.catalog.scheduler import BackgroundSyncScheduler
            enc_key = os.environ.get("AUTOPIL_CATALOG_ENCRYPTION_KEY")
            scheduler = BackgroundSyncScheduler(catalog_store, encryption_key=enc_key)
            # Start timers for all tenants with registered connections
            for tenant in tenant_store.list_tenants():
                if tenant.get("is_active"):
                    scheduler.start_all(tenant["tenant_id"])
            state["catalog_scheduler"] = scheduler
        except Exception:
            import logging
            logging.getLogger(__name__).warning(
                "Catalog scheduler failed to start (catalog extras may not be installed)",
                exc_info=True,
            )

    @app.on_event("shutdown")
    def _stop_catalog_scheduler():
        sched = state.get("catalog_scheduler")
        if sched:
            sched.stop_all()

    # ── auth dependencies ─────────────────────────────────────────────────────

    def require_api_key(request: Request, raw: str = Security(_api_key_header)) -> dict:
        if not raw:
            raise HTTPException(status_code=401, detail="Missing API key. Set X-API-Key header.")
        info = state["keys"].validate(raw)
        if not info:
            raise HTTPException(status_code=403, detail="Invalid or revoked API key.")
        scope = info.get("scope", "admin")
        method = request.method
        path = request.url.path
        is_evaluate = method == "POST" and path == "/v1/context/evaluate"
        if scope == "evaluate" and not is_evaluate:
            raise HTTPException(
                status_code=403,
                detail="This key is restricted to POST /v1/context/evaluate only.",
            )
        if scope == "read" and method not in {"GET", "HEAD", "OPTIONS"} and not is_evaluate:
            raise HTTPException(
                status_code=403,
                detail="This key is read-only. Write operations require an admin-scoped key.",
            )
        return info  # {"key_id", "tenant_id", "is_superadmin", "scope"}

    def require_superadmin(key_info: dict = Depends(require_api_key)) -> dict:
        if not key_info.get("is_superadmin"):
            raise HTTPException(status_code=403, detail="Superadmin key required.")
        return key_info

    # ── policy engine helper ──────────────────────────────────────────────────

    def _get_engine(tenant_id: str) -> PolicyEngine:
        db_policies = state["policy_store"].list_active(tenant_id)
        if db_policies:
            return PolicyEngine.from_list(db_policies)
        return state["engine"]

    # ── dashboard ─────────────────────────────────────────────────────────────

    @app.get("/", include_in_schema=False)
    def dashboard():
        return FileResponse(str(STATIC_DIR / "index.html"))

    # ── health ────────────────────────────────────────────────────────────────

    @app.get("/health", response_model=HealthResponse, tags=["System"])
    def health():
        return HealthResponse(
            status="ok", version="0.1.1",
            policy_file=state["policy_path"], audit_db=state["audit_db"],
        )

    # ── HubSpot trial webhook (public, secret-protected) ─────────────────────

    @app.post("/v1/trial/hubspot", tags=["Trials"])
    async def hubspot_trial_webhook(request: Request):
        """
        Receive HubSpot form submission, provision a trial, and send welcome email.

        Authentication: pass TRIAL_WEBHOOK_SECRET as ?secret=<value> query param.
        HubSpot payload format:
          {"data": [{"name": "email", "value": "..."}, ...], "submittedAt": ...}
        """
        from autopil.trial import seed_demo_data, send_welcome_email, TRIAL_BASE_URL
        from datetime import timezone, timedelta

        # Validate shared secret
        webhook_secret = os.environ.get("TRIAL_WEBHOOK_SECRET", "")
        if webhook_secret:
            provided = request.query_params.get("secret", "")
            if not secrets.compare_digest(provided, webhook_secret):
                raise HTTPException(status_code=401, detail="Invalid webhook secret.")

        body = await request.json()

        # Parse HubSpot's data array: [{"name": "email", "value": "..."}, ...]
        fields = {}
        for item in body.get("data", []):
            fields[item.get("name", "")] = item.get("value", "")

        # Also support flat JSON (for testing / other sources)
        if not fields:
            fields = body

        email    = fields.get("email", "").strip()
        firstname = fields.get("firstname", fields.get("first_name", "")).strip()
        lastname  = fields.get("lastname",  fields.get("last_name",  "")).strip()
        name      = f"{firstname} {lastname}".strip() or fields.get("name", email)
        company   = fields.get("company",  fields.get("company_name", "Unknown")).strip()
        industry  = fields.get("industry", "financial_services").strip()

        if not email:
            raise HTTPException(status_code=422, detail="email field is required.")

        trial_id    = str(uuid.uuid4())
        tenant_name = f"trial-{company.lower().replace(' ', '-')[:24]}-{trial_id[:8]}"
        try:
            tenant_id = state["tenants"].create_tenant(tenant_name)
        except Exception as exc:
            if "UNIQUE" in str(exc).upper():
                raise HTTPException(status_code=409, detail="A trial already exists for this company.")
            raise
        _, api_key  = state["keys"].create_key("trial-admin", tenant_id=tenant_id)
        trial_end   = (datetime.now(timezone.utc) + timedelta(days=15)).isoformat()
        state["trial_store"].create_trial(
            trial_id, tenant_id, name, email, company, industry, trial_end
        )
        seed_demo_data(tenant_id, industry, state["log"], state["policy_store"])
        trial_end_dt = datetime.fromisoformat(trial_end)
        send_welcome_email(name, email, company, api_key, trial_end_dt)

        return {"status": "provisioned", "trial_id": trial_id, "tenant_id": tenant_id}

    # ── tenant management (superadmin only) ───────────────────────────────────

    @app.post("/v1/admin/tenants", response_model=CreateTenantResponse, tags=["Admin"])
    def create_tenant(req: CreateTenantRequest, _: dict = Depends(require_superadmin)):
        """Create a new tenant and return its initial admin API key."""
        try:
            tenant_id = state["tenants"].create_tenant(req.name)
        except Exception as exc:
            if "UNIQUE" in str(exc).upper() or "unique" in str(exc).lower():
                raise HTTPException(status_code=409, detail=f"Tenant name '{req.name}' already exists.")
            raise
        _, admin_key = state["keys"].create_key("admin", tenant_id=tenant_id)
        tenant = state["tenants"].get_tenant(tenant_id)
        return CreateTenantResponse(
            tenant_id=tenant_id,
            name=req.name,
            created_at=tenant["created_at"],
            admin_key=admin_key,
        )

    @app.get("/v1/admin/tenants", response_model=list[TenantResponse], tags=["Admin"])
    def list_tenants(_: dict = Depends(require_superadmin)):
        """List all tenants."""
        return [TenantResponse(**t) for t in state["tenants"].list_tenants()]

    @app.post("/v1/admin/trial/provision", response_model=TrialProvisionResponse, tags=["Admin"])
    def provision_trial(req: TrialProvisionRequest, _: dict = Depends(require_superadmin)):
        """Provision a 15-day trial tenant, seed demo data, and send a welcome email."""
        from autopil.trial import seed_demo_data, send_welcome_email, TRIAL_BASE_URL
        from datetime import timezone, timedelta
        trial_id = str(uuid.uuid4())
        tenant_name = f"trial-{req.company.lower().replace(' ', '-')}-{trial_id[:8]}"
        try:
            tenant_id = state["tenants"].create_tenant(tenant_name)
        except Exception as exc:
            if "UNIQUE" in str(exc).upper():
                raise HTTPException(status_code=409, detail="Trial already exists for this company.")
            raise
        _, api_key = state["keys"].create_key("trial-admin", tenant_id=tenant_id)
        trial_end = (datetime.now(timezone.utc) + timedelta(days=15)).isoformat()
        state["trial_store"].create_trial(
            trial_id, tenant_id, req.name, req.email, req.company, req.industry, trial_end
        )
        seed_demo_data(tenant_id, req.industry, state["log"], state["policy_store"])
        trial_end_dt = datetime.fromisoformat(trial_end)
        email_sent = send_welcome_email(req.name, req.email, req.company, api_key, trial_end_dt)
        return TrialProvisionResponse(
            trial_id=trial_id, tenant_id=tenant_id, api_key=api_key,
            dashboard_url=TRIAL_BASE_URL, trial_end=trial_end, email_sent=email_sent,
        )

    @app.get("/v1/admin/trials", response_model=list[TrialSummary], tags=["Admin"])
    def list_trials(_: dict = Depends(require_superadmin)):
        """List all trials (active and expired)."""
        return [
            TrialSummary(
                **{
                    **t,
                    "email_day7_sent":  bool(t.get("email_day7_sent")),
                    "email_day13_sent": bool(t.get("email_day13_sent")),
                    "trial_start":      str(t["trial_start"]),
                    "trial_end":        str(t["trial_end"]),
                }
            )
            for t in state["trial_store"].list_all_trials()
        ]

    @app.delete("/v1/admin/tenants/{tenant_id}", tags=["Admin"])
    def deactivate_tenant(tenant_id: str, _: dict = Depends(require_superadmin)):
        """Deactivate a tenant. Their keys and audit data are retained."""
        if not state["tenants"].deactivate_tenant(tenant_id):
            raise HTTPException(status_code=404, detail=f"Tenant '{tenant_id}' not found.")
        return {"status": "deactivated", "tenant_id": tenant_id}

    # ── API key management ────────────────────────────────────────────────────

    @app.post("/v1/keys", response_model=CreateKeyResponse, tags=["Keys"])
    def create_key(req: CreateKeyRequest, key_info: dict = Depends(require_api_key)):
        """Create a new API key scoped to the calling tenant."""
        tenant_id = key_info["tenant_id"]
        key_id, plaintext = state["keys"].create_key(
            req.name,
            tenant_id=tenant_id,
            scope=req.scope.value,
            expires_days=req.expires_days,
        )
        keys = state["keys"].list_keys(tenant_id)
        key_record = next(k for k in keys if k["key_id"] == key_id)
        return CreateKeyResponse(
            key_id=key_id,
            name=req.name,
            key=plaintext,
            scope=key_record.get("scope", "admin"),
            expires_at=key_record.get("expires_at"),
            created_at=key_record["created_at"],
        )

    @app.get("/v1/keys", response_model=list[KeySummary], tags=["Keys"])
    def list_keys(key_info: dict = Depends(require_api_key)):
        """List all keys for the calling tenant."""
        return [KeySummary(**k) for k in state["keys"].list_keys(key_info["tenant_id"])]

    @app.delete("/v1/keys/{key_id}", tags=["Keys"])
    def revoke_key(key_id: str, key_info: dict = Depends(require_api_key)):
        """Revoke a key. Only keys belonging to the calling tenant can be revoked."""
        if not state["keys"].revoke(key_id, key_info["tenant_id"]):
            raise HTTPException(status_code=404, detail=f"Key '{key_id}' not found or already revoked.")
        return {"status": "revoked", "key_id": key_id}

    @app.post("/v1/keys/{key_id}/rotate", response_model=RotateKeyResponse, tags=["Keys"])
    def rotate_key(key_id: str, key_info: dict = Depends(require_api_key)):
        """
        Rotate a key: create a replacement with the same name, scope, and expiry duration,
        then immediately revoke the old key. The new plaintext key is returned once.
        """
        result = state["keys"].rotate_key(key_id, key_info["tenant_id"])
        if result is None:
            raise HTTPException(status_code=404, detail=f"Key '{key_id}' not found.")
        new_key_id, plaintext = result
        keys = state["keys"].list_keys(key_info["tenant_id"])
        key_record = next((k for k in keys if k["key_id"] == new_key_id), {})
        return RotateKeyResponse(
            key_id=new_key_id,
            key=plaintext,
            scope=key_record.get("scope", "admin"),
            expires_at=key_record.get("expires_at"),
        )

    # ── context evaluation ────────────────────────────────────────────────────

    @app.post("/v1/context/evaluate", response_model=EvaluateResponse, tags=["Context"])
    def evaluate(req: EvaluateRequest, key_info: dict = Depends(require_api_key)):
        tenant_id = key_info["tenant_id"]

        # ── session lifecycle check ───────────────────────────────────────────
        sess_store = state["session_store"]
        existing = sess_store.get_session(req.session_id, tenant_id)
        if existing is None:
            # First use of this session — register it
            sess_store.create_session(
                session_id=req.session_id,
                agent_role=req.agent_role,
                user_id=req.user_id,
                tenant_id=tenant_id,
                ttl_minutes=state["session_ttl_minutes"],
            )
            session_decision = None
        else:
            valid, validity_reason = sess_store.is_valid(req.session_id, tenant_id)
            if not valid:
                session_decision = (
                    Decision.DENY,
                    validity_reason,            # "session_expired" or "session_revoked"
                    f"Session '{req.session_id[:8]}…' is {validity_reason.replace('_', ' ')}",
                )
            else:
                sess_store.touch_session(req.session_id, tenant_id)
                session_decision = None

        request = ContextRequest(
            query=req.query, agent_role=req.agent_role, user_id=req.user_id,
            source_id=req.source_id, sensitivity_level=req.sensitivity_level,
            session_id=req.session_id, task_type=req.task_type,
        )
        with telemetry.evaluate_span(
            agent_role=req.agent_role,
            source_id=req.source_id,
            tenant_id=tenant_id,
            session_id=req.session_id,
            sensitivity_level=req.sensitivity_level.value,
        ) as otel_span:
            if session_decision:
                decision, policy_name, reason = session_decision
            else:
                decision, policy_name, reason = _get_engine(tenant_id).evaluate(request)
            otel_span.set_result(decision=decision.value, policy_name=policy_name, reason=reason)
        event = AuditEvent(
            event_id=str(uuid.uuid4()), session_id=req.session_id,
            agent_role=req.agent_role, user_id=req.user_id, source_id=req.source_id,
            query=req.query, decision=decision, policy_name=policy_name,
            reason=reason, timestamp=datetime.utcnow(),
            sensitivity_level=req.sensitivity_level.value,
            source_type="rest",
        )
        state["log"].record(event, tenant_id=tenant_id)
        state["alert_engine"].check(event, tenant_id)
        return EvaluateResponse(
            decision=decision, policy_name=policy_name, reason=reason,
            event_id=event.event_id, session_id=req.session_id, timestamp=event.timestamp,
        )

    # ── session management ────────────────────────────────────────────────────

    def _session_status(sess: dict) -> str:
        if sess["is_revoked"]:
            return "revoked"
        if sess["expires_at"] is not None:
            from datetime import timezone
            now = datetime.now(timezone.utc)
            exp = datetime.fromisoformat(sess["expires_at"])
            if exp.tzinfo is None:
                exp = exp.replace(tzinfo=timezone.utc)
            if now > exp:
                return "expired"
        return "active"

    def _to_session_response(sess: dict) -> SessionResponse:
        return SessionResponse(
            session_id=sess["session_id"],
            agent_role=sess["agent_role"],
            user_id=sess["user_id"],
            created_at=sess["created_at"],
            last_active=sess["last_active"],
            expires_at=sess["expires_at"],
            ttl_minutes=sess["ttl_minutes"],
            is_revoked=sess["is_revoked"],
            status=_session_status(sess),
        )

    @app.get("/v1/sessions", response_model=list[SessionResponse], tags=["Sessions"])
    def list_sessions(
        include_expired: bool = Query(default=False, description="Include expired and revoked sessions"),
        key_info: dict = Depends(require_api_key),
    ):
        """List sessions for this tenant. By default only active (non-expired, non-revoked) sessions."""
        tenant_id = key_info["tenant_id"]
        sessions = state["session_store"].list_sessions(tenant_id, include_expired=include_expired)
        return [_to_session_response(s) for s in sessions]

    @app.get("/v1/sessions/{session_id}", response_model=SessionResponse, tags=["Sessions"])
    def get_session(session_id: str, key_info: dict = Depends(require_api_key)):
        """Return current status of a session."""
        tenant_id = key_info["tenant_id"]
        sess = state["session_store"].get_session(session_id, tenant_id)
        if sess is None:
            raise HTTPException(status_code=404, detail=f"Session '{session_id}' not found.")
        return _to_session_response(sess)

    @app.delete("/v1/sessions/{session_id}", tags=["Sessions"])
    def revoke_session(session_id: str, key_info: dict = Depends(require_api_key)):
        """Immediately revoke a session. Any subsequent evaluation on this session will be denied."""
        tenant_id = key_info["tenant_id"]
        if not state["session_store"].revoke_session(session_id, tenant_id):
            raise HTTPException(status_code=404, detail=f"Session '{session_id}' not found.")
        return {"status": "revoked", "session_id": session_id}

    # ── audit trail ───────────────────────────────────────────────────────────

    @app.get("/v1/audit/sessions/{session_id}", response_model=AuditTrailResponse, tags=["Audit"])
    def get_session_audit(session_id: str, key_info: dict = Depends(require_api_key)):
        tenant_id = key_info["tenant_id"]
        events = state["log"].get_session_events(session_id, tenant_id)
        if not events:
            raise HTTPException(status_code=404, detail=f"No events found for session '{session_id}'")
        return AuditTrailResponse(
            session_id=session_id, total_events=len(events),
            allowed=sum(1 for e in events if e.decision == Decision.ALLOW),
            denied=sum(1 for e in events if e.decision == Decision.DENY),
            events=[_to_response(e) for e in events],
        )

    @app.get("/v1/audit/stats", tags=["Audit"])
    def get_stats(key_info: dict = Depends(require_api_key)):
        tenant_id = key_info["tenant_id"]
        events   = state["log"].get_all_events(tenant_id)
        sessions = list({e.session_id for e in events})
        allowed  = [e for e in events if e.decision == Decision.ALLOW]
        denied   = [e for e in events if e.decision == Decision.DENY]
        roles, sources, by_source_type = {}, {}, {}
        for e in events:
            roles[e.agent_role]  = roles.get(e.agent_role, 0) + 1
            sources[e.source_id] = sources.get(e.source_id, 0) + 1
            st = getattr(e, "source_type", "rest") or "rest"
            if st not in by_source_type:
                by_source_type[st] = {"total": 0, "allowed": 0, "denied": 0}
            by_source_type[st]["total"] += 1
            if e.decision == Decision.ALLOW:
                by_source_type[st]["allowed"] += 1
            else:
                by_source_type[st]["denied"] += 1
        return {
            "total_events":    len(events),
            "allowed":         len(allowed),
            "denied":          len(denied),
            "active_sessions": len(sessions),
            "top_roles":       sorted(roles.items(),   key=lambda x: -x[1])[:5],
            "top_sources":     sorted(sources.items(), key=lambda x: -x[1])[:5],
            "recent_events":   [_to_response(e) for e in reversed(events[-10:])],
            "by_source_type":  by_source_type,
        }

    @app.get("/v1/audit/events", response_model=list[AuditEventResponse], tags=["Audit"])
    def get_all_events(
        decision:   Optional[Decision] = Query(default=None),
        agent_role: Optional[str]      = Query(default=None),
        limit:      int                = Query(default=100, le=1000),
        key_info:   dict               = Depends(require_api_key),
    ):
        tenant_id = key_info["tenant_id"]
        events = state["log"].get_all_events(tenant_id)
        if decision:
            events = [e for e in events if e.decision == decision]
        if agent_role:
            events = [e for e in events if e.agent_role == agent_role]
        return [_to_response(e) for e in events[:limit]]

    _EXPORT_FIELDS = [
        "event_id", "timestamp", "session_id", "agent_role", "user_id",
        "source_id", "query", "decision", "policy_name", "reason", "context_hash",
    ]

    @app.get("/v1/audit/export", tags=["Audit"])
    def export_audit_events(
        format:     str                = Query(default="csv", pattern="^(csv|json)$"),
        decision:   Optional[Decision] = Query(default=None),
        agent_role: Optional[str]      = Query(default=None),
        source_id:  Optional[str]      = Query(default=None),
        from_time:  Optional[datetime] = Query(default=None, alias="from"),
        to_time:    Optional[datetime] = Query(default=None, alias="to"),
        key_info:   dict               = Depends(require_api_key),
    ):
        tenant_id = key_info["tenant_id"]
        events = state["log"].get_all_events(tenant_id)

        if decision:
            events = [e for e in events if e.decision == decision]
        if agent_role:
            events = [e for e in events if e.agent_role == agent_role]
        if source_id:
            events = [e for e in events if e.source_id == source_id]
        if from_time:
            events = [e for e in events if e.timestamp >= from_time]
        if to_time:
            events = [e for e in events if e.timestamp <= to_time]

        timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")

        if format == "json":
            import json
            payload = json.dumps(
                [
                    {
                        "event_id":     e.event_id,
                        "timestamp":    e.timestamp.isoformat(),
                        "session_id":   e.session_id,
                        "agent_role":   e.agent_role,
                        "user_id":      e.user_id,
                        "source_id":    e.source_id,
                        "query":        e.query,
                        "decision":     e.decision.value,
                        "policy_name":  e.policy_name,
                        "reason":       e.reason,
                        "context_hash": e.context_hash,
                    }
                    for e in events
                ],
                indent=2,
            )
            return StreamingResponse(
                io.BytesIO(payload.encode()),
                media_type="application/json",
                headers={"Content-Disposition": f'attachment; filename="autopil_audit_{timestamp}.json"'},
            )

        # CSV (default)
        buf = io.StringIO()
        writer = csv.DictWriter(buf, fieldnames=_EXPORT_FIELDS)
        writer.writeheader()
        for e in events:
            writer.writerow({
                "event_id":     e.event_id,
                "timestamp":    e.timestamp.isoformat(),
                "session_id":   e.session_id,
                "agent_role":   e.agent_role,
                "user_id":      e.user_id,
                "source_id":    e.source_id,
                "query":        e.query,
                "decision":     e.decision.value,
                "policy_name":  e.policy_name,
                "reason":       e.reason,
                "context_hash": e.context_hash or "",
            })
        return StreamingResponse(
            io.BytesIO(buf.getvalue().encode()),
            media_type="text/csv",
            headers={"Content-Disposition": f'attachment; filename="autopil_audit_{timestamp}.csv"'},
        )

    # ── action lineage ────────────────────────────────────────────────────────

    @app.post("/v1/audit/lineage", response_model=AgentActionResponse, tags=["Audit"])
    def record_action(req: RecordActionRequest, key_info: dict = Depends(require_api_key)):
        """Record an agent action driven by a specific audit event."""
        tenant_id = key_info["tenant_id"]
        # Verify the event exists and belongs to this tenant
        events = state["log"].get_all_events(tenant_id)
        event = next((e for e in events if e.event_id == req.event_id), None)
        if not event:
            raise HTTPException(status_code=404, detail=f"Audit event '{req.event_id}' not found.")
        action = AgentAction(
            action_id=str(uuid.uuid4()),
            event_id=req.event_id,
            session_id=req.session_id,
            agent_role=event.agent_role,
            action_type=req.action_type,
            action_detail=req.action_detail,
            outcome=req.outcome,
            timestamp=datetime.utcnow(),
        )
        state["lineage_store"].record_action(action, tenant_id)
        return AgentActionResponse(
            action_id=action.action_id, event_id=action.event_id,
            session_id=action.session_id, agent_role=action.agent_role,
            action_type=action.action_type, action_detail=action.action_detail,
            outcome=action.outcome, timestamp=action.timestamp,
        )

    @app.get("/v1/audit/verify", response_model=ChainVerifyResponse, tags=["Audit"])
    def verify_audit_chain(key_info: dict = Depends(require_api_key)):
        """Verify the SHA-256 tamper-evidence hash chain for this tenant's audit log."""
        tenant_id = key_info["tenant_id"]
        result = state["log"].verify_chain(tenant_id)
        return ChainVerifyResponse(**result)

    @app.get("/v1/audit/retention", response_model=AuditRetentionResponse, tags=["Audit"])
    def get_audit_retention(key_info: dict = Depends(require_api_key)):
        """Return the active log retention policy and last purge anchor for this tenant."""
        tenant_id = key_info["tenant_id"]
        anchor = state["log"]._get_chain_anchor(tenant_id)
        days = AUDIT_RETENTION_DAYS
        policy = (
            f"Events older than {days} days are automatically purged daily."
            if days > 0
            else "Automatic purge is disabled (AUDIT_RETENTION_DAYS=0)."
        )
        return AuditRetentionResponse(retention_days=days, policy=policy, anchor=anchor)

    @app.get("/v1/audit/lineage/{event_id}", response_model=LineageResponse, tags=["Audit"])
    def get_lineage(event_id: str, key_info: dict = Depends(require_api_key)):
        """Return the audit event and all downstream actions linked to it."""
        tenant_id = key_info["tenant_id"]
        events = state["log"].get_all_events(tenant_id)
        event = next((e for e in events if e.event_id == event_id), None)
        if not event:
            raise HTTPException(status_code=404, detail=f"Audit event '{event_id}' not found.")
        actions = state["lineage_store"].get_actions_for_event(event_id, tenant_id)
        return LineageResponse(
            event_id=event_id,
            event=_to_response(event),
            actions=[
                AgentActionResponse(
                    action_id=a.action_id, event_id=a.event_id,
                    session_id=a.session_id, agent_role=a.agent_role,
                    action_type=a.action_type, action_detail=a.action_detail,
                    outcome=a.outcome, timestamp=a.timestamp,
                )
                for a in actions
            ],
        )

    # ── policy management ─────────────────────────────────────────────────────

    @app.get("/v1/policies", response_model=list[PolicyDetailResponse], tags=["Policies"])
    def list_policies(
        key_info: dict = Depends(require_api_key),
        industry: Optional[str] = Query(default=None, description="Filter by industry (e.g. healthcare)"),
        category: Optional[str] = Query(default=None, description="Filter by category (e.g. clinical_operations)"),
    ):
        tenant_id = key_info["tenant_id"]
        db_policies = state["policy_store"].list_active(tenant_id)
        source = db_policies if db_policies else [
            {
                "policy_id": "", "version": 0, "created_at": "", "updated_at": "",
                "changed_by": None, **p,
                "industry": p.get("industry"),
                "category": p.get("category"),
                "allowed_tasks": p.get("allowed_tasks", []),
                "denied_tasks": p.get("denied_tasks", []),
            }
            for p in _load_yaml_policies(state["policy_path"])
        ]
        if industry:
            source = [p for p in source if p.get("industry") == industry]
        if category:
            source = [p for p in source if p.get("category") == category]
        return [PolicyDetailResponse(**p) for p in source]

    @app.post("/v1/policies/reload", tags=["Policies"])
    def reload_policies(key_info: dict = Depends(require_api_key)):
        tenant_id = key_info["tenant_id"]
        state["engine"] = PolicyEngine(state["policy_path"])
        policies = _load_yaml_policies(state["policy_path"])
        # Import into DB for this tenant
        for p in policies:
            state["policy_store"].upsert(
                {
                    "name":            p["name"],
                    "agent_role":      p["agent_role"],
                    "allowed_sources": p.get("allowed_sources", []),
                    "denied_sources":  p.get("denied_sources", []),
                    "allowed_tasks":   p.get("allowed_tasks", []),
                    "denied_tasks":    p.get("denied_tasks", []),
                    "max_sensitivity": p.get("max_sensitivity", "restricted"),
                    "industry":        p.get("industry"),
                    "category":        p.get("category"),
                },
                tenant_id=tenant_id,
                changed_by=key_info["key_id"],
            )
        return {"status": "reloaded", "policies_loaded": len(policies)}

    @app.post("/v1/policies", response_model=PolicyDetailResponse, tags=["Policies"])
    def create_policy(req: CreatePolicyRequest, key_info: dict = Depends(require_api_key)):
        """Create a new policy for this tenant."""
        tenant_id = key_info["tenant_id"]
        if state["policy_store"].get(req.name, tenant_id):
            raise HTTPException(status_code=409, detail=f"Policy '{req.name}' already exists.")
        state["policy_store"].upsert(req.model_dump(), tenant_id=tenant_id, changed_by=key_info["key_id"])
        policy = state["policy_store"].get(req.name, tenant_id)
        return PolicyDetailResponse(**policy)

    @app.put("/v1/policies/{name}", response_model=PolicyDetailResponse, tags=["Policies"])
    def update_policy(name: str, req: UpdatePolicyRequest, key_info: dict = Depends(require_api_key)):
        """Update an existing policy. Saves the previous version to history."""
        tenant_id = key_info["tenant_id"]
        if not state["policy_store"].get(name, tenant_id):
            raise HTTPException(status_code=404, detail=f"Policy '{name}' not found.")
        state["policy_store"].upsert(
            {"name": name, **req.model_dump()},
            tenant_id=tenant_id,
            changed_by=key_info["key_id"],
        )
        policy = state["policy_store"].get(name, tenant_id)
        return PolicyDetailResponse(**policy)

    @app.delete("/v1/policies/{name}", tags=["Policies"])
    def delete_policy(name: str, key_info: dict = Depends(require_api_key)):
        """Soft-delete a policy."""
        tenant_id = key_info["tenant_id"]
        if not state["policy_store"].delete(name, tenant_id):
            raise HTTPException(status_code=404, detail=f"Policy '{name}' not found.")
        return {"status": "deleted", "name": name}

    @app.get("/v1/policies/{name}/history", response_model=list[PolicyVersionResponse], tags=["Policies"])
    def get_policy_history(name: str, key_info: dict = Depends(require_api_key)):
        """Return version history for a policy, newest first."""
        tenant_id = key_info["tenant_id"]
        history = state["policy_store"].get_history(name, tenant_id)
        if not history:
            raise HTTPException(status_code=404, detail=f"No history found for policy '{name}'.")
        return [PolicyVersionResponse(**v) for v in history]

    # ── alert rules ──────────────────────────────────────────────────────────

    @app.post("/v1/alerts/rules", response_model=AlertRuleResponse, tags=["Alerts"])
    def create_alert_rule(req: CreateAlertRuleRequest, key_info: dict = Depends(require_api_key)):
        """Create a new alert rule for this tenant."""
        tenant_id = key_info["tenant_id"]
        rule_id = state["alert_store"].create_rule(req.model_dump(), tenant_id)
        rule = state["alert_store"].get_rule(rule_id, tenant_id)
        return AlertRuleResponse(**rule)

    @app.get("/v1/alerts/rules", response_model=list[AlertRuleResponse], tags=["Alerts"])
    def list_alert_rules(key_info: dict = Depends(require_api_key)):
        """List all alert rules for this tenant."""
        rules = state["alert_store"].list_rules(key_info["tenant_id"])
        return [AlertRuleResponse(**r) for r in rules]

    @app.put("/v1/alerts/rules/{rule_id}", response_model=AlertRuleResponse, tags=["Alerts"])
    def update_alert_rule(rule_id: str, req: UpdateAlertRuleRequest, key_info: dict = Depends(require_api_key)):
        """Update an alert rule."""
        tenant_id = key_info["tenant_id"]
        updates = {k: v for k, v in req.model_dump().items() if v is not None}
        if not state["alert_store"].update_rule(rule_id, updates, tenant_id):
            raise HTTPException(status_code=404, detail=f"Alert rule '{rule_id}' not found.")
        rule = state["alert_store"].get_rule(rule_id, tenant_id)
        return AlertRuleResponse(**rule)

    @app.delete("/v1/alerts/rules/{rule_id}", tags=["Alerts"])
    def delete_alert_rule(rule_id: str, key_info: dict = Depends(require_api_key)):
        """Delete an alert rule."""
        if not state["alert_store"].delete_rule(rule_id, key_info["tenant_id"]):
            raise HTTPException(status_code=404, detail=f"Alert rule '{rule_id}' not found.")
        return {"status": "deleted", "rule_id": rule_id}

    @app.get("/v1/alerts/fired", response_model=list[AlertDeliveryResponse], tags=["Alerts"])
    def list_fired_alerts(
        limit: int = Query(default=50, le=500),
        key_info: dict = Depends(require_api_key),
    ):
        """Return recent alert deliveries for this tenant, newest first."""
        deliveries = state["alert_store"].list_deliveries(key_info["tenant_id"], limit=limit)
        return [AlertDeliveryResponse(**d) for d in deliveries]

    # ── catalog connections ───────────────────────────────────────────────────

    @app.post("/v1/catalog/connections", response_model=CatalogConnectionResponse, tags=["Catalog"])
    def register_catalog_connection(
        body: RegisterCatalogConnectionRequest,
        key_info: dict = Depends(require_api_key),
    ):
        """
        Register a new catalog connection. Credentials are tested before saving
        and stored encrypted at rest. The plaintext credentials are never returned.
        """
        from autopil.catalog.registry import CatalogRegistry
        from autopil.catalog.credentials import encrypt_credentials

        if not CatalogRegistry.is_supported(body.catalog_type):
            supported = CatalogRegistry.supported_types()
            raise HTTPException(
                status_code=422,
                detail=(
                    f"Unsupported catalog_type '{body.catalog_type}'. "
                    f"Supported: {supported or ['(none installed)']}"
                ),
            )

        credentials_enc = encrypt_credentials(body.credentials)
        now = datetime.utcnow()
        conn_obj = CatalogConnection(
            connection_id="cat_" + secrets.token_hex(8),
            tenant_id=key_info["tenant_id"],
            catalog_type=body.catalog_type,
            display_name=body.display_name,
            base_url=body.base_url,
            credentials_enc=credentials_enc,
            sync_interval_minutes=body.sync_interval_minutes,
            is_enabled=True,
            created_at=now,
            updated_at=now,
            webhook_secret=body.webhook_secret,
        )

        # Test before persisting
        try:
            connector = CatalogRegistry.get_connector(conn_obj)
            ok, msg = connector.test_connection()
        except Exception as exc:
            raise HTTPException(status_code=422, detail=f"Connector error: {exc}")
        if not ok:
            raise HTTPException(status_code=422, detail=f"Connection test failed: {msg}")

        state["catalog_store"].create_connection(conn_obj)

        # Start background sync timer for this connection
        sched = state.get("catalog_scheduler")
        if sched:
            sched.start_connection(conn_obj)

        return _conn_to_response(conn_obj)

    @app.get("/v1/catalog/connections", response_model=list[CatalogConnectionResponse], tags=["Catalog"])
    def list_catalog_connections(key_info: dict = Depends(require_api_key)):
        """List all catalog connections for this tenant."""
        conns = state["catalog_store"].list_connections(key_info["tenant_id"])
        return [_conn_to_response(c) for c in conns]

    @app.get("/v1/catalog/connections/{connection_id}", response_model=CatalogConnectionResponse, tags=["Catalog"])
    def get_catalog_connection(connection_id: str, key_info: dict = Depends(require_api_key)):
        """Get a specific catalog connection."""
        conn = state["catalog_store"].get_connection(connection_id, key_info["tenant_id"])
        if not conn:
            raise HTTPException(status_code=404, detail="Connection not found.")
        return _conn_to_response(conn)

    @app.put("/v1/catalog/connections/{connection_id}", response_model=CatalogConnectionResponse, tags=["Catalog"])
    def update_catalog_connection(
        connection_id: str,
        body: UpdateCatalogConnectionRequest,
        key_info: dict = Depends(require_api_key),
    ):
        """Update mutable fields on a catalog connection."""
        updates = {k: v for k, v in body.model_dump().items() if v is not None}
        found = state["catalog_store"].update_connection(connection_id, updates, key_info["tenant_id"])
        if not found:
            raise HTTPException(status_code=404, detail="Connection not found.")
        conn = state["catalog_store"].get_connection(connection_id, key_info["tenant_id"])

        # Restart sync timer with new interval if changed
        sched = state.get("catalog_scheduler")
        if sched and conn:
            sched.start_connection(conn)

        return _conn_to_response(conn)

    @app.patch("/v1/catalog/connections/{connection_id}/credentials", response_model=CatalogConnectionResponse, tags=["Catalog"])
    def rotate_catalog_credentials(
        connection_id: str,
        body: UpdateCatalogCredentialsRequest,
        key_info: dict = Depends(require_api_key),
    ):
        """
        Replace the credentials for a catalog connection.

        The new credentials are tested against the live catalog before being saved.
        On success the connection's credentials_enc is updated in-place.
        Returns 422 if the connection test fails with the new credentials.
        """
        from autopil.catalog.registry import CatalogRegistry
        from autopil.catalog.credentials import encrypt_credentials

        conn = state["catalog_store"].get_connection(connection_id, key_info["tenant_id"])
        if not conn:
            raise HTTPException(status_code=404, detail="Connection not found.")

        new_enc = encrypt_credentials(body.credentials)

        # Temporarily build a connection object with new creds to test
        import dataclasses
        test_conn = dataclasses.replace(conn, credentials_enc=new_enc)
        try:
            connector = CatalogRegistry.get_connector(test_conn)
            ok, msg = connector.test_connection()
        except Exception as exc:
            raise HTTPException(status_code=422, detail=f"Connector error: {exc}")
        if not ok:
            raise HTTPException(status_code=422, detail=f"Connection test failed: {msg}")

        state["catalog_store"].update_connection(
            connection_id,
            {"credentials_enc": new_enc},
            key_info["tenant_id"],
        )
        updated = state["catalog_store"].get_connection(connection_id, key_info["tenant_id"])
        return _conn_to_response(updated)

    @app.delete("/v1/catalog/connections/{connection_id}", tags=["Catalog"])
    def delete_catalog_connection(connection_id: str, key_info: dict = Depends(require_api_key)):
        """Delete a catalog connection and all its cached classifications."""
        sched = state.get("catalog_scheduler")
        if sched:
            sched.stop_connection(connection_id)
        found = state["catalog_store"].delete_connection(connection_id, key_info["tenant_id"])
        if not found:
            raise HTTPException(status_code=404, detail="Connection not found.")
        return {"status": "deleted", "connection_id": connection_id}

    @app.post("/v1/catalog/connections/{connection_id}/sync", response_model=SyncResultResponse, tags=["Catalog"])
    def trigger_catalog_sync(connection_id: str, key_info: dict = Depends(require_api_key)):
        """Trigger an immediate sync for a catalog connection."""
        sched = state.get("catalog_scheduler")
        if not sched:
            raise HTTPException(status_code=503, detail="Catalog scheduler not running.")
        try:
            result = sched.trigger_now(connection_id, key_info["tenant_id"])
        except ValueError as exc:
            raise HTTPException(status_code=404, detail=str(exc))
        except Exception as exc:
            raise HTTPException(status_code=500, detail=f"Sync error: {exc}")
        return SyncResultResponse(
            sync_id=result.sync_id,
            connection_id=result.connection_id,
            status=result.status,
            assets_synced=result.assets_synced,
            assets_skipped=result.assets_skipped,
            started_at=result.started_at.isoformat(),
            completed_at=result.completed_at.isoformat() if result.completed_at else None,
            error_message=result.error_message,
        )

    @app.get("/v1/catalog/connections/{connection_id}/sync", response_model=list[SyncResultResponse], tags=["Catalog"])
    def list_catalog_sync_history(
        connection_id: str,
        limit: int = Query(default=25, le=100),
        key_info: dict = Depends(require_api_key),
    ):
        """Return sync history for a catalog connection, newest first."""
        history = state["catalog_store"].list_sync_history(
            connection_id, key_info["tenant_id"], limit=limit
        )
        return [
            SyncResultResponse(
                sync_id=r.sync_id, connection_id=r.connection_id, status=r.status,
                assets_synced=r.assets_synced, assets_skipped=r.assets_skipped,
                started_at=r.started_at.isoformat(),
                completed_at=r.completed_at.isoformat() if r.completed_at else None,
                error_message=r.error_message,
            )
            for r in history
        ]

    # ── catalog classifications ───────────────────────────────────────────────

    @app.get("/v1/catalog/classifications", response_model=list[CatalogClassificationResponse], tags=["Catalog"])
    def list_catalog_classifications(
        connection_id: Optional[str] = Query(default=None),
        sensitivity_label: Optional[str] = Query(default=None),
        key_info: dict = Depends(require_api_key),
    ):
        """List cached catalog classifications for this tenant, with optional filters."""
        clss = state["catalog_store"].list_classifications(
            key_info["tenant_id"],
            connection_id=connection_id,
            sensitivity_label=sensitivity_label,
        )
        return [_cls_to_response(c) for c in clss]

    @app.get("/v1/catalog/classifications/lookup", response_model=list[CatalogClassificationResponse], tags=["Catalog"])
    def lookup_classification(
        asset_fqn: str = Query(..., description="Asset FQN to look up (e.g. catalog.schema.table)"),
        key_info: dict = Depends(require_api_key),
    ):
        """Return active classifications for a specific asset FQN."""
        clss = state["catalog_store"].get_classifications_for_asset(
            key_info["tenant_id"], asset_fqn
        )
        return [_cls_to_response(c) for c in clss]

    @app.get("/v1/catalog/stats", response_model=CatalogStatsResponse, tags=["Catalog"])
    def get_catalog_stats(key_info: dict = Depends(require_api_key)):
        """
        Return aggregate catalog health metrics for this tenant.

        Includes connection counts by status, total cached classifications,
        sensitivity distribution, and per-catalog-type coverage counts.
        """
        stats = state["catalog_store"].get_catalog_stats(key_info["tenant_id"])
        return CatalogStatsResponse(**stats)

    # ── catalog webhook ───────────────────────────────────────────────────────

    @app.post("/v1/catalog/webhook/{connection_id}", tags=["Catalog"])
    async def catalog_webhook(
        connection_id: str,
        request: Request,
        key_info: dict = Depends(require_api_key),
    ):
        """
        Receive push events from a data catalog.

        Supports Collibra, Alation, Purview (Azure Event Grid), and a generic
        fallback format.  HMAC signature is verified if webhook_secret is set:
          - Collibra:  X-Collibra-Signature  (HMAC-SHA256 hex, no prefix)
          - Alation:   X-Alation-Signature   (HMAC-SHA256 hex, no prefix)
          - Purview:   aeg-sas-key           (shared-access-key equality check)
          - Generic:   X-Hub-Signature-256   (sha256=<hex>, GitHub-style)

        Parses asset FQNs from the payload and triggers an incremental background
        sync for the affected assets only.
        """
        conn = state["catalog_store"].get_connection(connection_id, key_info["tenant_id"])
        if not conn:
            raise HTTPException(status_code=404, detail="Connection not found.")

        body_bytes = await request.body()

        # ── HMAC / signature verification per catalog type ─────────────────
        if conn.webhook_secret:
            verified = _verify_webhook_signature(
                catalog_type=conn.catalog_type,
                body_bytes=body_bytes,
                headers=dict(request.headers),
                secret=conn.webhook_secret,
            )
            if not verified:
                raise HTTPException(status_code=401, detail="Invalid webhook signature.")

        # ── payload parsing: extract asset FQNs per catalog format ─────────
        asset_fqns = _parse_webhook_asset_fqns(conn.catalog_type, body_bytes)

        # ── trigger incremental sync in background ──────────────────────────
        sched = state.get("catalog_scheduler")
        if sched:
            import threading
            threading.Thread(
                target=sched.trigger_now,
                args=(connection_id, key_info["tenant_id"]),
                daemon=True,
            ).start()

        return {
            "status":        "accepted",
            "connection_id": connection_id,
            "assets_queued": len(asset_fqns),
        }

    # ── webhook helpers (module-level so they're testable) ────────────────────

    def _verify_webhook_signature(
        catalog_type: str,
        body_bytes: bytes,
        headers: dict,
        secret: str,
    ) -> bool:
        """
        Verify inbound webhook HMAC/key signatures.

        Each catalog uses a different header name and format:
          - Collibra:  X-Collibra-Signature  → HMAC-SHA256 hex digest (no prefix)
          - Alation:   X-Alation-Signature   → HMAC-SHA256 hex digest (no prefix)
          - Purview:   aeg-sas-key           → shared-access-key string equality
          - Generic:   X-Hub-Signature-256   → "sha256=" + hex digest (GitHub-style)
        """
        # Normalise header keys to lowercase for case-insensitive lookup
        h = {k.lower(): v for k, v in headers.items()}

        if catalog_type == "collibra":
            received = h.get("x-collibra-signature", "")
            expected = hmac.new(secret.encode(), body_bytes, "sha256").hexdigest()
            return hmac.compare_digest(received, expected)

        if catalog_type == "alation":
            received = h.get("x-alation-signature", "")
            expected = hmac.new(secret.encode(), body_bytes, "sha256").hexdigest()
            return hmac.compare_digest(received, expected)

        if catalog_type == "purview":
            # Azure Event Grid uses a shared-access-key (pre-shared string equality)
            received = h.get("aeg-sas-key", "")
            return hmac.compare_digest(received, secret)

        # Generic / unity_catalog: GitHub-style X-Hub-Signature-256
        received = h.get("x-hub-signature-256", "")
        expected = "sha256=" + hmac.new(secret.encode(), body_bytes, "sha256").hexdigest()
        return hmac.compare_digest(received, expected)

    def _parse_webhook_asset_fqns(catalog_type: str, body_bytes: bytes) -> list[str]:
        """
        Extract asset FQNs from a catalog webhook payload.

        Each catalog uses a different payload structure:
          - Collibra:  {"resourceId": "...", "resourceName": "...", "domainName": "..."}
          - Alation:   {"table_ids": [...], "change_type": "..."}
          - Purview:   Azure Event Grid envelope: [{"subject": "/catalogs/...", ...}]
          - Generic:   {"asset_fqns": [...]} or {"assets": [...]}
        """
        try:
            payload = json.loads(body_bytes)
        except Exception:
            return []

        if catalog_type == "collibra":
            # Collibra sends one event per asset change
            if isinstance(payload, dict):
                domain = payload.get("domainName", "")
                name   = payload.get("resourceName", "")
                if name:
                    fqn = f"{domain}.{name}" if domain else name
                    return [fqn]
            return []

        if catalog_type == "alation":
            # Alation sends table_ids; we can't resolve FQNs without an API call,
            # so return empty list — the incremental sync will handle it.
            return []

        if catalog_type == "purview":
            # Azure Event Grid: array of event objects with a "subject" field
            if isinstance(payload, list):
                fqns = []
                for event in payload:
                    subject = event.get("subject", "")
                    # subject format: /catalogs/{account}/entities/{qualifiedName}
                    if "/entities/" in subject:
                        fqns.append(subject.split("/entities/")[-1])
                return fqns
            return []

        # Generic / unity_catalog
        if isinstance(payload, dict):
            return payload.get("asset_fqns") or payload.get("assets") or []
        return []

    # ── catalog helpers ───────────────────────────────────────────────────────

    def _conn_to_response(c: CatalogConnection) -> CatalogConnectionResponse:
        return CatalogConnectionResponse(
            connection_id=c.connection_id,
            catalog_type=c.catalog_type,
            display_name=c.display_name,
            base_url=c.base_url,
            sync_interval_minutes=c.sync_interval_minutes,
            is_enabled=c.is_enabled,
            status=c.status,
            created_at=c.created_at.isoformat(),
            updated_at=c.updated_at.isoformat(),
            last_sync_at=c.last_sync_at.isoformat() if c.last_sync_at else None,
        )

    def _cls_to_response(c) -> CatalogClassificationResponse:
        return CatalogClassificationResponse(
            classification_id=c.classification_id,
            connection_id=c.connection_id,
            asset_fqn=c.asset_fqn,
            asset_type=c.asset_type,
            sensitivity_label=c.sensitivity_label,
            raw_label=c.raw_label,
            data_classes=c.data_classes,
            is_certified=c.is_certified,
            trust_score=c.trust_score,
            fetched_at=c.fetched_at.isoformat(),
            expires_at=c.expires_at.isoformat(),
        )

    # ── helpers ───────────────────────────────────────────────────────────────

    def _to_response(e: AuditEvent) -> AuditEventResponse:
        return AuditEventResponse(
            event_id=e.event_id, session_id=e.session_id, agent_role=e.agent_role,
            user_id=e.user_id, source_id=e.source_id, query=e.query,
            decision=e.decision, policy_name=e.policy_name, reason=e.reason,
            timestamp=e.timestamp, context_hash=e.context_hash,
            sensitivity_level=e.sensitivity_level,
            source_type=e.source_type,
            catalog_enriched=e.catalog_enriched,
            catalog_connection_id=e.catalog_connection_id,
        )

    return app
