"""
Sheet webhook subscriptions and ingest endpoints.
"""

from __future__ import annotations

import asyncio
from datetime import datetime, timedelta, timezone
import json
import logging
import os
import secrets
import time
from typing import Any, Dict, List, Optional
from uuid import UUID

from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import JSONResponse, Response
import redis
from rq import Queue
from sqlalchemy import select
from sqlalchemy import func as sa_func
from sqlalchemy.exc import IntegrityError, TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.ext.asyncio import AsyncSession
from pydantic import BaseModel, Field
from starlette.requests import ClientDisconnect

from ... import settings as saas_settings
from ...auth_security import require_account
from ...db import AsyncSessionLocal, get_session
from ...model_defs.sheet_webhook_models import SheetWebhookSubscription
from fmatch.saas.model_defs.webhook_models import InboundWebhook
from ...models import Account
from ...security.crypto import decrypt_str
from ...services.direct_sheets_writer import (
    DirectSheetsWriter,
    DirectSheetsWriterError,
    RateLimitedError,
    SheetNotFoundError,
)
from ...services.google_sheets_oauth import ensure_google_access_token
from ...services.usage_gateway import UsageGateway

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/api/v2/webhooks/sheets", tags=["sheet_webhooks"])

WRITE_MODE_APPEND = "append"
WRITE_MODE_REPLACE = "replace"
MAX_PAYLOAD_BYTES = 1_000_000
MAX_PAYLOAD_PREVIEW_BYTES = 10_000
DEFAULT_SHEET_WEBHOOK_RETRY_BASE_SECONDS = 2
SHEET_WEBHOOK_MAX_RETRIES = max(
    0,
    int(
        os.getenv(
            "SHEET_WEBHOOK_MAX_RETRIES",
            str(getattr(saas_settings, "WEBHOOK_MAX_RETRIES", 3)),
        )
    ),
)
SHEET_WEBHOOK_JOB_TIMEOUT_SECONDS = max(
    30,
    int(os.getenv("SHEET_WEBHOOK_JOB_TIMEOUT_SECONDS", "600")),
)
SHEET_WEBHOOK_RESULT_TTL_SECONDS = max(
    60,
    int(os.getenv("SHEET_WEBHOOK_RESULT_TTL_SECONDS", "3600")),
)
SHEET_WEBHOOK_FAILURE_TTL_SECONDS = max(
    60,
    int(os.getenv("SHEET_WEBHOOK_FAILURE_TTL_SECONDS", "86400")),
)

_rq_conn: Optional[redis.Redis] = None
_rq_queue: Optional[Queue] = None
_sheet_webhook_db_overloaded_until_monotonic = 0.0


def _env_int(name: str, default: int, *, minimum: int) -> int:
    try:
        value = int(os.getenv(name, str(default)))
    except (TypeError, ValueError):
        value = default
    return max(minimum, value)


def _env_float(name: str, default: float, *, minimum: float) -> float:
    try:
        value = float(os.getenv(name, str(default)))
    except (TypeError, ValueError):
        value = default
    return max(minimum, value)


SHEET_WEBHOOK_RETRY_AFTER_SECONDS = _env_int(
    "SHEET_WEBHOOK_RETRY_AFTER_SECONDS",
    2,
    minimum=1,
)
SHEET_WEBHOOK_INGEST_MAX_INFLIGHT = _env_int(
    "SHEET_WEBHOOK_INGEST_MAX_INFLIGHT",
    max(1, int(getattr(saas_settings, "DATABASE_POOL_SIZE", 10))),
    minimum=1,
)
SHEET_WEBHOOK_INGEST_ACQUIRE_TIMEOUT_SECONDS = _env_float(
    "SHEET_WEBHOOK_INGEST_ACQUIRE_TIMEOUT_SECONDS",
    0.1,
    minimum=0.01,
)
SHEET_WEBHOOK_DB_COOLDOWN_SECONDS = _env_float(
    "SHEET_WEBHOOK_DB_COOLDOWN_SECONDS",
    float(SHEET_WEBHOOK_RETRY_AFTER_SECONDS),
    minimum=0.1,
)
_sheet_webhook_ingest_semaphore = asyncio.Semaphore(SHEET_WEBHOOK_INGEST_MAX_INFLIGHT)


def _is_db_capacity_error(exc: BaseException) -> bool:
    if isinstance(exc, SQLAlchemyTimeoutError):
        return True

    markers = (
        "toomanyconnectionserror",
        "remaining connection slots",
        "remaining connection slots are reserved",
        "queuepool limit",
        "connection pool",
        "connection pool is full",
        "timed out waiting for connection",
        "sorry, too many clients already",
        "too many connections",
        "too many clients",
    )

    seen = set()
    stack: List[Optional[BaseException]] = [exc]
    while stack:
        current = stack.pop()
        if current is None:
            continue

        marker = id(current)
        if marker in seen:
            continue
        seen.add(marker)

        text = str(current).lower()
        if any(candidate in text for candidate in markers):
            return True

        stack.append(getattr(current, "orig", None))
        stack.append(getattr(current, "__cause__", None))
        stack.append(getattr(current, "__context__", None))

    return False


def _mark_db_overloaded() -> None:
    global _sheet_webhook_db_overloaded_until_monotonic
    overloaded_until = time.monotonic() + SHEET_WEBHOOK_DB_COOLDOWN_SECONDS
    if overloaded_until > _sheet_webhook_db_overloaded_until_monotonic:
        _sheet_webhook_db_overloaded_until_monotonic = overloaded_until


def _db_overload_window_active() -> bool:
    return time.monotonic() < _sheet_webhook_db_overloaded_until_monotonic


def _service_overloaded_response(message: str) -> JSONResponse:
    return JSONResponse(
        status_code=503,
        content={
            "ok": False,
            "error": "service_overloaded",
            "message": message,
        },
        headers={"Retry-After": str(SHEET_WEBHOOK_RETRY_AFTER_SECONDS)},
    )


def _ingest_over_capacity_response(message: str) -> JSONResponse:
    return JSONResponse(
        status_code=429,
        content={
            "ok": False,
            "error": "service_overloaded",
            "code": "ingest_over_capacity",
            "message": message,
        },
        headers={"Retry-After": str(SHEET_WEBHOOK_RETRY_AFTER_SECONDS)},
    )


def _service_overloaded_exception(message: str) -> HTTPException:
    return HTTPException(
        status_code=503,
        detail=message,
        headers={"Retry-After": str(SHEET_WEBHOOK_RETRY_AFTER_SECONDS)},
    )


async def _acquire_ingest_slot() -> bool:
    try:
        await asyncio.wait_for(
            _sheet_webhook_ingest_semaphore.acquire(),
            timeout=SHEET_WEBHOOK_INGEST_ACQUIRE_TIMEOUT_SECONDS,
        )
        return True
    except asyncio.TimeoutError:
        return False


class CreateSheetWebhookRequest(BaseModel):
    name: str = Field(..., min_length=1, max_length=255)
    spreadsheet_id: str = Field(..., min_length=1, max_length=255)
    sheet_name: str = Field(..., min_length=1, max_length=255)
    create_sheet: bool = False
    write_mode: str = WRITE_MODE_APPEND
    active: bool = True


class UpdateSheetWebhookRequest(BaseModel):
    name: Optional[str] = Field(default=None, min_length=1, max_length=255)
    spreadsheet_id: Optional[str] = Field(default=None, min_length=1, max_length=255)
    sheet_name: Optional[str] = Field(default=None, min_length=1, max_length=255)
    create_sheet: Optional[bool] = None
    write_mode: Optional[str] = None
    active: Optional[bool] = None


def _normalize_write_mode(mode: Optional[str]) -> str:
    value = str(mode or WRITE_MODE_APPEND).strip().lower()
    if value not in {WRITE_MODE_APPEND, WRITE_MODE_REPLACE}:
        raise HTTPException(status_code=400, detail="write_mode must be append or replace")
    return value


def _normalize_sheet_name(name: Optional[str]) -> str:
    value = str(name or "").strip()
    if not value:
        raise HTTPException(status_code=400, detail="sheet_name is required")
    return value


def _generate_secret() -> str:
    return secrets.token_hex(16)


def _build_ingest_url(request: Request, secret: str) -> str:
    configured_base = (
        os.getenv("PUBLIC_API_BASE_URL")
        or getattr(saas_settings, "PUBLIC_API_BASE_URL", None)
        or getattr(saas_settings, "PUBLIC_BASE_URL", None)
    )
    if configured_base:
        normalized_base = str(configured_base).strip().rstrip("/")
        if normalized_base.startswith("https://") or normalized_base.startswith("http://"):
            return f"{normalized_base}/api/v2/webhooks/sheets/{secret}/ingest"
    return str(request.url_for("ingest_sheet_webhook", secret=secret))


def _get_sheet_webhook_queue() -> Queue:
    global _rq_conn
    global _rq_queue

    if _rq_queue is not None:
        return _rq_queue

    redis_url = saas_settings.REDIS_URL or "redis://127.0.0.1:6379/0"
    _rq_conn = redis.from_url(redis_url)
    _rq_queue = Queue(
        getattr(saas_settings, "RQ_QUEUE_NAME", "default"),
        connection=_rq_conn,
    )
    return _rq_queue


def _enqueue_sheet_webhook_delivery(
    inbound_id: str,
    *,
    delay_seconds: int = 0,
    db_retry_attempt: int = 0,
) -> str:
    queue = _get_sheet_webhook_queue()

    safe_db_retry_attempt = max(0, int(db_retry_attempt))
    enqueue_kwargs: Dict[str, Any] = {
        "job_timeout": SHEET_WEBHOOK_JOB_TIMEOUT_SECONDS,
        "result_ttl": SHEET_WEBHOOK_RESULT_TTL_SECONDS,
        "failure_ttl": SHEET_WEBHOOK_FAILURE_TTL_SECONDS,
        "description": (
            f"sheet_webhook_delivery:{inbound_id}:db_retry={safe_db_retry_attempt}"
        ),
    }

    if delay_seconds > 0:
        job = queue.enqueue_in(
            timedelta(seconds=delay_seconds),
            "fmatch.saas.worker.run_sheet_webhook_delivery",
            inbound_id,
            safe_db_retry_attempt,
            **enqueue_kwargs,
        )
    else:
        job = queue.enqueue(
            "fmatch.saas.worker.run_sheet_webhook_delivery",
            inbound_id,
            safe_db_retry_attempt,
            **enqueue_kwargs,
        )
    return str(job.id)


def _compute_retry_delay_seconds(retry_attempt: int) -> int:
    # Exponential backoff with a conservative cap to avoid huge retry gaps.
    delay = DEFAULT_SHEET_WEBHOOK_RETRY_BASE_SECONDS * (2 ** max(0, retry_attempt - 1))
    return max(DEFAULT_SHEET_WEBHOOK_RETRY_BASE_SECONDS, min(60, delay))


def _next_month_start_utc(now: Optional[datetime] = None) -> datetime:
    current = now or datetime.now(timezone.utc)
    if current.month == 12:
        return datetime(current.year + 1, 1, 1, tzinfo=timezone.utc)
    return datetime(current.year, current.month + 1, 1, tzinfo=timezone.utc)


def _retry_after_seconds_until(reset_at: datetime) -> int:
    now = datetime.now(timezone.utc)
    return max(1, int((reset_at - now).total_seconds()))


def _build_quota_exceeded_response(
    *,
    used: int,
    limit: int,
    rows_attempted: int,
) -> JSONResponse:
    reset_at = _next_month_start_utc()
    retry_after_seconds = _retry_after_seconds_until(reset_at)
    return JSONResponse(
        status_code=429,
        headers={"Retry-After": str(retry_after_seconds)},
        content={
            "ok": False,
            "error": (
                f"Monthly webhook row limit reached "
                f"({used}/{limit}, attempted +{rows_attempted})."
            ),
            "reset_at": reset_at.isoformat(),
        },
    )


def _serialize_subscription(
    sub: SheetWebhookSubscription,
    request: Request,
    include_secret: bool = False,
) -> Dict[str, Any]:
    data = {
        "id": str(sub.id),
        "name": sub.name,
        "account_id": sub.account_id,
        "spreadsheet_id": sub.spreadsheet_id,
        "sheet_name": sub.sheet_name,
        "create_sheet": bool(sub.create_sheet),
        "write_mode": sub.write_mode,
        "active": bool(sub.active),
        "webhook_url": _build_ingest_url(request, sub.secret),
        "last_received_at": sub.last_received_at.isoformat() if sub.last_received_at else None,
        "total_deliveries": int(sub.total_deliveries or 0),
        "total_rows_written": int(sub.total_rows_written or 0),
        "last_error": sub.last_error,
        "created_at": sub.created_at.isoformat() if sub.created_at else None,
        "updated_at": sub.updated_at.isoformat() if sub.updated_at else None,
    }
    if include_secret:
        data["secret"] = sub.secret
    return data


def _truncate_json_payload_for_storage(payload: Any) -> Dict[str, Any]:
    if isinstance(payload, dict):
        out: Dict[str, Any] = payload
    elif isinstance(payload, list):
        out = {"records": payload}
    else:
        out = {"value": payload}

    try:
        raw = json.dumps(out, ensure_ascii=False, default=str)
    except Exception:
        return {"preview": "<unserializable payload>"}

    if len(raw.encode("utf-8")) <= MAX_PAYLOAD_PREVIEW_BYTES:
        return out

    preview = raw.encode("utf-8")[:MAX_PAYLOAD_PREVIEW_BYTES].decode("utf-8", "ignore")
    return {"_truncated": True, "preview": preview}


def _normalize_payload_to_records(payload: Any) -> List[Dict[str, Any]]:
    if isinstance(payload, dict):
        return [payload]
    if not isinstance(payload, list):
        raise HTTPException(
            status_code=400,
            detail="Payload must be a JSON object or an array of JSON objects",
        )

    records: List[Dict[str, Any]] = []
    for idx, item in enumerate(payload):
        if not isinstance(item, dict):
            raise HTTPException(
                status_code=400,
                detail=f"Payload item at index {idx} must be a JSON object",
            )
        records.append(item)
    return records


def _parse_and_validate_payload(raw_body: str) -> Dict[str, Any]:
    try:
        parsed_payload = json.loads(raw_body or "{}")
    except json.JSONDecodeError as exc:
        raise HTTPException(status_code=400, detail=f"Invalid JSON payload: {exc.msg}") from exc

    records = _normalize_payload_to_records(parsed_payload)
    if not records:
        raise HTTPException(status_code=400, detail="Payload must contain at least one record")

    incoming_keys = _collect_incoming_keys(records)
    if not incoming_keys:
        raise HTTPException(
            status_code=400,
            detail="Payload records must contain at least one key",
        )

    return {
        "parsed_payload": parsed_payload,
        "records": records,
        "incoming_keys": incoming_keys,
        "payload_preview": _truncate_json_payload_for_storage(parsed_payload),
    }


def _collect_incoming_keys(records: List[Dict[str, Any]]) -> List[str]:
    ordered: List[str] = []
    seen = set()
    for record in records:
        for key in record.keys():
            k = str(key)
            if k not in seen:
                seen.add(k)
                ordered.append(k)
    return ordered


def _normalize_cell_value(value: Any) -> Any:
    if value is None:
        return ""
    if isinstance(value, (str, int, float, bool)):
        return value
    if isinstance(value, datetime):
        return value.isoformat()
    try:
        return json.dumps(value, ensure_ascii=False, default=str)
    except Exception:
        return str(value)


def _payload_preview_from_inbound(webhook: InboundWebhook) -> str:
    if webhook.raw_body:
        text = webhook.raw_body
    else:
        try:
            text = json.dumps(webhook.payload or {}, ensure_ascii=False, default=str)
        except Exception:
            text = "{}"
    if len(text.encode("utf-8")) <= 1024:
        return text
    return text.encode("utf-8")[:1024].decode("utf-8", "ignore") + "..."


async def _get_subscription_for_account(
    session: AsyncSession, account_id: str, sub_id: UUID
) -> SheetWebhookSubscription:
    result = await session.execute(
        select(SheetWebhookSubscription).where(
            SheetWebhookSubscription.id == sub_id,
            SheetWebhookSubscription.account_id == str(account_id),
        )
    )
    sub = result.scalar_one_or_none()
    if not sub:
        raise HTTPException(status_code=404, detail="Sheet webhook subscription not found")
    return sub


async def _check_active_webhook_limit(
    session: AsyncSession,
    account_id: str,
    *,
    exclude_sub_id: Optional[UUID] = None,
) -> None:
    """Raise 429 when the account is at its active webhook cap."""
    gateway = UsageGateway(session)
    limits = await gateway.get_webhook_limits(str(account_id))
    max_webhooks = limits.get("max_webhooks")
    if max_webhooks is None:
        return

    # Serialize active-count checks per account to reduce race windows on create/reactivate.
    await session.execute(
        select(Account.id).where(Account.id == str(account_id)).with_for_update()
    )

    query = select(sa_func.count(SheetWebhookSubscription.id)).where(
        SheetWebhookSubscription.account_id == str(account_id),
        SheetWebhookSubscription.active.is_(True),
    )
    if exclude_sub_id is not None:
        query = query.where(SheetWebhookSubscription.id != exclude_sub_id)
    active_count = int((await session.execute(query)).scalar() or 0)

    if active_count >= int(max_webhooks):
        raise HTTPException(
            status_code=429,
            detail=(
                f"Webhook limit reached ({active_count}/{max_webhooks}). "
                "Upgrade your plan for more webhooks."
            ),
        )


async def _mark_failed_delivery(
    session: AsyncSession,
    sub: SheetWebhookSubscription,
    inbound: Optional[InboundWebhook],
    *,
    error_message: str,
    error_code: str,
    status_code: int,
    payload_for_storage: Dict[str, Any],
    started_at: datetime,
) -> None:
    now = datetime.utcnow()
    sub.last_received_at = now
    sub.total_deliveries = int(sub.total_deliveries or 0) + 1
    sub.last_error = error_message[:500]
    sub.last_payload = payload_for_storage
    sub.updated_at = now

    if inbound:
        inbound.status = "failed"
        inbound.error = error_message[:500]
        inbound.error_code = error_code
        inbound.should_retry = False
        inbound.response_status_code = status_code
        inbound.response_body = {"ok": False, "error": error_message[:500]}
        inbound.processed_at = now
        inbound.processing_time_ms = max(0, int((now - started_at).total_seconds() * 1000))

    await session.commit()


async def _safe_mark_failed_delivery(
    session: AsyncSession,
    sub: SheetWebhookSubscription,
    inbound: Optional[InboundWebhook],
    *,
    error_message: str,
    error_code: str,
    status_code: int,
    payload_for_storage: Dict[str, Any],
    started_at: datetime,
) -> None:
    try:
        await _mark_failed_delivery(
            session,
            sub,
            inbound,
            error_message=error_message,
            error_code=error_code,
            status_code=status_code,
            payload_for_storage=payload_for_storage,
            started_at=started_at,
        )
    except Exception:
        logger.warning(
            "Failed to record sheet webhook delivery error",
            exc_info=True,
        )
        try:
            await session.rollback()
        except Exception:
            logger.warning("Failed to rollback session after delivery logging error", exc_info=True)


async def _mark_failed_inbound_only(
    session: AsyncSession,
    inbound: InboundWebhook,
    *,
    error_message: str,
    error_code: str,
    status_code: int,
    started_at: datetime,
) -> None:
    now = datetime.utcnow()
    inbound.status = "failed"
    inbound.error = error_message[:500]
    inbound.error_code = error_code
    inbound.should_retry = False
    inbound.response_status_code = status_code
    inbound.response_body = {"ok": False, "error": error_message[:500]}
    inbound.processed_at = now
    inbound.processing_time_ms = max(0, int((now - started_at).total_seconds() * 1000))
    await session.commit()


async def process_queued_sheet_webhook_delivery(inbound_id: str) -> None:
    started_at = datetime.utcnow()
    try:
        inbound_uuid = UUID(str(inbound_id))
    except Exception:
        logger.warning("Invalid sheet webhook inbound id: %s", inbound_id)
        return

    async with AsyncSessionLocal() as session:
        inbound = await session.get(InboundWebhook, inbound_uuid)
        if not inbound:
            logger.warning("Sheet webhook inbound delivery not found: %s", inbound_id)
            return

        if inbound.status == "processed":
            logger.debug("Sheet webhook inbound delivery already processed: %s", inbound_id)
            return

        payload_for_storage: Dict[str, Any] = (
            inbound.payload if isinstance(inbound.payload, dict) else {"preview": inbound.raw_body or ""}
        )

        sub: Optional[SheetWebhookSubscription] = None
        integration_id = str(inbound.integration_id or "").strip()
        try:
            sub_uuid = UUID(integration_id) if integration_id else None
        except Exception:
            sub_uuid = None

        if sub_uuid:
            result = await session.execute(
                select(SheetWebhookSubscription).where(
                    SheetWebhookSubscription.id == sub_uuid,
                    SheetWebhookSubscription.account_id == str(inbound.account_id or ""),
                )
            )
            sub = result.scalar_one_or_none()

        if not sub:
            await _mark_failed_inbound_only(
                session,
                inbound,
                error_message="Sheet webhook subscription not found",
                error_code="SUBSCRIPTION_NOT_FOUND",
                status_code=404,
                started_at=started_at,
            )
            return

        if not bool(sub.active):
            await _safe_mark_failed_delivery(
                session,
                sub,
                inbound,
                error_message="Webhook not found",
                error_code="SUBSCRIPTION_INACTIVE",
                status_code=404,
                payload_for_storage=payload_for_storage,
                started_at=started_at,
            )
            return

        try:
            gateway = UsageGateway(session)
            records_blob = None
            if isinstance(inbound.parsed_data, dict):
                candidate = inbound.parsed_data.get("records")
                if isinstance(candidate, list):
                    records_blob = candidate

            if records_blob is None:
                parsed = _parse_and_validate_payload(inbound.raw_body or "")
                records = parsed["records"]
                incoming_keys = parsed["incoming_keys"]
                payload_for_storage = parsed["payload_preview"]
                inbound.payload = payload_for_storage
            else:
                records = _normalize_payload_to_records(records_blob)
                incoming_keys = _collect_incoming_keys(records)
                if not incoming_keys:
                    raise HTTPException(
                        status_code=400,
                        detail="Payload records must contain at least one key",
                    )

            limits = await gateway.get_webhook_limits(str(sub.account_id))
            webhook_limit = limits.get("webhook_rows_per_month")
            webhook_used = int(limits.get("webhook_rows_used") or 0)
            if webhook_limit is not None and webhook_used + len(records) > int(webhook_limit):
                reset_at = _next_month_start_utc()
                retry_after_seconds = _retry_after_seconds_until(reset_at)
                now = datetime.utcnow()

                sub.last_received_at = now
                sub.total_deliveries = int(sub.total_deliveries or 0) + 1
                sub.last_error = (
                    f"Monthly webhook row limit reached ({webhook_used}/{webhook_limit})"
                )
                sub.last_payload = payload_for_storage
                sub.updated_at = now

                inbound.status = "failed"
                inbound.error = sub.last_error
                inbound.error_code = "WEBHOOK_ROWS_LIMIT"
                inbound.should_retry = False
                inbound.response_status_code = 429
                inbound.response_body = {
                    "ok": False,
                    "error": sub.last_error,
                    "reset_at": reset_at.isoformat(),
                    "retry_after_seconds": retry_after_seconds,
                }
                inbound.processed_at = now
                inbound.processing_time_ms = max(
                    0, int((now - started_at).total_seconds() * 1000)
                )
                await session.commit()
                return

            inbound.status = "processing"
            inbound.error = None
            inbound.error_code = None
            await session.commit()

            try:
                access_token, integration = await ensure_google_access_token(sub.account_id, session)
            except Exception as exc:
                raise HTTPException(
                    status_code=403,
                    detail=(
                        "Google Sheets authorization expired or invalid. "
                        "Reconnect Google OAuth in Schedule Manager."
                    ),
                ) from exc
            if not access_token:
                raise HTTPException(
                    status_code=403,
                    detail=(
                        "Google Sheets authorization required. "
                        "Connect Google OAuth in Schedule Manager."
                    ),
                )

            refresh_token = None
            if integration and integration.refresh_token:
                refresh_token = (
                    decrypt_str(integration.refresh_token)
                    if saas_settings.ENCRYPTION_KEY
                    else integration.refresh_token
                )

            writer = DirectSheetsWriter(
                access_token=access_token,
                refresh_token=refresh_token,
                expires_at=integration.token_expires_at if integration else None,
                account_id=sub.account_id,
                max_429_retries=1,
            )

            if sub.create_sheet:
                await asyncio.to_thread(
                    writer.ensure_sheet_exists,
                    sub.spreadsheet_id,
                    sub.sheet_name,
                )

            header_info = await asyncio.to_thread(
                writer.read_header_row,
                spreadsheet_id=sub.spreadsheet_id,
                sheet_name=sub.sheet_name,
                cached_title=sub.sheet_name,
                create_sheet=sub.create_sheet,
            )
            existing_headers = [
                str(h).strip() for h in (header_info.get("headers") or []) if str(h).strip()
            ]
            resolved_sheet_name = str(header_info.get("sheet_name") or sub.sheet_name)

            merged_headers = list(existing_headers)
            seen_headers = set(existing_headers)
            new_headers: List[str] = []
            for key in incoming_keys:
                if key not in seen_headers:
                    seen_headers.add(key)
                    merged_headers.append(key)
                    new_headers.append(key)

            if not merged_headers:
                raise HTTPException(status_code=400, detail="No writable columns detected in payload")

            if not existing_headers or new_headers:
                await asyncio.to_thread(
                    writer.update_header_row,
                    spreadsheet_id=sub.spreadsheet_id,
                    sheet_name=resolved_sheet_name,
                    headers=merged_headers,
                )

            aligned_rows: List[List[Any]] = []
            for record in records:
                aligned_rows.append(
                    [_normalize_cell_value(record.get(header, "")) for header in merged_headers]
                )

            if sub.write_mode == WRITE_MODE_REPLACE:
                await asyncio.to_thread(
                    writer.write_to_sheet,
                    spreadsheet_id=sub.spreadsheet_id,
                    sheet_name=resolved_sheet_name,
                    headers=merged_headers,
                    rows=aligned_rows,
                    cached_title=resolved_sheet_name,
                )
            else:
                await asyncio.to_thread(
                    writer.append_to_sheet,
                    spreadsheet_id=sub.spreadsheet_id,
                    sheet_name=resolved_sheet_name,
                    rows=aligned_rows,
                    cached_title=resolved_sheet_name,
                    create_sheet=sub.create_sheet,
                )

            rows_written = len(aligned_rows)
            if rows_written > 0:
                consume = await gateway.check_and_consume(
                    account_id=sub.account_id,
                    resource="webhook_rows",
                    amount=rows_written,
                    source=f"sheet_webhook:{sub.id}",
                    idempotency_key=f"sheet_webhook_rows:{inbound.id}",
                )
                if not consume.allowed:
                    logger.warning(
                        "Post-write webhook row consumption blocked account=%s inbound=%s used=%s limit=%s",
                        sub.account_id,
                        inbound.id,
                        consume.used,
                        consume.limit,
                    )

            now = datetime.utcnow()
            sub.last_received_at = now
            sub.total_deliveries = int(sub.total_deliveries or 0) + 1
            sub.total_rows_written = int(sub.total_rows_written or 0) + rows_written
            sub.last_error = None
            sub.last_payload = payload_for_storage
            sub.updated_at = now

            inbound.status = "processed"
            inbound.payload = payload_for_storage
            inbound.record_count = len(records)
            inbound.should_retry = False
            inbound.error = None
            inbound.error_code = None
            inbound.response_status_code = 200
            inbound.response_body = {"ok": True, "rows_written": rows_written}
            inbound.processed_at = now
            inbound.processing_time_ms = max(0, int((now - started_at).total_seconds() * 1000))

            await session.commit()
            return

        except HTTPException as exc:
            await _safe_mark_failed_delivery(
                session,
                sub,
                inbound,
                error_message=str(exc.detail),
                error_code="INGEST_HTTP_ERROR",
                status_code=exc.status_code,
                payload_for_storage=payload_for_storage,
                started_at=started_at,
            )
            return
        except RateLimitedError as exc:
            retry_count = int(inbound.retry_count or 0)
            if retry_count < SHEET_WEBHOOK_MAX_RETRIES:
                next_retry = retry_count + 1
                delay_seconds = _compute_retry_delay_seconds(next_retry)
                now = datetime.utcnow()

                inbound.status = "queued"
                inbound.should_retry = True
                inbound.retry_count = next_retry
                inbound.error = str(exc)[:500]
                inbound.error_code = "GOOGLE_RATE_LIMIT_RETRY"
                inbound.response_status_code = 429
                inbound.response_body = {
                    "ok": False,
                    "queued": True,
                    "retry_in_seconds": delay_seconds,
                    "retry_count": next_retry,
                }
                inbound.processed_at = now
                inbound.processing_time_ms = max(
                    0, int((now - started_at).total_seconds() * 1000)
                )

                sub.last_received_at = now
                sub.last_error = inbound.error
                sub.last_payload = payload_for_storage
                sub.updated_at = now
                await session.commit()

                try:
                    job_id = await asyncio.to_thread(
                        _enqueue_sheet_webhook_delivery,
                        str(inbound.id),
                        delay_seconds=delay_seconds,
                    )
                    logger.info(
                        "Requeued sheet webhook inbound delivery %s (retry=%s delay=%ss rq_job=%s)",
                        inbound.id,
                        next_retry,
                        delay_seconds,
                        job_id,
                    )
                except Exception as enqueue_exc:
                    logger.exception(
                        "Failed to enqueue retry for sheet webhook inbound delivery %s",
                        inbound.id,
                    )
                    await _safe_mark_failed_delivery(
                        session,
                        sub,
                        inbound,
                        error_message=(
                            f"{str(exc)[:250]}; retry enqueue failed: {str(enqueue_exc)[:200]}"
                        ),
                        error_code="QUEUE_RETRY_FAILED",
                        status_code=503,
                        payload_for_storage=payload_for_storage,
                        started_at=started_at,
                    )
                return

            await _safe_mark_failed_delivery(
                session,
                sub,
                inbound,
                error_message=str(exc),
                error_code="GOOGLE_RATE_LIMIT",
                status_code=429,
                payload_for_storage=payload_for_storage,
                started_at=started_at,
            )
            return
        except SheetNotFoundError as exc:
            await _safe_mark_failed_delivery(
                session,
                sub,
                inbound,
                error_message=str(exc),
                error_code="SHEET_NOT_FOUND",
                status_code=404,
                payload_for_storage=payload_for_storage,
                started_at=started_at,
            )
            return
        except DirectSheetsWriterError as exc:
            await _safe_mark_failed_delivery(
                session,
                sub,
                inbound,
                error_message=str(exc),
                error_code="GOOGLE_WRITE_FAILED",
                status_code=502,
                payload_for_storage=payload_for_storage,
                started_at=started_at,
            )
            return
        except Exception as exc:  # pragma: no cover
            logger.exception("Sheet webhook worker delivery failed")
            await _safe_mark_failed_delivery(
                session,
                sub,
                inbound,
                error_message=str(exc),
                error_code="INGEST_INTERNAL_ERROR",
                status_code=500,
                payload_for_storage=payload_for_storage,
                started_at=started_at,
            )
            return


@router.get("")
async def list_sheet_webhooks(
    request: Request,
    spreadsheet_id: Optional[str] = Query(default=None),
    account_id: str = Depends(require_account),
    session: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    query = select(SheetWebhookSubscription).where(
        SheetWebhookSubscription.account_id == str(account_id)
    )
    if spreadsheet_id:
        query = query.where(SheetWebhookSubscription.spreadsheet_id == spreadsheet_id)
    query = query.order_by(SheetWebhookSubscription.created_at.desc())

    result = await session.execute(query)
    subscriptions = result.scalars().all()
    active_count_result = await session.execute(
        select(sa_func.count(SheetWebhookSubscription.id)).where(
            SheetWebhookSubscription.account_id == str(account_id),
            SheetWebhookSubscription.active.is_(True),
        )
    )
    active_count = int(active_count_result.scalar() or 0)
    gateway = UsageGateway(session)
    limits = await gateway.get_webhook_limits(str(account_id))
    return {
        "ok": True,
        "data": {
            "subscriptions": [_serialize_subscription(sub, request) for sub in subscriptions],
            "limits": {
                **limits,
                "active_webhooks": active_count,
            },
        },
    }


@router.post("")
async def create_sheet_webhook(
    request: Request,
    body: CreateSheetWebhookRequest,
    account_id: str = Depends(require_account),
    session: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    write_mode = _normalize_write_mode(body.write_mode)
    sheet_name = _normalize_sheet_name(body.sheet_name)
    name = str(body.name or "").strip()
    spreadsheet_id = str(body.spreadsheet_id or "").strip()
    if not name:
        raise HTTPException(status_code=400, detail="name is required")
    if not spreadsheet_id:
        raise HTTPException(status_code=400, detail="spreadsheet_id is required")
    if bool(body.active):
        await _check_active_webhook_limit(session, account_id)
    secret = _generate_secret()

    for _ in range(5):
        sub = SheetWebhookSubscription(
            account_id=str(account_id),
            name=name,
            spreadsheet_id=spreadsheet_id,
            sheet_name=sheet_name,
            create_sheet=bool(body.create_sheet),
            write_mode=write_mode,
            secret=secret,
            active=bool(body.active),
            created_at=datetime.utcnow(),
            updated_at=datetime.utcnow(),
        )
        session.add(sub)
        try:
            await session.commit()
            await session.refresh(sub)
            return {
                "ok": True,
                "data": {
                    "subscription": _serialize_subscription(
                        sub,
                        request,
                        include_secret=True,
                    )
                },
            }
        except IntegrityError:
            await session.rollback()
            secret = _generate_secret()

    raise HTTPException(status_code=500, detail="Failed to allocate unique webhook secret")


@router.put("/{sub_id}")
async def update_sheet_webhook(
    sub_id: UUID,
    request: Request,
    body: UpdateSheetWebhookRequest,
    account_id: str = Depends(require_account),
    session: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    sub = await _get_subscription_for_account(session, account_id, sub_id)

    if body.name is not None:
        name = body.name.strip()
        if not name:
            raise HTTPException(status_code=400, detail="name cannot be empty")
        sub.name = name
    if body.spreadsheet_id is not None:
        spreadsheet_id = body.spreadsheet_id.strip()
        if not spreadsheet_id:
            raise HTTPException(status_code=400, detail="spreadsheet_id cannot be empty")
        sub.spreadsheet_id = spreadsheet_id
    if body.sheet_name is not None:
        sub.sheet_name = _normalize_sheet_name(body.sheet_name)
    if body.create_sheet is not None:
        sub.create_sheet = bool(body.create_sheet)
    if body.write_mode is not None:
        sub.write_mode = _normalize_write_mode(body.write_mode)
    if body.active is not None:
        new_active = bool(body.active)
        if new_active and not bool(sub.active):
            await _check_active_webhook_limit(
                session,
                account_id,
                exclude_sub_id=sub.id,
            )
        sub.active = new_active

    sub.updated_at = datetime.utcnow()
    await session.commit()
    await session.refresh(sub)
    return {
        "ok": True,
        "data": {"subscription": _serialize_subscription(sub, request)},
    }


@router.delete("/{sub_id}")
async def delete_sheet_webhook(
    sub_id: UUID,
    account_id: str = Depends(require_account),
    session: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    sub = await _get_subscription_for_account(session, account_id, sub_id)
    await session.delete(sub)
    await session.commit()
    return {"ok": True}


@router.get("/{sub_id}/history")
async def get_sheet_webhook_history(
    sub_id: UUID,
    limit: int = Query(default=20, ge=1, le=50),
    account_id: str = Depends(require_account),
    session: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    sub = await _get_subscription_for_account(session, account_id, sub_id)
    gateway = UsageGateway(session)
    limits = await gateway.get_webhook_limits(str(account_id))
    retention_days = int(limits.get("history_retention_days") or 1)
    retention_cutoff = datetime.utcnow() - timedelta(days=retention_days)
    result = await session.execute(
        select(InboundWebhook)
        .where(
            InboundWebhook.integration_id == str(sub.id),
            InboundWebhook.source == "sheet_webhook",
            InboundWebhook.received_at >= retention_cutoff,
        )
        .order_by(InboundWebhook.received_at.desc())
        .limit(limit)
    )
    rows = result.scalars().all()

    deliveries: List[Dict[str, Any]] = []
    for row in rows:
        response_body = row.response_body if isinstance(row.response_body, dict) else {}
        deliveries.append(
            {
                "id": str(row.id),
                "received_at": row.received_at.isoformat() if row.received_at else None,
                "status": row.status,
                "rows_written": response_body.get("rows_written", row.record_count),
                "response_status_code": row.response_status_code,
                "error": row.error,
                "payload_preview": _payload_preview_from_inbound(row),
            }
        )

    return {
        "ok": True,
        "data": {
            "subscription_id": str(sub.id),
            "deliveries": deliveries,
        },
    }


@router.post("/{sub_id}/rotate-secret")
async def rotate_sheet_webhook_secret(
    sub_id: UUID,
    request: Request,
    account_id: str = Depends(require_account),
    session: AsyncSession = Depends(get_session),
) -> Dict[str, Any]:
    sub = await _get_subscription_for_account(session, account_id, sub_id)
    for _ in range(5):
        sub.secret = _generate_secret()
        sub.updated_at = datetime.utcnow()
        try:
            await session.commit()
            await session.refresh(sub)
            return {
                "ok": True,
                "data": {
                    "subscription": _serialize_subscription(
                        sub,
                        request,
                        include_secret=True,
                    )
                },
            }
        except IntegrityError:
            await session.rollback()

    raise HTTPException(status_code=500, detail="Failed to rotate webhook secret")


@router.post("/{secret}/ingest", name="ingest_sheet_webhook")
async def ingest_sheet_webhook(
    secret: str,
    request: Request,
):
    started_at = datetime.utcnow()
    try:
        body_bytes = await request.body()
    except ClientDisconnect:
        logger.warning(
            "Client disconnected before payload body could be read for sheet webhook secret=%s...",
            secret[:8],
        )
        return Response(status_code=499)
    if len(body_bytes) > MAX_PAYLOAD_BYTES:
        raise HTTPException(status_code=413, detail="Payload exceeds 1MB limit")

    raw_body = body_bytes.decode("utf-8", errors="replace")
    parsed = _parse_and_validate_payload(raw_body)
    records = parsed["records"]
    payload_for_storage = parsed["payload_preview"]

    if _db_overload_window_active():
        return _ingest_over_capacity_response(
            "Database temporarily overloaded; retry shortly"
        )

    slot_acquired = await _acquire_ingest_slot()
    if not slot_acquired:
        logger.warning(
            "Sheet webhook ingest backpressure for secret=%s... inflight_limit=%s",
            secret[:8],
            SHEET_WEBHOOK_INGEST_MAX_INFLIGHT,
        )
        return _ingest_over_capacity_response("Webhook ingest overloaded; retry shortly")

    try:
        async with AsyncSessionLocal() as session:
            try:
                result = await session.execute(
                    select(SheetWebhookSubscription).where(
                        SheetWebhookSubscription.secret == secret,
                        SheetWebhookSubscription.active == True,
                    )
                )
            except Exception as exc:
                if _is_db_capacity_error(exc):
                    _mark_db_overloaded()
                    raise _service_overloaded_exception(
                        "Database temporarily overloaded; retry shortly"
                    ) from exc
                raise HTTPException(status_code=500, detail="Database query failed") from exc

            sub = result.scalar_one_or_none()
            if not sub:
                raise HTTPException(status_code=404, detail="Webhook not found")

            gateway = UsageGateway(session)
            limits = await gateway.get_webhook_limits(str(sub.account_id))
            webhook_limit = limits.get("webhook_rows_per_month")
            webhook_used = int(limits.get("webhook_rows_used") or 0)
            if (
                webhook_limit is not None
                and webhook_used + len(records) > int(webhook_limit)
            ):
                return _build_quota_exceeded_response(
                    used=webhook_used,
                    limit=int(webhook_limit),
                    rows_attempted=len(records),
                )

            inbound = InboundWebhook(
                account_id=sub.account_id,
                source="sheet_webhook",
                source_ip=request.client.host if request.client else None,
                user_agent=request.headers.get("User-Agent"),
                method=request.method,
                path=str(request.url.path),
                headers=dict(request.headers),
                query_params=dict(request.query_params),
                raw_body=raw_body[:MAX_PAYLOAD_PREVIEW_BYTES],
                content_type=request.headers.get("Content-Type"),
                payload_size=len(body_bytes),
                status="received",
                data_format="json",
                integration_id=str(sub.id),
                correlation_id=request.headers.get("X-Correlation-Id"),
            )
            try:
                session.add(inbound)
                await session.flush()
            except Exception as exc:
                if _is_db_capacity_error(exc):
                    _mark_db_overloaded()
                    raise _service_overloaded_exception(
                        "Database temporarily overloaded; retry shortly"
                    ) from exc
                raise HTTPException(status_code=500, detail="Failed to persist webhook event") from exc

            try:
                inbound.payload = payload_for_storage
                inbound.parsed_data = {"records": records}
                inbound.record_count = len(records)
                inbound.status = "queued"
                inbound.should_retry = True
                inbound.error = None
                inbound.error_code = None
                inbound.response_status_code = 202
                inbound.response_body = {"ok": True, "queued": True}
                inbound.processed_at = datetime.utcnow()
                inbound.processing_time_ms = max(
                    0,
                    int((inbound.processed_at - started_at).total_seconds() * 1000),
                )

                await session.commit()

                try:
                    rq_job_id = await asyncio.to_thread(
                        _enqueue_sheet_webhook_delivery,
                        str(inbound.id),
                    )
                except Exception as exc:
                    logger.exception("Failed to enqueue sheet webhook delivery %s", inbound.id)
                    await _safe_mark_failed_delivery(
                        session,
                        sub,
                        inbound,
                        error_message=f"Failed to enqueue webhook delivery: {str(exc)[:200]}",
                        error_code="QUEUE_ENQUEUE_FAILED",
                        status_code=503,
                        payload_for_storage=payload_for_storage,
                        started_at=started_at,
                    )
                    raise _service_overloaded_exception("Webhook queue unavailable") from exc

                return JSONResponse(
                    status_code=202,
                    content={
                        "ok": True,
                        "queued": True,
                        "delivery_id": str(inbound.id),
                        "job_id": rq_job_id,
                    },
                )

            except HTTPException as exc:
                await _safe_mark_failed_delivery(
                    session,
                    sub,
                    inbound,
                    error_message=str(exc.detail),
                    error_code="INGEST_HTTP_ERROR",
                    status_code=exc.status_code,
                    payload_for_storage=payload_for_storage,
                    started_at=started_at,
                )
                raise
            except Exception as exc:  # pragma: no cover
                if _is_db_capacity_error(exc):
                    _mark_db_overloaded()
                    await _safe_mark_failed_delivery(
                        session,
                        sub,
                        inbound,
                        error_message="Database temporarily overloaded; retry shortly",
                        error_code="DB_CAPACITY",
                        status_code=503,
                        payload_for_storage=payload_for_storage,
                        started_at=started_at,
                    )
                    raise _service_overloaded_exception(
                        "Database temporarily overloaded; retry shortly"
                    ) from exc

                logger.exception("Sheet webhook ingest queueing failed")
                await _safe_mark_failed_delivery(
                    session,
                    sub,
                    inbound,
                    error_message=str(exc),
                    error_code="INGEST_INTERNAL_ERROR",
                    status_code=500,
                    payload_for_storage=payload_for_storage,
                    started_at=started_at,
                )
                raise HTTPException(status_code=500, detail="Webhook ingest failed") from exc
    finally:
        _sheet_webhook_ingest_semaphore.release()
