"""SQLite-backed index for fast receipt lookups and integrity tooling."""
from __future__ import annotations

import asyncio
import contextlib
import json
import logging
import math
import os
import random
import sqlite3
import threading
import time
from asyncio import LifoQueue, QueueEmpty
from dataclasses import dataclass
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, TypeVar

try:  # pragma: no cover - optional dependency
    import aiosqlite  # type: ignore
except Exception:  # pragma: no cover - fall back to thread executor path
    aiosqlite = None

from config.paths import DATA_ROOT

_RETENTION_CONFIG_DEFAULT = Path(__file__).resolve().parents[1] / "config" / "retention.yaml"
_RETENTION_POLICY_CACHE: Optional[Any] = None
from receipts.partitioning import partition_slug

__all__ = [
    "ReceiptIndexEntry",
    "VerifyReport",
    "ReconcileReport",
    "upsert_receipt_index",
    "lookup_receipt",
    "lookup_receipt_async",
    "resolve_receipt_path",
    "load_receipt_from_index",
    "rebuild_index",
    "apply_index_migrations",
    "index_stats",
    "index_health_summary",
    "verify_index",
    "reconcile_index",
    "backup_index",
    "vacuum_index",
    "load_cohort_ttls",
    "persist_cohort_ttl",
    "delete_cohort_ttl",
    "prune_cohort_ttls",
    "record_tombstone",
    "lookup_tombstone",
    "get_receipt_ids_by_type",
    "get_receipt_ids_by_type_window",
    "IndexMigrationSummary",
    "batch_upsert_receipt_vectors",
    "search_receipt_vectors",
]

_DB_FILENAME = "receipts_index.sqlite3"
_INDEX_DIRNAME = "indexes"


def _get_agent_namespace() -> str:
    """Return the agent namespace from environment (sonnet|codex)."""
    return os.getenv("AGENT_NS", "")
_PRAGMA_STATEMENTS: Sequence[str] = (
    "PRAGMA journal_mode=WAL;",
    "PRAGMA synchronous=NORMAL;",
    "PRAGMA temp_store=MEMORY;",
    "PRAGMA foreign_keys=ON;",
)
_BUSY_ERROR = "database is locked"
_MAX_BUSY_ATTEMPTS = 5
_BUSY_SLEEP_MIN = 0.005
_BUSY_SLEEP_MAX = 0.05
_LOCK = threading.RLock()
logger = logging.getLogger("loom.receipts.index")


def _env_flag_enabled(value: Optional[str]) -> bool:
    if value is None:
        return True
    token = value.strip().lower()
    return token not in {"0", "false", "off", "no"}


def _should_use_aiosqlite() -> bool:
    if aiosqlite is None:
        return False
    return _env_flag_enabled(os.getenv("AIOSQLITE"))


def _resolve_pool_size() -> int:
    token = os.getenv("LOOM_RECEIPT_INDEX_POOL_SIZE") or os.getenv("LOOM_TELEMETRY_POOL_SIZE")
    try:
        value = int(str(token)) if token is not None else 4
    except (TypeError, ValueError):
        value = 4
    return max(1, min(value, 32))


class _AsyncConnectionPool:
    """Simple LIFO connection pool for aiosqlite connections."""

    def __init__(self, *, root: Optional[Path], maxsize: int) -> None:
        self._root = Path(root) if root is not None else None
        self._maxsize = max(1, maxsize)
        self._queue: LifoQueue[Any] = LifoQueue(maxsize=self._maxsize)
        self._created = 0
        self._lock = asyncio.Lock()

    async def acquire(self) -> Any:
        try:
            return self._queue.get_nowait()
        except QueueEmpty:
            async with self._lock:
                if self._created < self._maxsize:
                    conn = await _connect_async(root=self._root)
                    self._created += 1
                    return conn
        return await self._queue.get()

    async def release(self, conn: Any, *, discard: bool = False) -> None:
        if conn is None:
            return
        if discard:
            with contextlib.suppress(Exception):
                await conn.close()
            async with self._lock:
                self._created = max(0, self._created - 1)
            return
        await self._queue.put(conn)


_POOL_REGISTRY: Dict[Tuple[str, int], _AsyncConnectionPool] = {}
_POOL_REGISTRY_LOCK = threading.Lock()


def _async_pool_for(root: Optional[Path]) -> _AsyncConnectionPool:
    if aiosqlite is None:
        raise RuntimeError("aiosqlite is not available")
    loop = asyncio.get_running_loop()
    base = Path(root or DATA_ROOT).resolve()
    key = (str(base), id(loop))
    pool = _POOL_REGISTRY.get(key)
    if pool is None:
        with _POOL_REGISTRY_LOCK:
            pool = _POOL_REGISTRY.get(key)
            if pool is None:
                pool_root = base if root is not None else None
                pool = _AsyncConnectionPool(root=pool_root, maxsize=_resolve_pool_size())
                _POOL_REGISTRY[key] = pool
    return pool


@dataclass(frozen=True)
class ReceiptIndexEntry:
    """Typed representation of a row in the receipt index."""

    receipt_id: str
    ts: int
    receipt_type: str
    jcs_sha256: str
    path: Path


@dataclass(frozen=True)
class VerifyReport:
    """Summary of index consistency relative to the ledger."""

    total_indexed: int
    total_ledger: int
    missing_receipts: List[str]
    orphan_receipts: List[str]
    hash_mismatches: Mapping[str, Tuple[str, str]]

    @property
    def has_drift(self) -> bool:
        return bool(self.missing_receipts or self.orphan_receipts or self.hash_mismatches)


@dataclass(frozen=True)
class ReconcileReport:
    """Result of an index reconciliation attempt."""

    before: VerifyReport
    after: VerifyReport
    inserted: int
    updated: int
    deleted: int
    dry_run: bool

    @property
    def repaired(self) -> bool:
        return not self.dry_run and self.before.has_drift and not self.after.has_drift

    @property
    def has_drift(self) -> bool:
        return self.after.has_drift


@dataclass(frozen=True)
class IndexMigrationSummary:
    """Summary of index migrations applied to the receipt index."""

    version: str
    statements_applied: List[str]
    result: str
    started_at: datetime
    finished_at: datetime
    receipt_id: Optional[str] = None


T = TypeVar("T")


def _base_dir(root: Optional[Path] = None) -> Path:
    base = Path(root or DATA_ROOT)
    directory = base / "receipts" / _INDEX_DIRNAME
    directory.mkdir(parents=True, exist_ok=True)
    return directory


def _db_path(root: Optional[Path] = None, ns: Optional[str] = None) -> Path:
    """Resolve database path with optional namespace prefix.
    
    If AGENT_NS is set (or ns param provided), uses receipts_{ns}.sqlite for isolation.
    """
    namespace = ns or _get_agent_namespace()
    if namespace:
        filename = f"receipts_{namespace}.sqlite"
    else:
        filename = _DB_FILENAME
    return _base_dir(root=root) / filename


def get_receipt_ids_by_type(
    receipt_type: str,
    limit: int = 10000,
    *,
    root: Optional[Path] = None,
) -> List[str]:
    """
    Fetches all receipt IDs of a specific type from the index.

    Args:
        receipt_type: The type of receipt to search for (e.g., "CompostStone").
        limit: A safety limit on the number of IDs to return.
        root: Override DATA_ROOT when querying from alternate ledgers.

    Returns:
        A list of receipt IDs.
    """
    db_path = _db_path(root=root)
    if not db_path.exists():
        logger.warning(f"Receipt index database not found at {db_path}")
        return []

    # Use a retry loop for database locks
    for attempt in range(_MAX_BUSY_ATTEMPTS):
        try:
            with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=0.5) as conn:
                cursor = conn.cursor()
                cursor.execute(
                    """
                    SELECT receipt_id FROM receipts_index
                    WHERE type = ?
                    ORDER BY ts DESC
                    LIMIT ?
                    """,
                    (receipt_type, limit),
                )
                results = cursor.fetchall()
                return [row[0] for row in results]
        except sqlite3.OperationalError as e:
            if _BUSY_ERROR in str(e).lower():
                sleep_time = random.uniform(_BUSY_SLEEP_MIN, _BUSY_SLEEP_MAX) * (2**attempt)
                logger.warning(f"Database is locked, retrying in {sleep_time:.3f}s...")
                time.sleep(sleep_time)
                continue
            raise
    logger.error(f"Failed to query receipt index after {_MAX_BUSY_ATTEMPTS} attempts.")
    return []


def get_receipt_ids_by_type_window(
    receipt_type: str,
    *,
    since: Optional[datetime | str | int | float] = None,
    until: Optional[datetime | str | int | float] = None,
    limit: int = 10000,
    root: Optional[Path] = None,
) -> List[str]:
    """Fetch receipt IDs constrained to an optional time window."""

    db_path = _db_path(root=root)
    if not db_path.exists():
        logger.warning(f"Receipt index database not found at {db_path}")
        return []

    since_epoch = _ensure_epoch(since) if since is not None else None
    until_epoch = _ensure_epoch(until) if until is not None else None

    clauses = ["type = ?"]
    params: List[Any] = [receipt_type]
    if since_epoch is not None:
        clauses.append("ts >= ?")
        params.append(since_epoch)
    if until_epoch is not None:
        clauses.append("ts <= ?")
        params.append(until_epoch)
    params.append(limit)

    query = (
        "SELECT receipt_id FROM receipts_index "
        f"WHERE {' AND '.join(clauses)} "
        "ORDER BY ts DESC "
        "LIMIT ?"
    )

    for attempt in range(_MAX_BUSY_ATTEMPTS):
        try:
            with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=0.5) as conn:
                cursor = conn.cursor()
                cursor.execute(query, params)
                return [row[0] for row in cursor.fetchall()]
        except sqlite3.OperationalError as e:
            if _BUSY_ERROR in str(e).lower():
                sleep_time = random.uniform(_BUSY_SLEEP_MIN, _BUSY_SLEEP_MAX) * (2**attempt)
                logger.warning(f"Database is locked, retrying in {sleep_time:.3f}s...")
                time.sleep(sleep_time)
                continue
            raise
    logger.error(f"Failed to query receipt index window after {_MAX_BUSY_ATTEMPTS} attempts.")
    return []


def _connect(root: Optional[Path] = None, *, run_migrations: bool = True) -> sqlite3.Connection:
    """Return an initialised SQLite connection."""
    db_path = _db_path(root=root)
    conn = sqlite3.connect(
        str(db_path),
        timeout=0.25,
        isolation_level=None,  # autocommit for robustness
        detect_types=sqlite3.PARSE_DECLTYPES,
    )
    conn.row_factory = sqlite3.Row
    for pragma in _PRAGMA_STATEMENTS:
        conn.execute(pragma)
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS receipts_index (
          receipt_id TEXT PRIMARY KEY,
          ts INTEGER NOT NULL,
          type TEXT NOT NULL,
          jcs_sha256 TEXT NOT NULL,
          path TEXT NOT NULL,
          decision TEXT,
          action TEXT,
          resource TEXT,
          organ TEXT
        )
        """
    )
    conn.execute("CREATE INDEX IF NOT EXISTS idx_ts ON receipts_index(ts)")
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS cohort_ttls (
          cohort_id TEXT PRIMARY KEY,
          expires_at INTEGER NOT NULL
        )
        """
    )
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS receipt_tombstones (
          receipt_id TEXT PRIMARY KEY,
          tombstone_receipt_id TEXT NOT NULL,
          reason TEXT,
          recorded_at INTEGER NOT NULL
        )
        """
    )
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS receipt_vectors (
          receipt_id TEXT PRIMARY KEY,
          type TEXT NOT NULL,
          domain TEXT,
          asset_id TEXT,
          summary TEXT,
          path TEXT NOT NULL,
          embedding TEXT NOT NULL,
          embedding_dim INTEGER NOT NULL,
          embedding_backend TEXT DEFAULT 'hash',
          updated_at INTEGER NOT NULL
        )
        """
    )
    conn.execute(
        """
        CREATE INDEX IF NOT EXISTS idx_receipt_vectors_domain_type
        ON receipt_vectors(domain, type)
        """
    )
    if run_migrations:
        _apply_index_migrations(conn, root=root)
    return conn


async def _connect_async(root: Optional[Path] = None):
    if aiosqlite is None:  # pragma: no cover - exercised only when aiosqlite installed
        raise RuntimeError("aiosqlite is not available")
    db_path = _db_path(root=root)
    conn = await aiosqlite.connect(
        str(db_path),
        timeout=0.25,
        isolation_level=None,
        detect_types=sqlite3.PARSE_DECLTYPES,
    )
    conn.row_factory = aiosqlite.Row  # type: ignore[attr-defined]
    for pragma in _PRAGMA_STATEMENTS:
        await conn.execute(pragma)
    await conn.execute(
        """
        CREATE TABLE IF NOT EXISTS receipts_index (
          receipt_id TEXT PRIMARY KEY,
          ts INTEGER NOT NULL,
          type TEXT NOT NULL,
          jcs_sha256 TEXT NOT NULL,
          path TEXT NOT NULL,
          decision TEXT,
          action TEXT,
          resource TEXT,
          organ TEXT
        )
        """
    )
    await conn.execute("CREATE INDEX IF NOT EXISTS idx_ts ON receipts_index(ts)")
    await conn.execute(
        """
        CREATE TABLE IF NOT EXISTS cohort_ttls (
          cohort_id TEXT PRIMARY KEY,
          expires_at INTEGER NOT NULL
        )
        """
    )
    await conn.execute(
        """
        CREATE TABLE IF NOT EXISTS receipt_tombstones (
          receipt_id TEXT PRIMARY KEY,
          tombstone_receipt_id TEXT NOT NULL,
          reason TEXT,
          recorded_at INTEGER NOT NULL
        )
        """
    )
    await conn.execute(
        """
        CREATE TABLE IF NOT EXISTS receipt_vectors (
          receipt_id TEXT PRIMARY KEY,
          type TEXT NOT NULL,
          domain TEXT,
          asset_id TEXT,
          summary TEXT,
          path TEXT NOT NULL,
          embedding TEXT NOT NULL,
          embedding_dim INTEGER NOT NULL,
          embedding_backend TEXT DEFAULT 'hash',
          updated_at INTEGER NOT NULL
        )
        """
    )
    await conn.execute(
        """
        CREATE INDEX IF NOT EXISTS idx_receipt_vectors_domain_type
        ON receipt_vectors(domain, type)
        """
    )
    return conn


def _migration_idx_ts_type(conn: sqlite3.Connection) -> List[str]:
    cursor = conn.execute("PRAGMA index_list('receipts_index')")
    existing = {row["name"] for row in cursor.fetchall()}
    if "idx_ts_type" in existing:
        return []
    statement = "CREATE INDEX IF NOT EXISTS idx_ts_type ON receipts_index(ts, type)"
    conn.execute(statement)
    return [statement]


def _add_column_if_missing(
    conn: sqlite3.Connection,
    table: str,
    column: str,
    alter_statement: str,
) -> List[str]:
    cursor = conn.execute(f"PRAGMA table_info('{table}')")
    existing_cols = {row["name"] for row in cursor.fetchall()}
    if column in existing_cols:
        return []
    conn.execute(alter_statement)
    return [alter_statement]


_INDEX_MIGRATIONS: Sequence[Tuple[str, Callable[[sqlite3.Connection], List[str]]]] = (
    ("P-OBS-003", _migration_idx_ts_type),
    (
        "P-OBS-004",
        lambda conn: _add_column_if_missing(conn, "receipts_index", "resource", "ALTER TABLE receipts_index ADD COLUMN resource TEXT"),
    ),
    (
        "P-OBS-005",
        lambda conn: _add_column_if_missing(conn, "receipts_index", "decision", "ALTER TABLE receipts_index ADD COLUMN decision TEXT"),
    ),
    (
        "P-OBS-006",
        lambda conn: _add_column_if_missing(conn, "receipts_index", "action", "ALTER TABLE receipts_index ADD COLUMN action TEXT"),
    ),
    (
        "P-OBS-007",
        lambda conn: _add_column_if_missing(conn, "receipts_index", "organ", "ALTER TABLE receipts_index ADD COLUMN organ TEXT"),
    ),
)


def _persist_index_migration_receipt(
    *,
    root: Optional[Path],
    migration_id: str,
    statements: Sequence[str],
    started_at: datetime,
    finished_at: datetime,
    result: str,
) -> Optional[str]:
    if not statements:
        return None
    try:
        from receipts.models import IndexMigrationReceipt  # noqa: WPS433 - runtime import avoids cycles
        from receipts.writer import write_receipt  # noqa: WPS433 - runtime import avoids cycles
    except Exception:  # pragma: no cover - import errors surface in logs
        logger.exception("Unable to import receipt machinery for index migration emission")
        return None

    receipt_id = f"index-migration-{migration_id}-{int(finished_at.timestamp() * 1_000_000)}"
    try:
        receipt = IndexMigrationReceipt(
            receipt_id=receipt_id,
            migration_id=migration_id,
            sql_applied=list(statements),
            ts_started=started_at,
            ts_finished=finished_at,
            result=result,
        )
        write_receipt(receipt, root=root, _skip_recovery=True)
    except Exception:  # pragma: no cover - failure recorded but migration already applied
        logger.exception(
            "Failed to persist IndexMigrationReceipt",
            extra={"migration_id": migration_id, "receipt_id": receipt_id},
        )
        return None
    return receipt_id


def _apply_index_migrations(
    conn: sqlite3.Connection,
    *,
    root: Optional[Path],
) -> List[IndexMigrationSummary]:
    """Apply pending index migrations with optional council gating."""
    # --- Council gate for index migrations ---
    try:
        from organism.system_modification_gate import gate_system_modification_sync
        
        pending_migrations = [v for v, _ in _INDEX_MIGRATIONS]
        change_spec = {
            "type": "index_migration",
            "target": "receipts_index",
            "pending_versions": pending_migrations,
            "root": str(root) if root else None,
        }
        
        allowed, decision = gate_system_modification_sync(change_spec)
        
        if not allowed:
            reason = decision.reason if decision else "Council denied index migration"
            logger.warning(f"Index migration blocked by council gate: {reason}")
            return []  # Return empty; don't raise to avoid breaking receipt writes
            
    except ImportError:
        # Gate not available; proceed without council check
        pass
    except Exception as exc:
        # Log but don't block on gate errors (backward compatible)
        logger.warning(f"Council gate check failed for index migration (proceeding): {exc}")
    # --- End council gate ---

    applied: List[IndexMigrationSummary] = []
    for version, migration in _INDEX_MIGRATIONS:
        started_at = datetime.now(timezone.utc)
        statements = migration(conn)
        finished_at = datetime.now(timezone.utc)
        if not statements:
            continue
        receipt_id = _persist_index_migration_receipt(
            root=root,
            migration_id=version,
            statements=statements,
            started_at=started_at,
            finished_at=finished_at,
            result="applied",
        )
        applied.append(
            IndexMigrationSummary(
                version=version,
                statements_applied=list(statements),
                result="applied",
                started_at=started_at,
                finished_at=finished_at,
                receipt_id=receipt_id,
            )
        )
    return applied


def _write_with_retry(
    operation: Callable[[sqlite3.Connection], T],
    *,
    root: Optional[Path] = None,
    attempts: int = _MAX_BUSY_ATTEMPTS,
    run_migrations: bool = True,
) -> T:
    """
    Execute ``operation`` with retry handling for SQLITE_BUSY contention.
    """
    attempts = max(1, min(attempts, _MAX_BUSY_ATTEMPTS))
    last_error: Optional[sqlite3.OperationalError] = None
    for attempt in range(1, attempts + 1):
        should_retry = False
        with _LOCK:
            conn = _connect(root=root, run_migrations=run_migrations)
            try:
                return operation(conn)
            except sqlite3.OperationalError as exc:
                last_error = exc
                should_retry = _BUSY_ERROR in str(exc).lower() and attempt < attempts
                if not should_retry:
                    raise
            finally:
                conn.close()
        if should_retry:
            time.sleep(random.uniform(_BUSY_SLEEP_MIN, _BUSY_SLEEP_MAX))
    if last_error is not None:
        raise last_error
    raise RuntimeError("SQLite write retry loop exhausted without error")  # pragma: no cover


def apply_index_migrations(*, root: Optional[Path] = None) -> List[IndexMigrationSummary]:
    """
    Run pending index migrations and return summaries of applied statements.
    """

    def _op(conn: sqlite3.Connection) -> List[IndexMigrationSummary]:
        return _apply_index_migrations(conn, root=root)

    return _write_with_retry(_op, root=root, run_migrations=False)


def _ensure_epoch(value: datetime | str | int | float) -> int:
    """Convert heterogeneous timestamp inputs into microseconds since epoch."""
    if isinstance(value, int):
        return value
    if isinstance(value, float):
        return int(value)
    if isinstance(value, datetime):
        ts = value if value.tzinfo else value.replace(tzinfo=timezone.utc)
        return int(ts.timestamp() * 1_000_000)
    if isinstance(value, str):
        candidate = value.strip()
        if candidate.endswith("Z"):
            candidate = candidate[:-1] + "+00:00"
        try:
            parsed = datetime.fromisoformat(candidate)
        except ValueError:
            raise ValueError(f"Unable to parse timestamp '{value}'") from None
        return _ensure_epoch(parsed)
    raise TypeError(f"Unsupported timestamp type {type(value)!r}")


def _partition_date(value: datetime | str | int | float) -> str:
    """Normalise timestamps to YYYY-MM-DD partitions."""
    if isinstance(value, datetime):
        candidate = value.astimezone(timezone.utc)
        return candidate.date().isoformat()
    if isinstance(value, (int, float)):
        seconds = float(value) / 1_000_000 if isinstance(value, int) and value > 10_000_000_000 else float(value)
        return datetime.fromtimestamp(seconds, tz=timezone.utc).date().isoformat()
    if isinstance(value, str):
        token = value.strip()
        if token.endswith("Z"):
            token = token[:-1] + "+00:00"
        try:
            parsed = datetime.fromisoformat(token)
        except ValueError:
            return token[:10]
        return parsed.astimezone(timezone.utc).date().isoformat()
    return datetime.now(timezone.utc).date().isoformat()


def _expected_json_relative_path(
    *,
    receipt_id: str,
    receipt_hash: Optional[str],
    tenant: Optional[str],
    domain: Optional[str],
    ts: datetime | str | int | float,
    root: Optional[Path],
) -> str:
    tenant_slug = partition_slug(tenant or "default")
    domain_slug = partition_slug(domain or "default")
    partition = _partition_date(ts)
    filename = f"{receipt_hash or receipt_id}.json"
    rel_path = Path("receipts") / "inbox" / tenant_slug / domain_slug / partition / filename
    return _normalise_path(Path(root or DATA_ROOT) / rel_path, root=root)


def upsert_receipt_index(
    *,
    receipt_id: str,
    ts: datetime | str | int | float,
    receipt_type: str,
    jcs_sha256: str,
    path: Path | str,
    root: Optional[Path] = None,
    retries: int = _MAX_BUSY_ATTEMPTS,
    retry_delay: float = 0.05,
    tombstone_for: Optional[str] = None,
    tombstone_reason: Optional[str] = None,
    decision: Optional[str] = None,
    action: Optional[str] = None,
    resource: Optional[str] = None,
    organ: Optional[str] = None,
) -> None:
    """
    Insert or update a receipt record in the SQLite index.

    Retries on SQLITE_BUSY to prevent transient lock contention from failing
    the write path. Raises the last OperationalError on persistent failures.
    """
    _ = retry_delay  # Backwards compatibility: jitter handled by _write_with_retry
    epoch_value = _ensure_epoch(ts)
    relative_path = _normalise_path(path, root=root)

    attempts = max(1, min(int(retries), _MAX_BUSY_ATTEMPTS))

    def _operation(conn: sqlite3.Connection) -> None:
        conn.execute(
            """
            INSERT INTO receipts_index(receipt_id, ts, type, jcs_sha256, path, decision, action, resource, organ)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(receipt_id) DO UPDATE SET
                ts = excluded.ts,
                type = excluded.type,
                jcs_sha256 = excluded.jcs_sha256,
                path = excluded.path,
                decision = excluded.decision,
                action = excluded.action,
                resource = excluded.resource,
                organ = excluded.organ
            """,
            (
                receipt_id,
                epoch_value,
                receipt_type,
                jcs_sha256,
                relative_path,
                decision,
                action,
                resource,
                organ,
            ),
        )
        if tombstone_for:
            conn.execute(
                """
                INSERT INTO receipt_tombstones(receipt_id, tombstone_receipt_id, reason, recorded_at)
                VALUES (?, ?, ?, ?)
                ON CONFLICT(receipt_id) DO UPDATE SET
                    tombstone_receipt_id = excluded.tombstone_receipt_id,
                    reason = excluded.reason,
                    recorded_at = excluded.recorded_at
                """,
                (
                    tombstone_for,
                    receipt_id,
                    tombstone_reason,
                    epoch_value,
                ),
            )

    _write_with_retry(_operation, root=root, attempts=attempts)


def lookup_receipt(receipt_id: str, *, root: Optional[Path] = None) -> Optional[ReceiptIndexEntry]:
    """Return the index entry for ``receipt_id`` if present."""
    with _LOCK:
        conn = _connect(root=root)
        try:
            row = conn.execute(
                "SELECT receipt_id, ts, type, jcs_sha256, path FROM receipts_index WHERE receipt_id = ?",
                (receipt_id,),
            ).fetchone()
        finally:
            conn.close()
    if row is None:
        return None
    return ReceiptIndexEntry(
        receipt_id=row["receipt_id"],
        ts=int(row["ts"]),
        receipt_type=row["type"],
        jcs_sha256=row["jcs_sha256"],
        path=_materialise_path(row["path"], root=root),
    )


async def lookup_receipt_async(
    receipt_id: str,
    *,
    root: Optional[Path] = None,
) -> Optional[ReceiptIndexEntry]:
    """Async variant of ``lookup_receipt`` using aiosqlite when available."""
    if not _should_use_aiosqlite():
        loop = asyncio.get_running_loop()
        return await loop.run_in_executor(
            None,
            lambda: lookup_receipt(receipt_id, root=root),  # pragma: no cover - executed when fallback path used
        )

    pool = _async_pool_for(root)
    conn = await pool.acquire()
    cursor = None
    row: Optional[Mapping[str, Any]] = None
    discard = False
    try:
        cursor = await conn.execute(
            "SELECT receipt_id, ts, type, jcs_sha256, path FROM receipts_index WHERE receipt_id = ?",
            (receipt_id,),
        )
        row = await cursor.fetchone()
    except Exception:
        discard = True
        raise
    finally:
        if cursor is not None:
            with contextlib.suppress(Exception):
                await cursor.close()
        await pool.release(conn, discard=discard)

    if row is None:
        return None
    return ReceiptIndexEntry(
        receipt_id=row["receipt_id"],
        ts=int(row["ts"]),
        receipt_type=row["type"],
        jcs_sha256=row["jcs_sha256"],
        path=_materialise_path(row["path"], root=root),
    )


def resolve_receipt_path(receipt_id: str, *, root: Optional[Path] = None) -> Path:
    """Resolve the on-disk JSON path for ``receipt_id`` using the index."""
    entry = lookup_receipt(receipt_id, root=root)
    if entry is None:
        raise FileNotFoundError(f"receipt '{receipt_id}' not indexed")
    return entry.path


def load_receipt_from_index(receipt_id: str, *, root: Optional[Path] = None) -> dict:
    """
    Load and parse the receipt JSON using the index. Raises FileNotFoundError if
    the receipt is neither indexed nor present on disk.
    """
    path = resolve_receipt_path(receipt_id, root=root)
    try:
        text = path.read_text(encoding="utf-8")
    except FileNotFoundError:
        raise
    return json.loads(text)


def _ledger_root(root: Optional[Path]) -> Path:
    return Path(root or DATA_ROOT) / "receipts" / "ledger"


def _iter_ledger_entries(*, root: Optional[Path] = None, limit: Optional[int] = None) -> Iterable[Dict[str, Any]]:
    """Yield ledger rows as dictionaries."""
    import pandas as pd  # Imported lazily to avoid mandatory dependency at import time

    ledger_dir = _ledger_root(root)
    if not ledger_dir.exists():
        return

    count = 0
    parts = sorted(ledger_dir.glob("**/*.parquet"))
    for part in parts:
        try:
            df = pd.read_parquet(
                part,
                columns=["receipt_id", "ts", "receipt_type", "receipt_hash", "tenant", "domain"],
            )
        except Exception:
            logger.exception("Failed to read ledger fragment %s", part)
            continue
        for record in df.to_dict(orient="records"):
            receipt_id = str(record.get("receipt_id") or "")
            if not receipt_id:
                continue
            yield {
                "receipt_id": receipt_id,
                "ts": record.get("ts"),
                "receipt_type": record.get("receipt_type") or record.get("type") or "UnknownReceipt",
                "receipt_hash": record.get("receipt_hash") or record.get("jcs_sha256") or "",
                "tenant": record.get("tenant"),
                "domain": record.get("domain"),
            }
            count += 1
            if limit is not None and count >= limit:
                return


def rebuild_index(*, root: Optional[Path] = None) -> int:
    """
    Rebuild the index by scanning receipt inbox directories.

    Returns:
        Number of indexed receipts.
    """
    base = Path(root or DATA_ROOT)
    inbox_root = base / "receipts" / "inbox"
    if not inbox_root.exists():
        return 0

    def _operation(conn: sqlite3.Connection) -> int:
        local_count = 0
        try:
            conn.execute("BEGIN IMMEDIATE;")
            conn.execute("DELETE FROM receipts_index;")
            for json_path in inbox_root.glob("**/*.json"):
                receipt_id = json_path.stem
                payload = json.loads(json_path.read_text(encoding="utf-8"))
                ts_value = payload.get("ts") or payload.get("timestamp") or payload.get("evaluated_at")
                if ts_value is None:
                    ts_value = datetime.now(timezone.utc)
                receipt_type = payload.get("receipt_type") or payload.get("type") or "UnknownReceipt"
                receipt_hash = payload.get("receipt_hash") or payload.get("jcs_sha256") or ""
                relative_path = _normalise_path(json_path, root=root)
                conn.execute(
                    """
                    INSERT INTO receipts_index(receipt_id, ts, type, jcs_sha256, path)
                    VALUES (?, ?, ?, ?, ?)
                    ON CONFLICT(receipt_id) DO UPDATE SET
                        ts = excluded.ts,
                        type = excluded.type,
                        jcs_sha256 = excluded.jcs_sha256,
                        path = excluded.path
                    """,
                    (
                        receipt_id,
                        _ensure_epoch(ts_value),
                        receipt_type,
                        receipt_hash,
                        relative_path,
                    ),
                )
                local_count += 1
            conn.execute("COMMIT;")
            return local_count
        except Exception:
            with contextlib.suppress(sqlite3.OperationalError):
                conn.execute("ROLLBACK;")
            raise

    return _write_with_retry(_operation, root=root)


def _normalise_path(path: Path | str, *, root: Optional[Path] = None) -> str:
    candidate = Path(path)
    try:
        base = Path(root or DATA_ROOT)
        return str(candidate.relative_to(base))
    except ValueError:
        return str(candidate)


def _materialise_path(rel_path: str, *, root: Optional[Path] = None) -> Path:
    candidate = Path(rel_path)
    if candidate.is_absolute():
        return candidate
    return Path(root or DATA_ROOT) / candidate


def _normalise_vector(vector: Sequence[float]) -> List[float]:
    """Normalise a raw embedding vector to unit length."""
    coerced: List[float] = []
    for component in vector:
        try:
            coerced.append(float(component))
        except (TypeError, ValueError):
            continue
    if not coerced:
        return []
    norm = math.sqrt(sum(value * value for value in coerced))
    if norm == 0.0:
        return coerced
    return [value / norm for value in coerced]


def _cosine_similarity(lhs: Sequence[float], rhs: Sequence[float]) -> float:
    """Compute cosine similarity between two vectors."""
    length = min(len(lhs), len(rhs))
    if length == 0:
        return 0.0
    dot = sum(lhs[i] * rhs[i] for i in range(length))
    lhs_norm = math.sqrt(sum(lhs[i] * lhs[i] for i in range(length)))
    rhs_norm = math.sqrt(sum(rhs[i] * rhs[i] for i in range(length)))
    if lhs_norm == 0.0 or rhs_norm == 0.0:
        return 0.0
    return dot / (lhs_norm * rhs_norm)


def batch_upsert_receipt_vectors(
    entries: Sequence[Mapping[str, Any]],
    *,
    root: Optional[Path] = None,
) -> int:
    """
    Persist a batch of receipt embeddings for semantic search.

    Args:
        entries: Iterable of mapping objects containing the keys
            ``receipt_id`` (str), ``vector`` (sequence of floats),
            ``type`` (receipt class name), ``domain`` (optional str),
            ``asset_id`` (optional str), ``summary`` (str),
            ``path`` (absolute or relative path) and optional
            ``embedding_backend`` (str, defaults to ``"hash"``).
        root: Optional data root override.

    Returns:
        Number of persisted embeddings.
    """

    if not entries:
        return 0

    def _operation(conn: sqlite3.Connection) -> int:
        now = int(time.time() * 1_000_000)
        payloads: List[Tuple[Any, ...]] = []
        for entry in entries:
            receipt_id = str(entry.get("receipt_id") or "")
            if not receipt_id:
                continue
            vector_raw = entry.get("vector")
            if not isinstance(vector_raw, Sequence):
                continue
            normalised = _normalise_vector(vector_raw)
            if not normalised:
                continue
            receipt_type = str(entry.get("type") or entry.get("receipt_type") or "UnknownReceipt")
            domain_val = entry.get("domain")
            domain = str(domain_val) if domain_val not in (None, "") else None
            asset_val = entry.get("asset_id")
            asset_id = str(asset_val) if asset_val not in (None, "") else None
            summary_text = str(entry.get("summary") or "")[:512]
            backend = str(entry.get("embedding_backend") or "hash")
            path_value = entry.get("path") or ""
            rel_path = _normalise_path(path_value, root=root)
            payloads.append(
                (
                    receipt_id,
                    receipt_type,
                    domain,
                    asset_id,
                    summary_text,
                    rel_path,
                    json.dumps(normalised),
                    len(normalised),
                    backend,
                    now,
                )
            )
        if not payloads:
            return 0
        conn.executemany(
            """
            INSERT INTO receipt_vectors (
              receipt_id,
              type,
              domain,
              asset_id,
              summary,
              path,
              embedding,
              embedding_dim,
              embedding_backend,
              updated_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(receipt_id) DO UPDATE SET
              type = excluded.type,
              domain = excluded.domain,
              asset_id = excluded.asset_id,
              summary = excluded.summary,
              path = excluded.path,
              embedding = excluded.embedding,
              embedding_dim = excluded.embedding_dim,
              embedding_backend = excluded.embedding_backend,
              updated_at = excluded.updated_at
            """,
            payloads,
        )
        return len(payloads)

    return _write_with_retry(_operation, root=root)


def search_receipt_vectors(
    query_vector: Sequence[float],
    *,
    root: Optional[Path] = None,
    domain: Optional[str] = None,
    receipt_types: Optional[Sequence[str]] = None,
    exclude_receipt_ids: Optional[Sequence[str]] = None,
    exclude_asset_ids: Optional[Sequence[str]] = None,
    limit: int = 5,
    min_score: float = 0.05,
) -> List[Dict[str, Any]]:
    """
    Perform cosine-similarity search over stored receipt embeddings.

    Args:
        query_vector: Pre-computed embedding for the query text.
        root: Optional data root override.
        domain: Optional domain filter.
        receipt_types: Optional iterable of receipt types to include.
        exclude_receipt_ids: Optional iterable of receipt IDs to skip.
        exclude_asset_ids: Optional iterable of asset IDs to skip.
        limit: Maximum number of matches to return (<=0 returns all).
        min_score: Minimum cosine similarity to keep.

    Returns:
        A list of dictionaries containing ``receipt_id``, ``type``,
        ``domain``, ``asset_id``, ``summary``, ``path`` and ``score``.
    """

    normalised_query = _normalise_vector(query_vector)
    if not normalised_query:
        return []

    filters: List[str] = []
    params: List[Any] = []
    if domain:
        filters.append("domain = ?")
        params.append(domain)
    if receipt_types:
        placeholders = ",".join("?" for _ in receipt_types)
        filters.append(f"type IN ({placeholders})")
        params.extend(receipt_types)

    sql = "SELECT receipt_id, type, domain, asset_id, summary, path, embedding FROM receipt_vectors"
    if filters:
        sql += " WHERE " + " AND ".join(filters)

    rows: List[sqlite3.Row] = []
    missing_table = False
    with _LOCK:
        conn = _connect(root=root)
        try:
            rows = conn.execute(sql, tuple(params)).fetchall()
        except sqlite3.OperationalError as exc:
            missing_table = "no such table" in str(exc).lower()
            if not missing_table:
                raise
        finally:
            conn.close()
    if missing_table:
        return []

    exclude_receipt_set = {str(value) for value in (exclude_receipt_ids or []) if value is not None}
    exclude_asset_set = {str(value) for value in (exclude_asset_ids or []) if value not in (None, "")}

    matches: List[Dict[str, Any]] = []
    for row in rows:
        receipt_id = row["receipt_id"]
        if receipt_id in exclude_receipt_set:
            continue
        asset_id = row["asset_id"]
        if asset_id is not None and str(asset_id) in exclude_asset_set:
            continue
        embedding_blob = row["embedding"]
        if not embedding_blob:
            continue
        try:
            vector = [float(component) for component in json.loads(embedding_blob)]
        except (TypeError, ValueError, json.JSONDecodeError):
            continue
        score = _cosine_similarity(normalised_query, vector)
        if score < min_score:
            continue
        matches.append(
            {
                "receipt_id": receipt_id,
                "type": row["type"],
                "domain": row["domain"],
                "asset_id": asset_id,
                "summary": row["summary"] or "",
                "path": row["path"],
                "score": float(score),
            }
        )

    matches.sort(key=lambda payload: payload["score"], reverse=True)
    if limit > 0:
        return matches[:limit]
    return matches


def _fetch_index_rows_by_type(
    types: Sequence[str],
    *,
    root: Optional[Path] = None,
) -> List[Tuple[str, Path]]:
    if not types:
        return []
    placeholders = ",".join("?" for _ in types)
    with _LOCK:
        conn = _connect(root=root)
        try:
            rows = conn.execute(
                f"SELECT receipt_id, path FROM receipts_index WHERE type IN ({placeholders})",
                tuple(types),
            ).fetchall()
        finally:
            conn.close()
    return [
        (row["receipt_id"], _materialise_path(row["path"], root=root))
        for row in rows
    ]


def _fetch_receipt_id_set(
    types: Sequence[str],
    *,
    root: Optional[Path] = None,
) -> Set[str]:
    if not types:
        return set()
    placeholders = ",".join("?" for _ in types)
    with _LOCK:
        conn = _connect(root=root)
        try:
            rows = conn.execute(
                f"SELECT receipt_id FROM receipts_index WHERE type IN ({placeholders})",
                tuple(types),
            ).fetchall()
        finally:
            conn.close()
    return {row["receipt_id"] for row in rows}


def _fetch_tombstone_rows(*, root: Optional[Path] = None) -> List[sqlite3.Row]:
    with _LOCK:
        conn = _connect(root=root)
        try:
            rows = conn.execute(
                "SELECT receipt_id, tombstone_receipt_id, recorded_at FROM receipt_tombstones"
            ).fetchall()
        finally:
            conn.close()
    return rows


def _last_index_migration_record(
    *,
    root: Optional[Path] = None,
) -> Optional[Dict[str, Any]]:
    with _LOCK:
        conn = _connect(root=root)
        try:
            row = conn.execute(
                """
                SELECT receipt_id, ts
                FROM receipts_index
                WHERE type = ?
                ORDER BY ts DESC
                LIMIT 1
                """,
                ("IndexMigrationReceipt",),
            ).fetchone()
        finally:
            conn.close()

    if row is None:
        return None

    receipt_id = row["receipt_id"]
    ts_value = int(row["ts"])
    payload: Optional[Mapping[str, Any]]
    try:
        loaded = load_receipt_from_index(receipt_id, root=root)
        payload = loaded if isinstance(loaded, Mapping) else None
    except FileNotFoundError:
        payload = None
    except json.JSONDecodeError:
        payload = None

    record: Dict[str, Any] = {
        "receipt_id": receipt_id,
        "ts": ts_value,
    }
    if payload is not None:
        record["migration_id"] = payload.get("migration_id") or payload.get("version")
        record["result"] = payload.get("result")
        record["ts_started"] = payload.get("ts_started")
        record["ts_finished"] = payload.get("ts_finished")
        record["sql_applied"] = payload.get("sql_applied")
    return record


def _load_retention_policy():  # type: ignore[return-annotation]
    global _RETENTION_POLICY_CACHE
    if _RETENTION_POLICY_CACHE is not None:
        return _RETENTION_POLICY_CACHE
    config_path = Path(os.getenv("LOOM_RETENTION_CONFIG", str(_RETENTION_CONFIG_DEFAULT)))
    path_obj = Path(config_path)
    if path_obj.exists():
        try:
            from tools.bin.retention_sweep import load_policy  # pylint: disable=import-outside-toplevel

            _RETENTION_POLICY_CACHE = load_policy(path_obj)
        except Exception:
            _RETENTION_POLICY_CACHE = None
    else:
        _RETENTION_POLICY_CACHE = None
    return _RETENTION_POLICY_CACHE


def _parse_retention_note(note: Optional[str]) -> Optional[int]:
    if not note or not isinstance(note, str):
        return None
    tokens = note.replace(",", " ").split()
    for token in tokens:
        if token.startswith("retention_ttl_days="):
            value = token.split("=", 1)[1]
            try:
                ttl = int(value)
            except (TypeError, ValueError):
                continue
            if ttl >= 0:
                return ttl
    return None


def _extract_tombstone_context(
    payload: Mapping[str, Any],
) -> Tuple[Optional[str], Optional[str], Optional[int]]:
    domain = str(payload.get("domain")) if payload.get("domain") is not None else None
    tenant = str(payload.get("tenant")) if payload.get("tenant") is not None else None
    note: Optional[str] = payload.get("note") if isinstance(payload.get("note"), str) else None

    context = payload.get("context")
    if isinstance(context, Mapping):
        tombstone_ctx = context.get("tombstone")
        if isinstance(tombstone_ctx, Mapping):
            domain = str(tombstone_ctx.get("domain")) if tombstone_ctx.get("domain") is not None else domain
            tenant = str(tombstone_ctx.get("tenant")) if tombstone_ctx.get("tenant") is not None else tenant
            note_candidate = tombstone_ctx.get("note")
            if isinstance(note_candidate, str):
                note = note_candidate
            ttl_candidate = tombstone_ctx.get("retention_ttl_days")
            if ttl_candidate is not None:
                try:
                    ttl_val = int(ttl_candidate)
                except (TypeError, ValueError):
                    ttl_val = None
                else:
                    if ttl_val >= 0:
                        return domain, tenant, ttl_val

    ttl_from_note = _parse_retention_note(note)
    return domain, tenant, ttl_from_note


def index_stats(*, root: Optional[Path] = None) -> dict:
    """
    Return index statistics for health monitoring.
    
    Returns:
        Dict with keys: total_receipts, oldest_ts, newest_ts, receipt_types
    """
    journal_mode = "unknown"
    with _LOCK:
        conn = _connect(root=root)
        try:
            row = conn.execute(
                "SELECT COUNT(*) as total, MIN(ts) as oldest, MAX(ts) as newest FROM receipts_index"
            ).fetchone()
            
            types_rows = conn.execute(
                "SELECT type, COUNT(*) as count FROM receipts_index GROUP BY type ORDER BY count DESC"
            ).fetchall()
            journal_row = conn.execute("PRAGMA journal_mode;").fetchone()
            if journal_row:
                mode = journal_row[0] if isinstance(journal_row, (list, tuple)) else journal_row
                journal_mode = str(mode).upper()
        finally:
            conn.close()

    db_path = _db_path(root=root)
    db_size = db_path.stat().st_size if db_path.exists() else 0
    wal_path = Path(f"{db_path}-wal")
    wal_size = wal_path.stat().st_size if wal_path.exists() else 0
    
    return {
        "total_receipts": int(row["total"]) if row else 0,
        "oldest_ts": int(row["oldest"]) if row and row["oldest"] else None,
        "newest_ts": int(row["newest"]) if row and row["newest"] else None,
        "receipt_types": {r["type"]: int(r["count"]) for r in types_rows},
        "journal_mode": journal_mode,
        "db_path": str(db_path),
        "db_size_bytes": db_size,
        "wal_size_bytes": wal_size,
    }


def index_health_summary(
    *,
    root: Optional[Path] = None,
    now: Optional[datetime] = None,
    near_window_days: int = 7,
) -> Dict[str, Any]:
    """Return summary health indicators for the receipt index."""

    stats = index_stats(root=root)
    total_receipts = int(stats.get("total_receipts", 0))
    newest_ts = stats.get("newest_ts")

    prophecy_ids = _fetch_receipt_id_set(["Prophecy", "ProphecyStone"], root=root)
    dip_ids = _fetch_receipt_id_set(["DIPReceipt", "DIP"], root=root)

    outcome_entries = _fetch_index_rows_by_type(["Outcome", "OutcomeReceipt"], root=root)
    prophecy_entries = _fetch_index_rows_by_type(["Prophecy", "ProphecyStone"], root=root)

    orphan_outcomes = 0
    for receipt_id, _ in outcome_entries:
        try:
            payload = load_receipt_from_index(receipt_id, root=root)
        except FileNotFoundError:
            orphan_outcomes += 1
            continue
        parent_id = payload.get("parent_prophecy_id") or payload.get("parent_id")
        if not parent_id:
            orphan_outcomes += 1
            continue
        parent_token = str(parent_id)
        if parent_token not in prophecy_ids:
            orphan_outcomes += 1

    orphan_prophecies = 0
    for receipt_id, _ in prophecy_entries:
        try:
            payload = load_receipt_from_index(receipt_id, root=root)
        except FileNotFoundError:
            orphan_prophecies += 1
            continue
        allow_orphan = bool(payload.get("allow_orphan"))
        parent_id = payload.get("parent_id")
        if parent_id:
            parent_token = str(parent_id)
            if parent_token not in dip_ids and not allow_orphan:
                orphan_prophecies += 1
        elif not allow_orphan:
            orphan_prophecies += 1

    tombstone_rows = _fetch_tombstone_rows(root=root)
    now_dt = now or datetime.now(timezone.utc)
    near_threshold = timedelta(days=max(int(near_window_days), 0))
    policy = _load_retention_policy()
    tombstones_near = 0
    tombstones_considered = 0

    for row in tombstone_rows:
        recorded_raw = row["recorded_at"]
        try:
            recorded_dt = datetime.fromtimestamp(int(recorded_raw) / 1_000_000, tz=timezone.utc)
        except (TypeError, ValueError, OverflowError):
            continue
        tombstone_id = row["tombstone_receipt_id"]
        tombstone_payload: Optional[Mapping[str, Any]]
        try:
            payload = load_receipt_from_index(tombstone_id, root=root)
            tombstone_payload = payload if isinstance(payload, Mapping) else None
        except FileNotFoundError:
            tombstone_payload = None

        domain = tenant = None
        ttl_days: Optional[int] = None
        if tombstone_payload is not None:
            domain, tenant, ttl_days = _extract_tombstone_context(tombstone_payload)
        if ttl_days is None and policy is not None:
            try:
                ttl_days = policy.ttl_for(tenant, domain)
            except Exception:
                ttl_days = None
        if ttl_days is None or ttl_days <= 0:
            continue

        tombstones_considered += 1
        expiry_dt = recorded_dt + timedelta(days=int(ttl_days))
        if expiry_dt >= now_dt and expiry_dt - now_dt <= near_threshold:
            tombstones_near += 1

    tombstones_pct = (
        (tombstones_near / tombstones_considered) * 100.0 if tombstones_considered else None
    )

    last_migration = _last_index_migration_record(root=root)

    return {
        "receipts": total_receipts,
        "total_receipts": total_receipts,
        "last_write_ts": newest_ts,
        "orphan_outcomes": orphan_outcomes,
        "orphan_outcome_count": orphan_outcomes,
        "orphan_prophecies": orphan_prophecies,
        "tombstones": len(tombstone_rows),
        "tombstones_total": len(tombstone_rows),
        "tombstones_considered": tombstones_considered,
        "tombstones_near_expiry": tombstones_near,
        "tombstones_near_expiry_pct": tombstones_pct,
        "last_migration": last_migration,
    }


def load_cohort_ttls(*, root: Optional[Path] = None) -> Dict[str, int]:
    """Return all persisted cohort TTL entries (microseconds)."""
    with _LOCK:
        conn = _connect(root=root)
        try:
            rows = conn.execute("SELECT cohort_id, expires_at FROM cohort_ttls").fetchall()
        finally:
            conn.close()
    return {row["cohort_id"]: int(row["expires_at"]) for row in rows}


def persist_cohort_ttl(
    cohort_id: str,
    expires_at: datetime | str | int | float,
    *,
    root: Optional[Path] = None,
) -> None:
    epoch = _ensure_epoch(expires_at)
    _write_with_retry(
        lambda conn: conn.execute(
            """
            INSERT INTO cohort_ttls(cohort_id, expires_at)
            VALUES (?, ?)
            ON CONFLICT(cohort_id) DO UPDATE SET expires_at = excluded.expires_at
            """,
            (cohort_id, epoch),
        ),
        root=root,
    )


def delete_cohort_ttl(cohort_id: str, *, root: Optional[Path] = None) -> None:
    _write_with_retry(
        lambda conn: conn.execute("DELETE FROM cohort_ttls WHERE cohort_id = ?", (cohort_id,)),
        root=root,
    )


def prune_cohort_ttls(
    reference: datetime | str | int | float,
    *,
    root: Optional[Path] = None,
) -> List[str]:
    threshold = _ensure_epoch(reference)
    return _write_with_retry(
        lambda conn: _delete_expired_cohorts(conn, threshold),
        root=root,
    )


def _delete_expired_cohorts(conn: sqlite3.Connection, threshold: int) -> List[str]:
    rows = conn.execute(
        "SELECT cohort_id FROM cohort_ttls WHERE expires_at <= ?",
        (threshold,),
    ).fetchall()
    conn.execute("DELETE FROM cohort_ttls WHERE expires_at <= ?", (threshold,))
    return [row["cohort_id"] for row in rows]


def record_tombstone(
    receipt_id: str,
    *,
    tombstone_receipt_id: str,
    reason: Optional[str],
    recorded_at: datetime | str | int | float,
    root: Optional[Path] = None,
) -> None:
    epoch = _ensure_epoch(recorded_at)
    _write_with_retry(
        lambda conn: conn.execute(
            """
            INSERT INTO receipt_tombstones(receipt_id, tombstone_receipt_id, reason, recorded_at)
            VALUES (?, ?, ?, ?)
            ON CONFLICT(receipt_id) DO UPDATE SET
                tombstone_receipt_id = excluded.tombstone_receipt_id,
                reason = excluded.reason,
                recorded_at = excluded.recorded_at
            """,
            (
                receipt_id,
                tombstone_receipt_id,
                reason,
                epoch,
            ),
        ),
        root=root,
    )


def lookup_tombstone(receipt_id: str, *, root: Optional[Path] = None) -> Optional[Dict[str, Any]]:
    with _LOCK:
        conn = _connect(root=root)
        try:
            row = conn.execute(
                "SELECT receipt_id, tombstone_receipt_id, reason, recorded_at FROM receipt_tombstones WHERE receipt_id = ?",
                (receipt_id,),
            ).fetchone()
        finally:
            conn.close()
    if row is None:
        return None
    return {
        "receipt_id": row["receipt_id"],
        "tombstone_receipt_id": row["tombstone_receipt_id"],
        "reason": row["reason"],
        "recorded_at": int(row["recorded_at"]),
    }


def vacuum_index(*, force: bool = False, root: Optional[Path] = None) -> None:
    """Run VACUUM on the index database, guarding against disruptive cases."""
    db_path = _db_path(root=root)
    wal_path = Path(f"{db_path}-wal")
    if wal_path.exists() and wal_path.stat().st_size > 0 and not force:
        raise RuntimeError(
            f"Refusing to VACUUM while WAL file is active ({wal_path}); rerun with force=True once idle."
        )
    with _LOCK:
        conn = _connect(root=root)
        try:
            conn.execute("PRAGMA wal_checkpoint(TRUNCATE);")
            conn.execute("VACUUM;")
        finally:
            conn.close()


def backup_index(out_path: Path, *, root: Optional[Path] = None) -> Path:
    """Create a SQLite backup of the index database."""
    destination = Path(out_path)
    destination.parent.mkdir(parents=True, exist_ok=True)
    with _LOCK:
        source = _connect(root=root)
        try:
            source.execute("PRAGMA wal_checkpoint(FULL);")
            dest = sqlite3.connect(str(destination))
            try:
                source.backup(dest)
            finally:
                dest.close()
        finally:
            source.close()
    return destination
def _load_index_entries(
    *,
    root: Optional[Path] = None,
    subset: Optional[Iterable[str]] = None,
) -> Dict[str, ReceiptIndexEntry]:
    with _LOCK:
        conn = _connect(root=root)
        try:
            if subset is not None:
                tokens = list(dict.fromkeys(subset))
                if not tokens:
                    rows: Sequence[sqlite3.Row] = ()
                else:
                    placeholders = ",".join(["?"] * len(tokens))
                    query = (
                        "SELECT receipt_id, ts, type, jcs_sha256, path "
                        f"FROM receipts_index WHERE receipt_id IN ({placeholders})"
                    )
                    rows = conn.execute(query, tokens).fetchall()
            else:
                rows = conn.execute(
                    "SELECT receipt_id, ts, type, jcs_sha256, path FROM receipts_index"
                ).fetchall()
        finally:
            conn.close()

    entries: Dict[str, ReceiptIndexEntry] = {}
    for row in rows:
        entries[row["receipt_id"]] = ReceiptIndexEntry(
            receipt_id=row["receipt_id"],
            ts=int(row["ts"]),
            receipt_type=row["type"],
            jcs_sha256=row["jcs_sha256"],
            path=_materialise_path(row["path"], root=root),
        )
    return entries


def _count_index_rows(*, root: Optional[Path] = None) -> int:
    with _LOCK:
        conn = _connect(root=root)
        try:
            row = conn.execute("SELECT COUNT(*) AS total FROM receipts_index").fetchone()
            return int(row["total"]) if row else 0
        finally:
            conn.close()
def _diff_index(
    *,
    root: Optional[Path] = None,
    limit: Optional[int] = None,
) -> Tuple[VerifyReport, Dict[str, Dict[str, Any]], Dict[str, ReceiptIndexEntry]]:
    ledger_rows = list(_iter_ledger_entries(root=root, limit=limit))
    ledger_map: Dict[str, Dict[str, Any]] = {row["receipt_id"]: row for row in ledger_rows}
    index_map = _load_index_entries(root=root)

    ledger_ids = set(ledger_map)
    index_ids = set(index_map)

    missing = sorted(ledger_ids - index_ids)
    if limit is None:
        orphan_ids = sorted(index_ids - ledger_ids)
    else:
        orphan_ids: List[str] = []

    mismatched: Dict[str, Tuple[str, str]] = {}
    for receipt_id in ledger_ids & index_ids:
        ledger_hash = str(ledger_map[receipt_id].get("receipt_hash") or "")
        index_hash = str(index_map[receipt_id].jcs_sha256 or "")
        if ledger_hash and index_hash and ledger_hash != index_hash:
            mismatched[receipt_id] = (ledger_hash, index_hash)

    report = VerifyReport(
        total_indexed=len(index_map),
        total_ledger=len(ledger_map),
        missing_receipts=missing,
        orphan_receipts=orphan_ids,
        hash_mismatches=mismatched,
    )
    return report, ledger_map, index_map


def verify_index(*, limit: Optional[int] = None, root: Optional[Path] = None) -> VerifyReport:
    """Compare the SQLite index against the ledger ground truth."""
    report, _ledger, _index = _diff_index(root=root, limit=limit)
    return report


def reconcile_index(
    *,
    dry_run: bool = False,
    limit: Optional[int] = None,
    root: Optional[Path] = None,
    progress: Optional[Callable[[str, str], None]] = None,
) -> ReconcileReport:
    """Repair index drift by reconciling with ledger rows."""
    before, ledger_map, index_map = _diff_index(root=root, limit=limit)
    if dry_run or not before.has_drift:
        return ReconcileReport(
            before=before,
            after=before,
            inserted=0,
            updated=0,
            deleted=0,
            dry_run=dry_run,
        )

    repaired_missing = sorted(before.missing_receipts)
    repaired_mismatched = sorted(before.hash_mismatches)
    deleted_orphans = sorted(before.orphan_receipts)

    def _operation(conn: sqlite3.Connection) -> Tuple[int, int, int]:
        local_inserted = 0
        local_updated = 0
        local_deleted = 0
        try:
            conn.execute("BEGIN IMMEDIATE;")
            for receipt_id in repaired_missing + repaired_mismatched:
                row = ledger_map.get(receipt_id)
                if row is None:
                    logger.warning("ledger row missing while reconciling receipt %s", receipt_id)
                    continue
                ts_value = _ensure_epoch(row.get("ts") or 0)
                receipt_type = str(row.get("receipt_type") or "UnknownReceipt")
                receipt_hash = str(row.get("receipt_hash") or "")
                rel_path = _expected_json_relative_path(
                    receipt_id=receipt_id,
                    receipt_hash=row.get("receipt_hash"),
                    tenant=row.get("tenant"),
                    domain=row.get("domain"),
                    ts=row.get("ts") or ts_value,
                    root=root,
                )
                if progress:
                    with contextlib.suppress(Exception):
                        progress("repair", receipt_id)
                conn.execute(
                    """
                    INSERT INTO receipts_index(receipt_id, ts, type, jcs_sha256, path)
                    VALUES (?, ?, ?, ?, ?)
                    ON CONFLICT(receipt_id) DO UPDATE SET
                        ts = excluded.ts,
                        type = excluded.type,
                        jcs_sha256 = excluded.jcs_sha256,
                        path = excluded.path
                    """,
                    (
                        receipt_id,
                        ts_value,
                        receipt_type,
                        receipt_hash,
                        rel_path,
                    ),
                )
                if receipt_id in repaired_missing:
                    local_inserted += 1
                else:
                    local_updated += 1

            for receipt_id in deleted_orphans:
                if progress:
                    with contextlib.suppress(Exception):
                        progress("delete", receipt_id)
                conn.execute("DELETE FROM receipts_index WHERE receipt_id = ?", (receipt_id,))
                local_deleted += 1

            conn.execute("COMMIT;")
            return local_inserted, local_updated, local_deleted
        except Exception:
            with contextlib.suppress(sqlite3.OperationalError):
                conn.execute("ROLLBACK;")
            raise

    inserted, updated, deleted = _write_with_retry(_operation, root=root)

    after, _, _ = _diff_index(root=root, limit=None)
    return ReconcileReport(
        before=before,
        after=after,
        inserted=inserted,
        updated=updated,
        deleted=deleted,
        dry_run=False,
    )
