"""SQLite-vec storage with semantic embeddings."""

from __future__ import annotations

import hashlib
import json
import logging
import math
import sqlite3
import struct
import threading
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING, Any

from .categories import CATEGORY_TTL, MemoryCategory, get_ttl_seconds, validate_category
from .memory_unit import MemoryUnit
from .migration import migrate_schema
from .semantic_cache import SemanticCache

if TYPE_CHECKING:
    from numpy import ndarray

logger = logging.getLogger(__name__)

_embedder_lock = threading.Lock()
_embedder = None


# Default embedding configuration
DEFAULT_EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
DEFAULT_EMBED_DIM = 384

# Recency decay configuration
DEFAULT_DECAY_HALF_LIFE_DAYS = 7.0
BREAKING_NEWS_WINDOW_HOURS = 24.0
# Categories that skip recency decay (always full weight)
PERMANENT_CATEGORIES = frozenset({"lesson"})

# Category-specific half-lives for recency decay (days)
CATEGORY_HALF_LIFE: dict[str, float] = {
    "decision": 30.0,
    "lesson": 14.0,
    "checkpoint": 7.0,
}
# Fallback half-life for unlisted categories
DEFAULT_CATEGORY_HALF_LIFE = 7.0

# Salience weights by category (intrinsic importance)
CATEGORY_SALIENCE: dict[str, float] = {
    "decision": 1.0,
    "lesson": 0.8,
    "checkpoint": 0.6,
}
DEFAULT_SALIENCE = 0.4

# Composite scoring weights
WEIGHT_SIMILARITY = 0.6
WEIGHT_SALIENCE = 0.2
WEIGHT_RECENCY = 0.1
WEIGHT_ACCESS_FREQ = 0.1
# Access count saturates at this value (normalized to 0-1)
ACCESS_FREQ_SATURATION = 10

# Conflict detection: pairwise similarity threshold for decision memories
CONFLICT_SIMILARITY_THRESHOLD = 0.80


def _get_embed_config() -> tuple[str, int]:
    """Read embedding model configuration from environment.

    Returns:
        Tuple of (model_name, expected_dimension).
    """
    import os

    model = os.environ.get("ENGRAM_EMBED_MODEL", DEFAULT_EMBED_MODEL)
    dim = int(os.environ.get("ENGRAM_EMBED_DIM", str(DEFAULT_EMBED_DIM)))
    return model, dim


def _get_embedder() -> Any:
    """Lazy load fastembed text embedding model (thread-safe).

    Model and dimension are configurable via environment variables:
        ENGRAM_EMBED_MODEL: fastembed model name (default: all-MiniLM-L6-v2)
        ENGRAM_EMBED_DIM: expected vector dimension (default: 384)
        ENGRAM_EMBED_THREADS: ONNX thread count (default: 4)

    Limits ONNX threads to reduce memory footprint in long-running daemon.
    """
    global _embedder
    if _embedder is None:
        with _embedder_lock:
            if _embedder is None:  # double-check locking
                import os

                from fastembed import TextEmbedding

                model_name, _ = _get_embed_config()
                threads = int(os.environ.get("ENGRAM_EMBED_THREADS", "4"))
                _embedder = TextEmbedding(
                    model_name=model_name,
                    threads=threads,
                )
    return _embedder


def _embed_text(text: str) -> ndarray:
    """Generate embedding for a single text string."""
    import numpy as np

    embedder = _get_embedder()
    result: ndarray = next(embedder.embed([text])).astype(np.float32)
    return result


def _summarize_content(content: str, max_tokens: int = 80) -> str:
    """Create a compact summary of memory content for Tier 1 retrieval.

    Uses simple heuristic: first sentence + truncation. A future version
    could use an LLM for better summaries.

    Args:
        content: Full memory content.
        max_tokens: Approximate max tokens (chars / 3.5 heuristic).

    Returns:
        Compact summary string.
    """
    max_chars = int(max_tokens * 3.5)  # Anthropic-recommended token-to-char ratio
    # Take first sentence or first max_chars
    first_sentence_end = content.find(". ")
    if 0 < first_sentence_end < max_chars:
        return content[: first_sentence_end + 1]
    if len(content) <= max_chars:
        return content
    return content[:max_chars].rsplit(" ", 1)[0] + "..."


def _compute_recency_decay(
    created_at: str,
    category: str,
    half_life_days: float | None = None,
    is_active: bool = True,
) -> float:
    """Compute recency decay factor for a memory.

    Formula: decay = 0.5 ^ (age_days / half_life)

    Special cases:
        - Memories < 24h old: no decay (breaking news window)
        - Permanent categories (lesson): no decay
        - Active decisions (is_active=True, category=decision): no decay
        - Returns value in [0.0, 1.0] where 1.0 = no decay

    Args:
        created_at: ISO timestamp of when memory was created.
        category: Memory category string.
        half_life_days: Override half-life (default: category-specific).
        is_active: Whether the memory is currently active.

    Returns:
        Decay multiplier between 0.0 and 1.0.
    """
    # Permanent categories never decay
    if category in PERMANENT_CATEGORIES:
        return 1.0

    # Active decisions are permanent (skip decay)
    if category == "decision" and is_active:
        return 1.0

    if half_life_days is None:
        half_life_days = CATEGORY_HALF_LIFE.get(category, DEFAULT_CATEGORY_HALF_LIFE)

    try:
        created = datetime.fromisoformat(created_at)
        now = datetime.now(tz=UTC)
        age = now - created
        age_days = age.total_seconds() / 86400.0

        # Breaking news window: no decay for very recent memories
        if age_days < (BREAKING_NEWS_WINDOW_HOURS / 24.0):
            return 1.0

        # Exponential decay: 0.5^(age_days / half_life)
        return math.pow(0.5, age_days / half_life_days)
    except (ValueError, TypeError):
        # If timestamp is unparseable, assume no decay
        return 1.0


def _compute_composite_score(
    similarity: float,
    category: str,
    created_at: str,
    access_count: int,
    is_active: bool = True,
) -> float:
    """Compute composite ranking score for a memory.

    Combines four signals into a weighted score:
        score = 0.6*similarity + 0.2*salience + 0.1*recency + 0.1*access_freq

    Args:
        similarity: Cosine similarity (0.0-1.0).
        category: Memory category string.
        created_at: ISO timestamp.
        access_count: Number of times returned in search results.
        is_active: Whether the memory is currently active.

    Returns:
        Composite score in [0.0, 1.0].
    """
    salience = CATEGORY_SALIENCE.get(category, DEFAULT_SALIENCE)
    recency = _compute_recency_decay(created_at, category, is_active=is_active)
    access_freq = min(access_count / ACCESS_FREQ_SATURATION, 1.0)

    return (
        WEIGHT_SIMILARITY * similarity
        + WEIGHT_SALIENCE * salience
        + WEIGHT_RECENCY * recency
        + WEIGHT_ACCESS_FREQ * access_freq
    )


def _estimate_token_count(content: str) -> int:
    """Estimate token count using chars/4 heuristic.

    Args:
        content: Text content.

    Returns:
        Estimated token count (minimum 1).
    """
    return max(1, int(len(content) / 3.5))


def _serialize_f32(vector: list[float] | ndarray) -> bytes:
    """Serialize vector to sqlite-vec format."""
    return struct.pack(f"{len(vector)}f", *vector)


def _compute_pairwise_conflicts(
    contents: list[str],
    memory_ids: list[int],
) -> dict[int, list[int]]:
    """Compute pairwise cosine similarity and flag conflicts above threshold.

    Batch-embeds all contents, then checks every pair. Returns a mapping
    from memory ID to list of conflicting memory IDs.

    Args:
        contents: Text contents of decision-category memories.
        memory_ids: Corresponding memory IDs (same order as contents).

    Returns:
        Dict mapping memory_id -> [conflicting_id, ...]. Only IDs with
        at least one conflict appear as keys.
    """
    import numpy as np

    if len(contents) < 2:
        return {}

    embedder = _get_embedder()
    vectors = [e.astype(np.float32) for e in embedder.embed(contents)]

    conflict_map: dict[int, list[int]] = {}
    for i in range(len(vectors)):
        for j in range(i + 1, len(vectors)):
            dot = float(np.dot(vectors[i], vectors[j]))
            norm_a = float(np.linalg.norm(vectors[i]))
            norm_b = float(np.linalg.norm(vectors[j]))
            if norm_a > 0 and norm_b > 0:
                sim = dot / (norm_a * norm_b)
                if sim > CONFLICT_SIMILARITY_THRESHOLD:
                    conflict_map.setdefault(memory_ids[i], []).append(memory_ids[j])
                    conflict_map.setdefault(memory_ids[j], []).append(memory_ids[i])

    return conflict_map


class MemoryStorage:
    """SQLite-vec based memory storage with semantic search."""

    def __init__(
        self,
        db_path: Path | None = None,
        cache_ttl: int = 300,
        cache_threshold: float = 0.92,
    ):
        if db_path is None:
            db_path = Path.home() / ".local" / "share" / "claude-engram" / "memories.db"
        db_path.parent.mkdir(parents=True, exist_ok=True)
        self.db_path = db_path
        self._local = threading.local()
        self._conn_lock = threading.Lock()
        self._all_conns: set[sqlite3.Connection] = set()
        self._init_db()
        self._semantic_cache: SemanticCache = SemanticCache(
            db_path=str(self.db_path),
            ttl_seconds=cache_ttl,
            similarity_threshold=cache_threshold,
        )

    @staticmethod
    def _setup_conn(conn: sqlite3.Connection) -> None:
        """Configure a SQLite connection with WAL mode and sqlite-vec extension."""
        conn.row_factory = sqlite3.Row
        # Enable WAL mode for concurrent access
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        # Load sqlite-vec extension
        conn.enable_load_extension(True)
        import sqlite_vec

        sqlite_vec.load(conn)
        conn.enable_load_extension(False)

    def _get_conn(self) -> sqlite3.Connection:
        """Get or create a thread-local connection with sqlite-vec loaded."""
        conn = getattr(self._local, "conn", None)
        if conn is None:
            conn = sqlite3.connect(str(self.db_path))
            self._setup_conn(conn)
            self._local.conn = conn
            with self._conn_lock:
                self._all_conns.add(conn)
        return conn

    def _init_db(self) -> None:
        """Initialize database schema."""
        # TODO: Fresh databases create v1 schema then migrate through v2→v3. Could create
        # the full current schema directly to skip ALTER TABLE steps. Works correctly as-is
        # but is mildly inefficient for first-time installations.
        conn = self._get_conn()
        conn.executescript("""
            CREATE TABLE IF NOT EXISTS memories (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                content TEXT NOT NULL,
                category TEXT DEFAULT 'general',
                metadata TEXT DEFAULT '{}',
                created_at TEXT NOT NULL,
                updated_at TEXT NOT NULL
            );

            CREATE INDEX IF NOT EXISTS idx_memories_category ON memories(category);
            CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at);

            CREATE VIRTUAL TABLE IF NOT EXISTS memory_vectors USING vec0(
                id INTEGER PRIMARY KEY,
                embedding FLOAT[384]
            );
        """)
        # Apply v2 migration (idempotent)
        migrate_schema(conn)
        conn.commit()

        # Validate embedding dimension matches schema
        self._validate_embed_dim(conn)

    def update_summary(self, memory_id: int, summary: str) -> bool:
        """Update the summary field for a memory.

        Args:
            memory_id: The memory ID.
            summary: The compact summary text.

        Returns:
            True if updated, False if memory not found.
        """
        conn = self._get_conn()
        cursor = conn.execute(
            "UPDATE memories SET summary = ? WHERE id = ?",
            (summary, memory_id),
        )
        conn.commit()
        return cursor.rowcount > 0

    def _validate_embed_dim(self, conn: sqlite3.Connection) -> None:
        """Check that configured embedding dimension matches the database schema.

        If ENGRAM_EMBED_DIM differs from the FLOAT[N] in the vec0 virtual table,
        vectors would be silently corrupted. This check prevents that.
        """
        _, expected_dim = _get_embed_config()
        if expected_dim != DEFAULT_EMBED_DIM:
            # Non-default dimension: warn if schema was created with default
            logger.warning(
                "ENGRAM_EMBED_DIM=%d differs from schema default %d. "
                "Existing vectors may be incompatible. If you changed the "
                "embedding model, you must re-embed all memories or reset the database.",
                expected_dim,
                DEFAULT_EMBED_DIM,
            )

    def store(
        self,
        content: str,
        category: str = "general",
        metadata: dict[str, Any] | None = None,
        proj_tag: str | None = None,
        valid_from: str | None = None,
        supersedes: int | None = None,
    ) -> int:
        """Store a memory with its embedding.

        Args:
            content: The memory content text.
            category: Memory category (e.g. "decision", "lesson", "general").
            metadata: Optional dict of extra metadata.
            proj_tag: Optional project tag for scoping (e.g. "proj:org/repo").
            valid_from: Optional ISO 8601 timestamp for when the memory became valid.
                If None, defaults to creation time.
            supersedes: Optional ID of the memory this one replaces. Creates a
                'supersedes' edge atomically and marks the old memory inactive.

        Returns:
            The ID of the stored memory.

        Raises:
            ValueError: If category is not a known MemoryCategory or "general".
        """
        # Validate category early, before computing the embedding
        validate_category(category)

        embedding = _embed_text(content)
        memory_id = self._store_with_embedding(
            content=content,
            category=category,
            metadata=metadata,
            proj_tag=proj_tag,
            embedding=embedding,
            valid_from=valid_from,
        )

        # Atomic edge creation when superseding another memory
        if supersedes is not None and memory_id > 0:
            from .graph import add_relation

            conn = self._get_conn()
            add_relation(conn, memory_id, supersedes, "supersedes")
            self.supersede_memory(supersedes, memory_id)

        # Invalidate semantic cache — new memory can affect any cached search
        if self._semantic_cache is not None:
            self._semantic_cache.invalidate()
        return memory_id

    def _build_where_clause(
        self,
        category: str | None = None,
        proj_tag: str | None = None,
        include_global: bool = True,
        include_inactive: bool = False,
        valid_at: str | None = None,
    ) -> tuple[str, list[str]]:
        """Build a dynamic WHERE clause for filtering memories.

        Args:
            category: Optional category filter.
            proj_tag: Optional project tag filter.
            include_global: If True, include memories with no proj_tag (global).
            include_inactive: If True, include superseded (inactive) memories.
            valid_at: Optional ISO 8601 timestamp. If provided, only return
                memories that were valid at this point in time.

        Returns:
            Tuple of (where_clause_string, params_list).
            The where_clause_string always starts with "WHERE" if non-empty.
        """
        conditions: list[str] = []
        params: list[str] = []

        # By default, only return active (non-superseded) memories
        if not include_inactive:
            conditions.append("m.is_active = 1")

        if category is not None:
            conditions.append("m.category = ?")
            params.append(category)

        if proj_tag is not None:
            if include_global:
                # '' represents global scope in DB (not NULL)
                conditions.append("(m.proj_tag = ? OR m.proj_tag = '')")
            else:
                conditions.append("m.proj_tag = ?")
            params.append(proj_tag)

        # Always exclude expired memories
        now = datetime.now(tz=UTC).isoformat()
        conditions.append("(m.expires_at IS NULL OR m.expires_at > ?)")
        params.append(now)

        # Temporal filter: only memories valid at the specified point in time
        if valid_at is not None:
            conditions.append("(m.valid_from IS NULL OR m.valid_from <= ?)")
            params.append(valid_at)
            conditions.append("(m.valid_to IS NULL OR m.valid_to > ?)")
            params.append(valid_at)

        where_str = "WHERE " + " AND ".join(conditions) if conditions else ""
        return where_str, params

    def supersede_memory(self, old_id: int, new_id: int) -> bool:
        """Mark a memory as superseded by a newer one.

        Sets old_id.superseded_by = new_id, old_id.is_active = 0,
        and old_id.valid_to = now (temporal boundary).
        The superseded memory is hidden from default searches but
        remains accessible with include_inactive=True.

        Args:
            old_id: The memory ID being replaced.
            new_id: The memory ID that replaces it.

        Returns:
            True if the old memory was found and updated, False otherwise.
        """
        conn = self._get_conn()
        now = datetime.now(tz=UTC).isoformat()
        cursor = conn.execute(
            "UPDATE memories SET superseded_by = ?, is_active = 0, valid_to = ? WHERE id = ?",
            (new_id, now, old_id),
        )
        conn.commit()
        return cursor.rowcount > 0

    def search(
        self,
        query: str,
        limit: int = 5,
        category: str | None = None,
        proj_tag: str | None = None,
        include_global: bool = True,
        detail_level: int = 2,
        include_inactive: bool = False,
        return_units: bool = False,
        enable_cache: bool = True,
        valid_at: str | None = None,
        search_mode: str = "vector",
    ) -> list[dict[str, Any]] | list[MemoryUnit]:
        """Semantic search for memories with progressive disclosure and recency decay.

        Applies recency decay to similarity scores:
            final_score = similarity * decay_factor
        where decay_factor = 0.5^(age_days / half_life).

        Args:
            query: Search query text.
            limit: Max number of results.
            category: Optional category filter.
            proj_tag: Optional project tag filter.
            include_global: If True, include global (untagged) memories.
            detail_level: Disclosure level (1=summary only, 2=full content, 3=full+related).
            include_inactive: If True, include superseded memories in results.
            return_units: If True, return list[MemoryUnit] instead of list[dict].
            enable_cache: If True, use semantic cache for faster repeated queries.
            valid_at: Optional ISO 8601 timestamp. If provided, only return
                memories that were valid at this point in time.
            search_mode: Search strategy. "vector" (default, semantic similarity),
                "hybrid" (keyword + semantic with RRF fusion), "keyword" (BM25 text match only).

        Returns:
            List of matching memory dicts (or MemoryUnit objects) sorted by
            recency-weighted similarity.
        """
        conn = self._get_conn()

        # Hybrid search mode (FTS5 + vector with RRF fusion)
        if search_mode in ("hybrid", "keyword"):
            from claude_engram.hybrid_search import (
                HybridSearchConfig,
                has_fts5_table,
                hybrid_search,
            )

            # Fallback to vector-only if FTS5 table doesn't exist (pre-v10 DB)
            if not has_fts5_table(conn):
                search_mode = "vector"
            else:
                # Skip semantic cache for hybrid/keyword mode
                # Keyword-only skips embedding; hybrid uses both
                if search_mode == "keyword":
                    embedding_list: list[float] = []
                else:
                    embedding_list = _embed_text(query).tolist()

                where_str, where_params = self._build_where_clause(
                    category, proj_tag, include_global, include_inactive, valid_at
                )

                config = HybridSearchConfig()
                # Fetch more candidates for re-ranking
                fetch_limit = min(limit * 3, 100)

                # hybrid_search() already has WHERE id IN (...), so convert
                # WHERE ... to AND ... for the additional filter clause.
                # Also strip table alias (m.) since hybrid_search uses bare table.
                hybrid_where = where_str.replace("m.", "")
                if hybrid_where.startswith("WHERE "):
                    hybrid_where = "AND " + hybrid_where[6:]

                raw_results = hybrid_search(
                    conn,
                    query_text=query,
                    query_embedding=embedding_list,
                    limit=fetch_limit,
                    config=config,
                    where_str=hybrid_where,
                    where_params=tuple(where_params),
                )

                # Apply composite scoring with RRF score instead of similarity
                # Normalize RRF score to [0, 1] range for consistency
                max_rrf = (config.weight_fts + config.weight_vec) / (config.rrf_k + 1)
                scored_hybrid: list[tuple[float, dict[str, Any]]] = []

                for r in raw_results:
                    # RRF score normalized to similarity-like range
                    normalized_rrf = r["rrf_score"] / max_rrf if max_rrf > 0 else 0.0
                    access_count = 0  # Not in raw_results yet, will update later

                    # Fetch access_count from DB for composite scoring
                    row = conn.execute(
                        "SELECT access_count FROM memories WHERE id = ?", (r["id"],)
                    ).fetchone()
                    if row:
                        access_count = row[0] or 0

                    final_score = _compute_composite_score(
                        similarity=normalized_rrf,
                        category=r["category"],
                        created_at=r["created_at"],
                        access_count=access_count,
                        is_active=True,  # hybrid_search already filters inactive
                    )
                    scored_hybrid.append((final_score, r))

                # Sort by composite score and take top limit
                scored_hybrid.sort(key=lambda x: x[0], reverse=True)
                top_results_hybrid = scored_hybrid[:limit]

                # Build memory results
                memories_hybrid: list[dict[str, Any]] = []
                for final_score, r in top_results_hybrid:
                    token_count = _estimate_token_count(r["content"])
                    mem_hybrid: dict[str, Any] = {
                        "id": r["id"],
                        "category": r["category"],
                        "created_at": r["created_at"],
                        "proj_tag": None,  # Will be fetched if needed
                        "similarity": final_score,
                        "token_count": token_count,
                        "is_active": True,
                        "rrf_score": r["rrf_score"],
                        "fts_rank": r["fts_rank"],
                        "vec_rank": r["vec_rank"],
                    }

                    if detail_level == 0:
                        mem_hybrid["detail_level"] = 0
                    elif detail_level == 1:
                        summary = _summarize_content(r["content"])
                        mem_hybrid["summary"] = summary
                        mem_hybrid["detail_level"] = 1
                    else:
                        mem_hybrid["content"] = r["content"]
                        mem_hybrid["metadata"] = json.loads(r["metadata"])
                        mem_hybrid["detail_level"] = detail_level

                    memories_hybrid.append(mem_hybrid)

                # Increment access_count for returned memories
                hit_ids = [mem["id"] for mem in memories_hybrid]
                if hit_ids:
                    placeholders = ",".join("?" * len(hit_ids))
                    # nosemgrep: sql-injection - placeholders built from safe data
                    conn.execute(
                        f"UPDATE memories SET access_count = COALESCE(access_count, 0) + 1"  # noqa: S608
                        f" WHERE id IN ({placeholders})",
                        hit_ids,
                    )
                    conn.commit()

                # Detect conflicts among decisions
                decision_contents_hybrid = [
                    m["content"] for m in memories_hybrid if m["category"] == "decision"
                ]
                decision_ids_hybrid = [
                    m["id"] for m in memories_hybrid if m["category"] == "decision"
                ]

                if len(decision_contents_hybrid) >= 2:
                    conflict_map = _compute_pairwise_conflicts(
                        decision_contents_hybrid, decision_ids_hybrid
                    )
                    for mem_item in memories_hybrid:
                        if mem_item["id"] in conflict_map:
                            mem_item["conflicts_with"] = conflict_map[mem_item["id"]]

                # No semantic cache for hybrid mode
                if return_units:
                    return [MemoryUnit.from_search_result(m) for m in memories_hybrid]
                return memories_hybrid

        # Generate query embedding
        query_embedding = _embed_text(query)

        # Semantic cache lookup — skip cache when temporal filter is active
        if enable_cache and valid_at is None and self._semantic_cache is not None:
            cached = self._semantic_cache.lookup(query_embedding.tolist())
            if cached is not None:
                if return_units:
                    return [MemoryUnit.from_search_result(m) for m in cached]
                return cached

        # Build dynamic WHERE clause
        where_str, where_params = self._build_where_clause(
            category, proj_tag, include_global, include_inactive, valid_at
        )

        # Fetch more candidates than requested to allow re-ranking after decay
        fetch_limit = min(limit * 3, 100)

        sql = f"""
            SELECT m.id, m.content, m.category, m.metadata, m.created_at,
                   m.proj_tag, m.summary, m.token_count,
                   m.is_active, m.superseded_by,
                   m.valid_from, m.valid_to, m.access_count,
                   vec_distance_cosine(v.embedding, ?) as distance
            FROM memory_vectors v
            JOIN memories m ON v.id = m.id
            {where_str}
            ORDER BY distance ASC
            LIMIT ?
        """
        params: list[Any] = [_serialize_f32(query_embedding), *where_params, fetch_limit]
        results = conn.execute(sql, params).fetchall()

        # Apply composite scoring and re-rank
        scored: list[tuple[float, Any]] = []
        for r in results:
            raw_similarity = max(0.0, 1 - r["distance"])
            access_count = r["access_count"] if r["access_count"] is not None else 0
            final_score = _compute_composite_score(
                similarity=raw_similarity,
                category=r["category"],
                created_at=r["created_at"],
                access_count=access_count,
                is_active=bool(r["is_active"]),
            )
            scored.append((final_score, r))

        # Sort by decayed score (descending) and take top `limit`
        scored.sort(key=lambda x: x[0], reverse=True)
        top_results = scored[:limit]

        memories = []
        for final_score, r in top_results:
            token_count = r["token_count"] or _estimate_token_count(r["content"])
            mem: dict[str, Any] = {
                "id": r["id"],
                "category": r["category"],
                "created_at": r["created_at"],
                "proj_tag": r["proj_tag"] or None,
                "similarity": final_score,
                "token_count": token_count,
                "is_active": bool(r["is_active"]),
            }
            if r["valid_from"] is not None:
                mem["valid_from"] = r["valid_from"]
            if r["valid_to"] is not None:
                mem["valid_to"] = r["valid_to"]
            if detail_level == 0:
                # Level 0 (index): metadata only, no content (~50 tokens)
                mem["detail_level"] = 0
            elif detail_level == 1:
                # Level 1 (summary): first 100 chars of content (~100 tokens)
                summary = r["summary"] if r["summary"] else _summarize_content(r["content"])
                mem["summary"] = summary
                mem["detail_level"] = 1
            else:
                # Level 2+ (full): complete content
                mem["content"] = r["content"]
                mem["metadata"] = json.loads(r["metadata"])
                mem["detail_level"] = detail_level
            memories.append(mem)

        # Increment access_count for returned memories
        hit_ids = [mem["id"] for mem in memories]
        if hit_ids:
            placeholders = ",".join("?" * len(hit_ids))
            # nosemgrep: sql-injection - placeholders built from safe data
            conn.execute(
                f"UPDATE memories SET access_count = COALESCE(access_count, 0) + 1"  # noqa: S608
                f" WHERE id IN ({placeholders})",
                hit_ids,
            )
            conn.commit()

        # Detect conflicts among decision-category results
        decision_contents: list[str] = []
        decision_ids: list[int] = []
        for _score, r in top_results:
            if r["category"] == "decision":
                decision_contents.append(r["content"])
                decision_ids.append(r["id"])

        if len(decision_contents) >= 2:
            conflict_map = _compute_pairwise_conflicts(decision_contents, decision_ids)
            for mem in memories:
                if mem["id"] in conflict_map:
                    mem["conflicts_with"] = conflict_map[mem["id"]]

        # Store results in semantic cache for future similar queries
        if enable_cache and self._semantic_cache is not None:
            self._semantic_cache.store(query_embedding.tolist(), query, memories)

        if return_units:
            return [MemoryUnit.from_search_result(m) for m in memories]

        return memories

    def get_recent(
        self,
        limit: int = 10,
        category: str | None = None,
        proj_tag: str | None = None,
        include_global: bool = True,
        since: str | None = None,
    ) -> list[dict[str, Any]]:
        """Get recent memories by creation time.

        Args:
            limit: Max number of results.
            category: Optional category filter.
            proj_tag: Optional project tag filter.
            include_global: If True, include global (untagged) memories.
            since: Optional ISO timestamp. If provided, only return memories
                   created AFTER this timestamp (SQL-level WHERE created_at > ?).

        Returns:
            List of recent memory dicts sorted by creation time (newest first).
        """
        conn = self._get_conn()

        # Build dynamic WHERE clause (uses alias "m" for memories table)
        where_str, where_params = self._build_where_clause(category, proj_tag, include_global)

        # Add since filter if provided (SQL-level time boundary)
        if since is not None:
            if where_str:
                where_str += " AND m.created_at > ?"
            else:
                where_str = "WHERE m.created_at > ?"
            where_params.append(since)

        sql = f"""
            SELECT m.id, m.content, m.category, m.metadata, m.created_at,
                   m.proj_tag, m.token_count
            FROM memories m
            {where_str}
            ORDER BY m.created_at DESC
            LIMIT ?
        """
        params: list[Any] = [*where_params, limit]
        results = conn.execute(sql, params).fetchall()

        return [
            {
                "id": r["id"],
                "content": r["content"],
                "category": r["category"],
                "metadata": json.loads(r["metadata"]),
                "created_at": r["created_at"],
                "proj_tag": r["proj_tag"] or None,
                "token_count": r["token_count"] or _estimate_token_count(r["content"]),
            }
            for r in results
        ]

    def get_by_id(self, memory_id: int) -> dict[str, Any] | None:
        """Get a single memory by its ID.

        Args:
            memory_id: The memory ID to look up.

        Returns:
            Memory dict with id, content, category, metadata, created_at, proj_tag,
            or None if not found.
        """
        conn = self._get_conn()
        row = conn.execute(
            "SELECT id, content, category, metadata, created_at, proj_tag, token_count "
            "FROM memories WHERE id = ?",
            (memory_id,),
        ).fetchone()
        if row is None:
            return None
        return {
            "id": row["id"],
            "content": row["content"],
            "category": row["category"],
            "metadata": json.loads(row["metadata"]),
            "created_at": row["created_at"],
            "proj_tag": row["proj_tag"] or None,
            "token_count": row["token_count"] or _estimate_token_count(row["content"]),
        }

    def get_by_ids(self, memory_ids: list[int]) -> list[dict[str, Any]]:
        """Get multiple memories by their IDs in a single query.

        Efficient batch fetch for progressive disclosure drill-down
        (expand multiple Tier 1 results to Tier 2).

        Args:
            memory_ids: List of memory IDs to fetch.

        Returns:
            List of memory dicts (preserves input order, skips missing IDs).
        """
        if not memory_ids:
            return []
        conn = self._get_conn()
        placeholders = ",".join("?" * len(memory_ids))
        # nosemgrep: sql-injection - placeholders is "?,?,?" built from safe data
        rows = conn.execute(
            f"SELECT id, content, category, metadata, created_at, proj_tag, "  # noqa: S608
            f"summary, token_count FROM memories WHERE id IN ({placeholders})",
            memory_ids,
        ).fetchall()

        # Build lookup for preserving input order
        lookup = {}
        for row in rows:
            lookup[row["id"]] = {
                "id": row["id"],
                "content": row["content"],
                "category": row["category"],
                "metadata": json.loads(row["metadata"]),
                "created_at": row["created_at"],
                "proj_tag": row["proj_tag"] or None,
                "token_count": row["token_count"] or _estimate_token_count(row["content"]),
            }

        # Return in input order, skip missing
        return [lookup[mid] for mid in memory_ids if mid in lookup]

    def delete(self, memory_id: int) -> bool:
        """Delete a memory by ID."""
        conn = self._get_conn()
        conn.execute("DELETE FROM memory_vectors WHERE id = ?", (memory_id,))
        cursor = conn.execute("DELETE FROM memories WHERE id = ?", (memory_id,))
        conn.commit()
        deleted = cursor.rowcount > 0
        # Invalidate cache entries that referenced this memory
        if deleted and self._semantic_cache is not None:
            self._semantic_cache.invalidate(memory_id)
        return deleted

    def cleanup_expired(self) -> int:
        """Delete all expired memories and their vectors.

        Returns:
            Number of expired memories deleted.
        """
        conn = self._get_conn()
        now = datetime.now(tz=UTC).isoformat()

        # Find expired memory IDs
        expired_rows = conn.execute(
            "SELECT id FROM memories WHERE expires_at IS NOT NULL AND expires_at < ?",
            (now,),
        ).fetchall()

        if not expired_rows:
            return 0

        expired_ids = [r["id"] for r in expired_rows]

        # Delete vectors first (foreign key reference), then memories
        placeholders = ",".join("?" * len(expired_ids))
        # nosemgrep: sql-injection - placeholders is "?,?,?" built from safe data
        conn.execute(
            f"DELETE FROM memory_vectors WHERE id IN ({placeholders})",  # noqa: S608
            expired_ids,
        )
        # nosemgrep: sql-injection - placeholders is "?,?,?" built from safe data
        conn.execute(
            f"DELETE FROM memories WHERE id IN ({placeholders})",  # noqa: S608
            expired_ids,
        )
        conn.commit()
        return len(expired_ids)

    def _store_with_embedding(
        self,
        content: str,
        category: str,
        metadata: dict[str, Any] | None,
        proj_tag: str | None,
        embedding: ndarray,
        valid_from: str | None = None,
    ) -> int:
        """Store a memory with a pre-computed embedding.

        Internal method used by batch_store for efficiency. Skips embedding
        computation since embedding is provided.

        Args:
            content: The memory content text.
            category: Memory category.
            metadata: Optional metadata dict.
            proj_tag: Optional project tag.
            embedding: Pre-computed embedding vector.
            valid_from: Optional ISO 8601 timestamp for temporal validity start.

        Returns:
            The ID of the stored memory.
        """
        conn = self._get_conn()
        now = datetime.now(tz=UTC).isoformat()

        # Normalize proj_tag: use "" instead of NULL
        db_proj_tag = proj_tag if proj_tag is not None else ""

        # Compute content hash for deduplication
        content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()

        # Check for existing duplicate
        existing = conn.execute(
            "SELECT id FROM memories WHERE content_hash = ? AND category = ? AND proj_tag = ?",
            (content_hash, category, db_proj_tag),
        ).fetchone()
        if existing:
            return int(existing["id"])

        # Compute TTL tier and expiration
        ttl_tier = "permanent"
        expires_at: str | None = None
        try:
            cat_enum = MemoryCategory(category)
            tier = CATEGORY_TTL[cat_enum]
            ttl_tier = tier.value
            ttl_seconds = get_ttl_seconds(cat_enum)
            if ttl_seconds is not None:
                expires_at = (datetime.now(tz=UTC) + timedelta(seconds=ttl_seconds)).isoformat()
        except ValueError:
            pass

        # Estimate token count for budget tracking
        token_count = _estimate_token_count(content)

        # Resolve valid_from: explicit value or default to creation time
        db_valid_from = valid_from if valid_from is not None else now

        cursor = conn.execute(
            """
            INSERT OR IGNORE INTO memories (
                content, category, metadata, created_at, updated_at,
                proj_tag, ttl_tier, expires_at, content_hash, token_count,
                valid_from
            )
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """,
            (
                content,
                category,
                json.dumps(metadata or {}),
                now,
                now,
                db_proj_tag,
                ttl_tier,
                expires_at,
                content_hash,
                token_count,
                db_valid_from,
            ),
        )
        memory_id: int = cursor.lastrowid or 0

        if cursor.rowcount == 0:
            existing = conn.execute(
                "SELECT id FROM memories WHERE content_hash = ? AND category = ? AND proj_tag = ?",
                (content_hash, category, db_proj_tag),
            ).fetchone()
            if existing:
                return int(existing["id"])

        # Guard against race condition: if both INSERT OR IGNORE and re-SELECT
        # failed to produce a valid id (concurrent insert+delete by another thread),
        # skip the vector insert to avoid inserting with id=0.
        if memory_id <= 0:
            logger.warning(
                "store race: memory_id=%d after INSERT+re-SELECT for hash=%s; skipping vector",
                memory_id,
                content_hash[:12],
            )
            return memory_id

        conn.execute(
            "INSERT INTO memory_vectors (id, embedding) VALUES (?, ?)",
            (memory_id, _serialize_f32(embedding)),
        )
        conn.commit()
        return memory_id

    def batch_store(self, items: list[dict[str, Any]]) -> list[int]:
        """Store multiple memories at once with batch embedding.

        Uses batch embedding for efficiency (10-20x faster than individual
        embedding per item). Items that are duplicates of existing memories
        skip embedding entirely.

        Each item is a dict with keys:
            - content (str, required): The memory text.
            - category (str, optional): Default "general".
            - metadata (dict, optional): Default None.
            - proj_tag (str, optional): Default None.

        Args:
            items: List of memory dicts to store.

        Returns:
            List of memory IDs (one per item).
        """
        if not items:
            return []

        import numpy as np

        conn = self._get_conn()

        # First pass: check for duplicates, collect texts needing embeddings
        results: list[tuple[int, int | None]] = []  # (item_index, existing_id or None)
        texts_to_embed: list[str] = []
        text_to_item_indices: list[int] = []  # maps embed index -> item index

        for i, item in enumerate(items):
            content = item["content"]
            category = item.get("category", "general")
            proj_tag = item.get("proj_tag")
            db_proj_tag = proj_tag if proj_tag is not None else ""
            content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()

            existing = conn.execute(
                "SELECT id FROM memories WHERE content_hash = ? AND category = ? AND proj_tag = ?",
                (content_hash, category, db_proj_tag),
            ).fetchone()

            if existing:
                results.append((i, int(existing["id"])))
            else:
                # Check if same content already queued in this batch
                found_in_batch = False
                for idx, eid in results:
                    if eid is None:
                        prev_item = items[idx]
                        prev_hash = hashlib.sha256(prev_item["content"].encode("utf-8")).hexdigest()
                        prev_proj = prev_item.get("proj_tag")
                        prev_db_proj = prev_proj if prev_proj is not None else ""
                        if (
                            prev_hash == content_hash
                            and prev_item.get("category", "general") == category
                            and prev_db_proj == db_proj_tag
                        ):
                            # Duplicate within batch - will resolve to same ID
                            results.append((i, None))  # marked as needing resolution
                            found_in_batch = True
                            break
                if not found_in_batch:
                    results.append((i, None))
                    texts_to_embed.append(content)
                    text_to_item_indices.append(i)

        # Batch embed all non-duplicate texts at once
        embeddings: list[ndarray] = []
        if texts_to_embed:
            embedder = _get_embedder()
            embeddings = [e.astype(np.float32) for e in embedder.embed(texts_to_embed)]

        # Build O(1) lookup map: item_idx -> embedding position
        idx_to_emb_pos = {idx: pos for pos, idx in enumerate(text_to_item_indices)}

        # Second pass: store with pre-computed embeddings
        ids: list[int | None] = [None] * len(items)
        for item_idx, existing_id in results:
            if existing_id is not None:
                ids[item_idx] = existing_id
            elif item_idx in idx_to_emb_pos:
                item = items[item_idx]
                emb_pos = idx_to_emb_pos[item_idx]
                mid = self._store_with_embedding(
                    content=item["content"],
                    category=item.get("category", "general"),
                    metadata=item.get("metadata"),
                    proj_tag=item.get("proj_tag"),
                    embedding=embeddings[emb_pos],
                )
                ids[item_idx] = mid
            else:
                # Batch-internal duplicate: find the first instance ID
                item = items[item_idx]
                content_hash = hashlib.sha256(item["content"].encode("utf-8")).hexdigest()
                category = item.get("category", "general")
                proj_tag = item.get("proj_tag")
                db_proj_tag = proj_tag if proj_tag is not None else ""
                existing = conn.execute(
                    "SELECT id FROM memories"
                    " WHERE content_hash = ? AND category = ? AND proj_tag = ?",
                    (content_hash, category, db_proj_tag),
                ).fetchone()
                if existing:
                    ids[item_idx] = int(existing["id"])
                else:
                    # Fallback: store individually
                    mid = self.store(
                        content=item["content"],
                        category=category,
                        metadata=item.get("metadata"),
                        proj_tag=proj_tag,
                    )
                    ids[item_idx] = mid

        # Invalidate semantic cache once after all items stored
        if self._semantic_cache is not None:
            self._semantic_cache.invalidate()

        return [i for i in ids if i is not None]

    def batch_search(self, queries: list[dict[str, Any]]) -> list[list[dict[str, Any]]]:
        """Run multiple semantic searches in one call.

        Each query is a dict with keys:
            - query (str, required): Search query text.
            - limit (int, optional): Default 5.
            - category (str, optional): Default None.
            - proj_tag (str, optional): Default None.
            - include_global (bool, optional): Default True.

        Args:
            queries: List of search query dicts.

        Returns:
            List of result lists (one per query).
        """
        results: list[list[dict[str, Any]]] = []
        for q in queries:
            result = self.search(
                query=q["query"],
                limit=q.get("limit", 5),
                category=q.get("category"),
                proj_tag=q.get("proj_tag"),
                include_global=q.get("include_global", True),
            )
            results.append(result)  # type: ignore[arg-type]  # return_units defaults to False
        return results

    def stats(self) -> dict[str, Any]:
        """Get storage statistics."""
        conn = self._get_conn()
        total = conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0]
        categories = conn.execute(
            "SELECT category, COUNT(*) as count FROM memories GROUP BY category"
        ).fetchall()

        # Schema version
        schema_version = conn.execute("PRAGMA user_version").fetchone()[0]

        # Count expired memories
        now = datetime.now(tz=UTC).isoformat()
        expired_count = conn.execute(
            "SELECT COUNT(*) FROM memories WHERE expires_at IS NOT NULL AND expires_at < ?",
            (now,),
        ).fetchone()[0]

        return {
            "total_memories": total,
            "categories": {r["category"]: r["count"] for r in categories},
            "db_path": str(self.db_path),
            "schema_version": schema_version,
            "expired_count": expired_count,
        }

    @property
    def embedding_model_loaded(self) -> bool:
        """Check if the embedding model is loaded."""
        return _embedder is not None

    # ------------------------------------------------------------------
    # Counter operations (no embeddings, no TTL — simple key-value)
    # ------------------------------------------------------------------

    def update_counter(self, key: str, delta: int = 1, proj_tag: str | None = None) -> int:
        """Atomically increment/decrement a counter.

        Creates the counter if it doesn't exist. Uses INSERT ... ON CONFLICT
        DO UPDATE for a single-statement atomic upsert.

        Args:
            key: Counter name (e.g. "serena_calls", "tool_use_count").
            delta: Amount to add (negative to decrement). Default 1.
            proj_tag: Project scope. None = global.

        Returns:
            The new counter value after the update.
        """
        conn = self._get_conn()
        db_proj_tag = proj_tag if proj_tag is not None else ""
        now = datetime.now(tz=UTC).isoformat()

        conn.execute(
            """INSERT INTO counters (key, value, proj_tag, updated_at)
               VALUES (?, ?, ?, ?)
               ON CONFLICT(key, proj_tag) DO UPDATE
               SET value = value + ?, updated_at = ?""",
            (key, delta, db_proj_tag, now, delta, now),
        )
        conn.commit()

        row = conn.execute(
            "SELECT value FROM counters WHERE key = ? AND proj_tag = ?",
            (key, db_proj_tag),
        ).fetchone()
        return int(row["value"])

    def set_counter(self, key: str, value: int, proj_tag: str | None = None) -> int:
        """Set a counter to an absolute value.

        Creates the counter if it doesn't exist.

        Args:
            key: Counter name.
            value: Absolute value to set.
            proj_tag: Project scope. None = global.

        Returns:
            The value that was set.
        """
        conn = self._get_conn()
        db_proj_tag = proj_tag if proj_tag is not None else ""
        now = datetime.now(tz=UTC).isoformat()

        conn.execute(
            """INSERT INTO counters (key, value, proj_tag, updated_at)
               VALUES (?, ?, ?, ?)
               ON CONFLICT(key, proj_tag) DO UPDATE
               SET value = ?, updated_at = ?""",
            (key, value, db_proj_tag, now, value, now),
        )
        conn.commit()
        return value

    def get_counter(self, key: str, proj_tag: str | None = None) -> int:
        """Get a single counter value.

        Args:
            key: Counter name.
            proj_tag: Project scope. None = global.

        Returns:
            The counter value, or 0 if not found.
        """
        conn = self._get_conn()
        db_proj_tag = proj_tag if proj_tag is not None else ""

        row = conn.execute(
            "SELECT value FROM counters WHERE key = ? AND proj_tag = ?",
            (key, db_proj_tag),
        ).fetchone()
        return int(row["value"]) if row else 0

    def get_counters(self, proj_tag: str | None = None) -> dict[str, int]:
        """Get all counters for a project scope.

        Args:
            proj_tag: Project scope. None = global (empty string in DB).

        Returns:
            Dict mapping counter keys to their values.
        """
        conn = self._get_conn()
        db_proj_tag = proj_tag if proj_tag is not None else ""

        rows = conn.execute(
            "SELECT key, value FROM counters WHERE proj_tag = ?",
            (db_proj_tag,),
        ).fetchall()
        return {row["key"]: int(row["value"]) for row in rows}

    def delete_counter(self, key: str, proj_tag: str | None = None) -> bool:
        """Delete a counter.

        Args:
            key: Counter name.
            proj_tag: Project scope. None = global.

        Returns:
            True if the counter existed and was deleted, False if not found.
        """
        conn = self._get_conn()
        db_proj_tag = proj_tag if proj_tag is not None else ""

        cursor = conn.execute(
            "DELETE FROM counters WHERE key = ? AND proj_tag = ?",
            (key, db_proj_tag),
        )
        conn.commit()
        return cursor.rowcount > 0

    def cleanup_duplicates(self) -> int:
        """Remove duplicate memories, keeping the newest copy of each.

        Finds groups with the same (content_hash, category, proj_tag) and
        deletes all but the row with the highest id (newest). Also cleans
        up orphaned vectors and re-creates the UNIQUE dedup index.

        Returns:
            Number of duplicate memories removed.
        """
        conn = self._get_conn()

        # Find IDs to delete: all but MAX(id) per (content_hash, category, proj_tag)
        dupe_ids = conn.execute(
            """SELECT id FROM memories
               WHERE id NOT IN (
                   SELECT MAX(id)
                   FROM memories
                   WHERE content_hash IS NOT NULL
                   GROUP BY content_hash, category, proj_tag
               )
               AND content_hash IS NOT NULL
               AND content_hash IN (
                   SELECT content_hash FROM memories
                   GROUP BY content_hash, category, proj_tag
                   HAVING COUNT(*) > 1
               )"""
        ).fetchall()

        if not dupe_ids:
            return 0

        ids_to_delete = [r["id"] for r in dupe_ids]
        placeholders = ",".join("?" * len(ids_to_delete))

        # Delete orphaned vectors first
        # nosemgrep: sql-injection - placeholders is "?,?,?" built from safe data
        conn.execute(
            f"DELETE FROM memory_vectors WHERE id IN ({placeholders})",  # noqa: S608
            ids_to_delete,
        )
        # Delete duplicate memories
        # nosemgrep: sql-injection - placeholders is "?,?,?" built from safe data
        conn.execute(
            f"DELETE FROM memories WHERE id IN ({placeholders})",  # noqa: S608
            ids_to_delete,
        )

        # Re-create UNIQUE index if it was dropped
        conn.execute(
            """CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_dedup
               ON memories(content_hash, category, proj_tag)"""
        )
        conn.commit()

        logger.info("cleanup_duplicates: removed %d duplicate memories", len(ids_to_delete))
        return len(ids_to_delete)

    def checkpoint_wal(self) -> int:
        """Checkpoint WAL file to prevent unbounded growth.

        SQLite's WAL mode auto-checkpoints when WAL reaches 1000 pages, but in
        long-running daemons with sporadic writes, WAL can accumulate. This
        forces a TRUNCATE checkpoint which resets WAL to zero bytes.

        Returns:
            Number of WAL pages checkpointed.
        """
        conn = self._get_conn()
        # TRUNCATE mode: checkpoint and truncate WAL to zero bytes
        # Returns (busy, log_pages, checkpointed_pages)
        result = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
        return result[2] if result else 0

    def close(self) -> None:
        """Close all tracked database connections across all threads."""
        import contextlib

        # Close semantic cache connections first
        if self._semantic_cache is not None:
            self._semantic_cache.close()

        with self._conn_lock:
            for conn in self._all_conns:
                with contextlib.suppress(Exception):
                    conn.close()
            self._all_conns.clear()
        self._local.conn = None
