"""
cortex/core/retrieval.py — Retrieval Pipeline (Module 8 / M8).

The read-side counterpart to the remember() pipeline. Given a natural language
query, finds and assembles the most relevant memories from storage.

Phase 1 (current): Basic hybrid retrieval
    - BM25 full-text search via FTS5 virtual tables
    - Vector similarity search via sqlite-vec (or brute-force cosine fallback)
    - Reciprocal Rank Fusion (RRF) to combine both ranked lists
    - Token budget enforcement (trim results to fit context window)

Phase 2 (issue #34): Agentic retrieval
    - Query expansion via LLM
    - Multi-round retrieval with refinement
    - LLM reranking of candidates
    - Brain graph traversal
    - Implemented in cortex.core.agentic_retrieval.AgenticRetrievalPipeline

Retrieval pipeline:
    1. Route query → mode ('lightweight' or 'agentic')
    2. Parallel BM25 + vector search (fetch candidate_multiplier × top_k candidates)
    3. RRF fusion → single ranked list
    4. Token budget trim → final items
    5. Log retrieval metadata for feedback/debugging

Dependencies:
    - cortex.storage.adapter.StorageAdapter (M1) — vector_search, fts_search
    - cortex.core.token_budget.TokenBudgetManager (M9) — budget allocation
    - cortex.config.CortexConfig — retrieval_token_budget, vector/bm25 weights
    - LLMProvider (optional) — for embedding generation and agentic mode

Data flow:
    CortexPlugin.retrieve(query)
      → RetrievalPipeline.retrieve(query, mode, constraints)
          → _bm25_search() + _vector_search() (parallel)
          → rrf_fusion() → sorted candidates
          → _apply_budget() → trimmed to token limit
      → RetrievalResult (items + metadata)
"""

from __future__ import annotations

import asyncio
import hashlib
import logging
import math
import os
import re
import time
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

if TYPE_CHECKING:
    from cortex.core.token_budget import TokenBudgetManager
    from cortex.storage.adapter import StorageAdapter

# Embedding cache — optional, imported lazily to keep retrieval.py importable
# even if cachetools is not installed (cache is simply disabled in that case).
try:
    from cortex.core.embedding_cache import EmbeddingCache as _EmbeddingCache
except ImportError:  # pragma: no cover
    _EmbeddingCache = None  # type: ignore[assignment,misc]

# Voyage AI SDK — optional, required only when retrieval_rerank_enabled=True.
try:
    import voyageai  # type: ignore
except ImportError:
    voyageai = None  # type: ignore

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Query complexity heuristics for dynamic top-k and selective rerank (#495).
# ---------------------------------------------------------------------------

# Keywords that signal a high-complexity query (architecture, decisions, etc.).
# These benefit from more candidates and reranking to cut through noise.
_COMPLEX_QUERY_SIGNALS = re.compile(
    r"\b(decision|architecture|migration|design|tradeoff|trade-off|why did|"
    r"technical|strategy|rationale|reasoning|compare|versus|pros and cons|"
    r"incident|postmortem|root cause|benchmark|evaluation)\b",
    re.IGNORECASE,
)

# Minimum query length (chars) to be considered complex regardless of keywords.
_COMPLEX_QUERY_MIN_LENGTH = 800


def _is_complex_query(query: str) -> bool:
    """Return True if a query is complex enough to warrant reranking + more candidates.

    Heuristic: keyword match OR query length exceeds threshold.  Simple identity
    / preference recall ('what is my name?', 'what timezone?') stays fast.
    """
    if len(query) >= _COMPLEX_QUERY_MIN_LENGTH:
        return True
    return bool(_COMPLEX_QUERY_SIGNALS.search(query))


# ---------------------------------------------------------------------------
# Sensitivity level ordering (lower index = less sensitive)
# ---------------------------------------------------------------------------

SENSITIVITY_ORDER = ["normal", "private", "secret", "restricted"]
_SENSITIVITY_ALIASES = {"sensitive": "secret", "personal": "private"}


def _sensitivity_level(value: str) -> int:
    """Return numeric level for a sensitivity string (fail-closed for unknowns)."""
    canonical = _SENSITIVITY_ALIASES.get(value, value)
    try:
        return SENSITIVITY_ORDER.index(canonical)
    except ValueError:
        return len(SENSITIVITY_ORDER)


# ---------------------------------------------------------------------------
# Public data-transfer objects
# ---------------------------------------------------------------------------

@dataclass
class RetrievalConstraints:
    """Configurable constraints for a retrieve() call.

    All fields are optional — unset fields use defaults from CortexConfig.

    Attributes:
        memory_types: Filter by item type — 'claim', 'event', 'procedure', etc.
                      None = all types.
        time_range: ISO 8601 (start, end) tuple for temporal filtering.
        entity_ids: Only return items related to these entity UUIDs.
        sensitivity_max: Maximum sensitivity level to return. Items above this
                         level are excluded. Valid: normal < private < secret <
                         restricted. Default: 'secret' (excludes 'restricted').
        token_budget: Override the global retrieval_token_budget for this call.
        top_k: Maximum number of items to return after fusion. Default: 20.
        before_date: ISO date string (YYYY-MM-DD). Exclude claims whose
                     event_date is after this date (inclusive upper bound).
                     Claims without event_date are never excluded.
        after_date: ISO date string (YYYY-MM-DD). Exclude claims whose
                    event_date is before this date (inclusive lower bound).
                    Claims without event_date are never excluded.
    """
    memory_types: Optional[List[str]] = None        # filter by item_type
    time_range: Optional[Tuple[str, str]] = None    # (iso_start, iso_end)
    entity_ids: Optional[List[str]] = None
    sensitivity_max: str = "secret"                 # excludes 'restricted' by default
    token_budget: Optional[int] = None              # override global budget
    top_k: int = 20
    before_date: Optional[str] = None              # ISO date upper bound for event_date
    after_date: Optional[str] = None               # ISO date lower bound for event_date


@dataclass
class RetrievedItem:
    """A single memory item returned from retrieval.

    Attributes:
        item_type: Memory layer type — 'claim', 'event', 'procedure', 'brain_graph_node'.
        item_id: UUID of the source record.
        content: Text content to inject into the prompt.
        score: Hybrid/RRF score — higher = more relevant.
        provenance: Optional trace info (retrieval explanation, Phase 2).
        metadata: Optional dict of classification metadata (memory_class, salience, status)
                  populated for claim items so the context compiler can use them.
    """
    item_type: str          # 'claim', 'event', 'procedure', 'brain_graph_node'
    item_id: str
    content: str
    score: float
    provenance: Optional[str] = None
    metadata: Optional[Dict[str, Any]] = None


@dataclass
class RetrievalResult:
    """Complete result from a retrieve() call.

    Returned by: RetrievalPipeline.retrieve().
    Consumed by: CortexPlugin.retrieve() → returned to caller / MCP / HTTP.

    Attributes:
        items: Ranked list of retrieved memory items.
        retrieval_id: UUID for this retrieval (used in feedback/logging).
        mode: Which retrieval mode was used — 'lightweight', 'agentic', etc.
        latency_ms: End-to-end retrieval time in milliseconds.
        tokens_used: Total tokens consumed by the returned items.
    """
    items: List[RetrievedItem]
    retrieval_id: str
    mode: str
    latency_ms: int
    tokens_used: int


@dataclass
class ContextBlock:
    """Assembled prompt-injection block produced by assemble_context().

    This is the final formatted output ready for system prompt injection.

    Attributes:
        parts: List of (type, text, tokens) tuples in injection order.
        total_tokens: Sum of all part token counts.
        budget_remaining: How many tokens are still available after this block.
        retrieval_id: UUID for tracing/feedback.
    """
    parts: List[Tuple[str, str, int]]  # (type, text, tokens)
    total_tokens: int
    budget_remaining: int
    retrieval_id: str


# ---------------------------------------------------------------------------
# Internal result types that mirror StorageAdapter returns
# (kept here so retrieval.py doesn't import from adapter directly)
# ---------------------------------------------------------------------------

@dataclass
class _VectorResult:
    item_type: str
    item_id: str
    content: str
    score: float


@dataclass
class _FTSResult:
    item_type: str
    item_id: str
    content: str
    rank: float  # BM25 rank (lower is better in SQLite FTS5 bm25())


# ---------------------------------------------------------------------------
# Claim content rendering (LoCoMo v10 fields)
# ---------------------------------------------------------------------------

def _render_claim_content(claim: Any) -> str:
    """Build enriched content string from a SemanticClaim, surfacing LoCoMo v10 fields.

    Format:
        [event_date] subject predicate object [status] (because: cause → effect) (was: before → now: after)

    Rules:
        - event_date: prefix with [date] if present
        - status: append [status] only if NOT 'active'
        - cause + effect: append (because: cause → effect) if both; single form if only one
        - state_before + state_after: append (was: X → now: Y) if both; single form if only one
    """
    event_date = getattr(claim, "event_date", None)
    status = getattr(claim, "status", "active") or "active"
    cause = getattr(claim, "cause", None)
    effect = getattr(claim, "effect", None)
    state_before = getattr(claim, "state_before", None)
    state_after = getattr(claim, "state_after", None)

    prefix = f"[{event_date}] " if event_date else ""
    core = f"{claim.subject} {claim.predicate} {claim.object_value}"

    suffixes: List[str] = []

    if status and status != "active":
        suffixes.append(f"[{status}]")

    if cause and effect:
        suffixes.append(f"(because: {cause} \u2192 {effect})")
    elif cause:
        suffixes.append(f"(cause: {cause})")
    elif effect:
        suffixes.append(f"(effect: {effect})")

    if state_before and state_after:
        suffixes.append(f"(was: {state_before} \u2192 now: {state_after})")
    elif state_before:
        suffixes.append(f"(was: {state_before})")
    elif state_after:
        suffixes.append(f"(now: {state_after})")

    result = prefix + core
    if suffixes:
        result += " " + " ".join(suffixes)
    return result


# ---------------------------------------------------------------------------
# Scoring helpers (pure functions — easy to unit-test)
# ---------------------------------------------------------------------------

def rrf_fusion(
    ranked_lists: List[List[Tuple[str, float]]],
    k: int = 60,
) -> List[Tuple[str, float]]:
    """Reciprocal Rank Fusion — combines multiple ranked lists into one.

    RRF is a simple, robust fusion method that's less sensitive to score
    scale differences between rankers than raw score averaging. It works by
    converting scores to reciprocal ranks: score(d) = Σ 1/(k + rank(d)).

    The k parameter controls how much weight top-ranked items get vs. lower-ranked
    ones. k=60 is the standard value from the original RRF paper (Cormack 2009).

    Args:
        ranked_lists: List of ranked lists, each containing (item_id, score)
                      tuples sorted best-first.
        k: Smoothing constant (default 60). Higher k = more uniform weighting.

    Returns:
        Fused list of (item_id, rrf_score) sorted by score descending.
    """
    scores: Dict[str, float] = {}
    for ranked_list in ranked_lists:
        for rank, (item_id, _) in enumerate(ranked_list):
            scores[item_id] = scores.get(item_id, 0.0) + 1.0 / (k + rank + 1)
    return sorted(scores.items(), key=lambda x: x[1], reverse=True)


def cosine_similarity(a: List[float], b: List[float]) -> float:
    """
    Brute-force cosine similarity for the sqlite-vec fallback path.
    Returns value in [-1, 1]; higher = more similar.
    """
    if len(a) != len(b):
        raise ValueError(f"Embedding dimension mismatch: {len(a)} vs {len(b)}")
    dot = sum(x * y for x, y in zip(a, b))
    norm_a = math.sqrt(sum(x * x for x in a))
    norm_b = math.sqrt(sum(x * x for x in b))
    if norm_a == 0.0 or norm_b == 0.0:
        return 0.0
    return dot / (norm_a * norm_b)


# ---------------------------------------------------------------------------
# Main pipeline
# ---------------------------------------------------------------------------

class RetrievalPipeline:
    """
    Phase 1 hybrid retrieval: BM25 (FTS5) + vector similarity, fused with RRF.

    Phase 2 additions (issue #34): query expansion, multi-round retrieval,
    LLM rerank, brain-graph traversal.
    """

    def __init__(
        self,
        storage: "StorageAdapter",
        config: Any,           # CortexConfig
        token_budget: Optional["TokenBudgetManager"] = None,
        cornerstone_guardian: Any = None,  # Phase 2
        llm: Any = None,                   # Phase 2 — agentic mode
        working_memory: Optional[Any] = None,  # WorkingMemoryCache (#371)
        embedding_cache: Optional[Any] = None,  # EmbeddingCache (v1.2.1)
        *,
        vector_weight: float = 0.7,
        bm25_weight: float = 0.3,
        rrf_k: int = 60,
        candidate_multiplier: int = 3,     # fetch N×top_k candidates before fusion
        recency_weight: Optional[float] = None,
    ) -> None:
        self.storage = storage
        self.config = config
        self.token_budget = token_budget
        self.cornerstone_guardian = cornerstone_guardian
        self.llm = llm
        self.working_memory = working_memory  # Optional[WorkingMemoryCache]

        # ------------------------------------------------------------------
        # Embedding cache (v1.2.1): build from config when not explicitly
        # provided.  Callers may inject a pre-built EmbeddingCache instance
        # (e.g. to share it across pipelines or in tests).
        # ------------------------------------------------------------------
        if embedding_cache is not None:
            self.embedding_cache: Optional[Any] = embedding_cache
        elif _EmbeddingCache is not None:
            _ec_enabled = getattr(config, "embedding_cache_enabled", True)
            _ec_maxsize = getattr(config, "embedding_cache_max_size", 1000)
            _ec_ttl = getattr(config, "embedding_cache_ttl_seconds", 3600)
            self.embedding_cache = _EmbeddingCache(
                maxsize=_ec_maxsize,
                ttl=_ec_ttl,
                enabled=_ec_enabled,
            )
        else:
            self.embedding_cache = None  # cachetools not installed

        # Scoring knobs
        self.vector_weight = vector_weight
        self.bm25_weight = bm25_weight
        # Read rrf_k from config if available, with constructor arg as fallback
        self.rrf_k = getattr(config, "retrieval_rrf_k", rrf_k)
        self.candidate_multiplier = candidate_multiplier
        if recency_weight is not None:
            self.recency_weight = recency_weight
        else:
            self.recency_weight = getattr(config, "retrieval_recency_weight", 0.0)

    # ------------------------------------------------------------------
    # Public API
    # ------------------------------------------------------------------

    async def retrieve(
        self,
        query: str,
        mode: str = "auto",
        constraints: Optional[RetrievalConstraints] = None,
        owner_id: str = "default",
        task_mode_context: Optional[Dict[str, str]] = None,
    ) -> RetrievalResult:
        """Main retrieval entry point — find relevant memories for a query.

        Modes:
            'lightweight': BM25 + vector hybrid, no LLM calls (<200 ms target).
            'auto': Routes to 'lightweight' (Phase 1). Phase 2 may upgrade to 'agentic'.
            'agentic': Multi-round with LLM reranking (delegates to AgenticRetrievalPipeline).
            'brain_graph': Graph-walk retrieval (delegates to AgenticRetrievalPipeline).

        Args:
            query: Natural language search query.
            mode: Retrieval strategy — 'auto', 'lightweight', 'agentic', 'brain_graph'.
            constraints: Optional RetrievalConstraints for filtering/budget.
            owner_id: Multi-tenancy scope key.
            task_mode_context: Optional dict with task-mode hints for policy-driven
                retrieval routing (#362). Keys: 'task_hint', 'session_type',
                'recent_activity'. When provided and retrieval_policy is enabled,
                the query is classified into a task mode and retrieval channels
                are selected based on the mode's policy. When None, existing
                behavior is unchanged (backward compatible).

        Returns:
            RetrievalResult with ranked items, retrieval_id, latency, and token usage.

        Note:
            Every retrieval is logged to the retrieval_log table for feedback tracking.
        """
        t0 = time.monotonic()
        retrieval_id = _new_id()
        constraints = constraints or RetrievalConstraints()

        # ------------------------------------------------------------------
        # Task-mode policy routing (#362): classify query → select policy.
        # Only active when task_mode_context is provided AND config enables it.
        # The policy is logged but does not yet alter retrieval routing
        # (Phase 1: classification + policy selection; Phase 2: channel routing).
        # ------------------------------------------------------------------
        _active_policy = None
        if task_mode_context is not None:
            try:
                from cortex.core.task_mode import TaskModeClassifier
                from cortex.core.retrieval_policy import PolicyEngine

                _policy_enabled = getattr(self.config, "retrieval_policy_enabled", True)
                if _policy_enabled:
                    _default_mode_str = getattr(
                        self.config, "retrieval_policy_default_mode", "recall_history"
                    )
                    from cortex.core.task_mode import TaskMode
                    try:
                        _default_mode = TaskMode(_default_mode_str)
                    except ValueError:
                        _default_mode = TaskMode.RECALL_HISTORY

                    classifier = TaskModeClassifier(default_mode=_default_mode)
                    mode_result = classifier.classify(query, context=task_mode_context)
                    engine = PolicyEngine(config=self.config)
                    _active_policy = engine.get_policy(mode_result)

                    logger.debug(
                        "retrieve: task_mode=%s confidence=%.3f policy=%s",
                        mode_result.mode.value,
                        mode_result.confidence,
                        _active_policy.description,
                    )
            except Exception:
                logger.debug("retrieve: task-mode policy routing failed", exc_info=True)

        # ------------------------------------------------------------------
        # Working Memory pre-fetch (#371): check in-memory cache FIRST.
        # Audit fix: WM results are PREPENDED to DB results, not a short-circuit.
        # Items are returned as RetrievedItems with item_type="working_memory"
        # and score=1.0 (working memory is authoritative for the current session).
        # ------------------------------------------------------------------
        wm_items = await self._working_memory_retrieve(query, constraints, owner_id)

        if mode == "auto":
            mode = self._route_query(query, constraints)

        # ------------------------------------------------------------------
        # Dynamic top-k + selective rerank (#495).
        #
        # Simple queries (identity, preferences) → top_k=5, no rerank.
        # Complex queries (decisions, architecture) → top_k=10, rerank ON.
        #
        # Lightweight mode ("fast" from plugin) skips rerank entirely for
        # latency — the plugin's 500ms budget cannot absorb a rerank RTT.
        # ------------------------------------------------------------------
        _rerank_cfg_enabled = getattr(self.config, "retrieval_rerank_enabled", False)
        _is_complex = _is_complex_query(query)

        # Rerank fires whenever the config flag is on.
        # (The previous selective guard — _is_complex and mode != "lightweight" —
        # was a #495 optimisation that broke the core contract: callers expect
        # rerank to engage whenever retrieval_rerank_enabled=True.  Dynamic
        # top-k and initial_candidates still apply as before.)
        _rerank_enabled = _rerank_cfg_enabled

        # Dynamic top-k: 5 for simple, 10 for complex.
        _dynamic_top_k = 10 if _is_complex else 5
        if constraints.top_k == 20:  # caller used default → apply dynamic
            from copy import copy as _copy
            constraints = _copy(constraints)
            constraints.top_k = _dynamic_top_k

        _rerank_initial_candidates = getattr(
            self.config, "retrieval_rerank_initial_candidates", 15
        )
        _retrieval_constraints = constraints
        if _rerank_enabled and _rerank_initial_candidates > constraints.top_k:
            from copy import copy as _copy
            _retrieval_constraints = _copy(constraints)
            _retrieval_constraints.top_k = _rerank_initial_candidates

        if mode == "lightweight":
            items = await self._lightweight_retrieve(query, _retrieval_constraints, owner_id)
        elif mode in ("agentic", "brain_graph"):
            # Agentic/brain_graph modes require Phase 2 pipeline.
            # If llm is available, delegate to AgenticRetrievalPipeline by re-routing
            # through _agentic_retrieve_fallback; otherwise fall back to lightweight.
            if self.llm is not None:
                from cortex.core.agentic_retrieval import AgenticRetrievalPipeline
                # Build a one-shot agentic pipeline reusing this instance's dependencies
                agentic = AgenticRetrievalPipeline(
                    storage=self.storage,
                    config=self.config,
                    token_budget=self.token_budget,
                    cornerstone_guardian=self.cornerstone_guardian,
                    llm=self.llm,
                    vector_weight=self.vector_weight,
                    bm25_weight=self.bm25_weight,
                    rrf_k=self.rrf_k,
                    candidate_multiplier=self.candidate_multiplier,
                )
                return await agentic.retrieve(
                    query, mode=mode, constraints=constraints, owner_id=owner_id
                )
            else:
                logger.warning(
                    "Mode '%s' requested but no LLM provided. Falling back to 'lightweight'.",
                    mode,
                )
                mode = "lightweight"
                items = await self._lightweight_retrieve(query, constraints, owner_id)
        else:
            # Fail-fast: unknown modes raise immediately so callers notice bugs (#346)
            raise ValueError(
                f"Invalid retrieval mode {mode!r}. "
                f"Supported modes: 'auto', 'lightweight', 'agentic', 'brain_graph'."
            )

        # Audit fix: merge WM items (prepended) with DB results, deduplicate by item_id
        if wm_items:
            seen_ids = {item.item_id for item in wm_items}
            db_deduped = [item for item in items if item.item_id not in seen_ids]
            items = wm_items + db_deduped
            mode = f"wm+{mode}"

        # ------------------------------------------------------------------
        # Rerank step (#502): Voyage rerank-2.5-lite applied after all
        # retrieval modes have produced candidates, but before token budget
        # trimming. Transparent to callers — they just get better results.
        #
        # Selective rerank (#495): only fires for complex queries outside
        # lightweight mode. Has a 400ms timeout to prevent blowing the
        # caller's latency budget.
        # ------------------------------------------------------------------
        if _rerank_enabled and items:
            try:
                items = await asyncio.wait_for(
                    self._rerank_results(query, items),
                    timeout=0.4,
                )
            except asyncio.TimeoutError:
                _rerank_top_k = getattr(self.config, "retrieval_rerank_top_k", 5)
                logger.warning(
                    "retrieve: rerank timed out after 400ms, returning top %d unranked",
                    _rerank_top_k,
                )
                items = items[:_rerank_top_k]

        # ------------------------------------------------------------------
        # Status penalty (#574): penalize contradicted/superseded claims.
        # Claims with non-active status get their score multiplied by a
        # penalty factor AFTER scoring/reranking, BEFORE final truncation.
        # This ensures stale claims don't crowd out active ones.
        # ------------------------------------------------------------------
        items = self._apply_status_penalty(items)

        # Re-sort after status penalty (scores changed), but respect
        # chronological ordering when date filters are active (the
        # _lightweight_retrieve path returns date-sorted results).
        _has_date_constraints = bool(
            constraints.before_date or constraints.after_date
        )
        if not _has_date_constraints:
            items.sort(key=lambda it: it.score, reverse=True)

        # Apply token budget
        tokens_used, items = await self._apply_budget(items, constraints)

        latency_ms = int((time.monotonic() - t0) * 1000)

        # Log every retrieval
        await self._log_retrieval(
            retrieval_id=retrieval_id,
            owner_id=owner_id,
            query=query,
            mode=mode,
            constraints=constraints,
            items=items,
            tokens_used=tokens_used,
            latency_ms=latency_ms,
        )

        return RetrievalResult(
            items=items,
            retrieval_id=retrieval_id,
            mode=mode,
            latency_ms=latency_ms,
            tokens_used=tokens_used,
        )

    async def assemble_context(
        self,
        query: str,
        owner_id: str = "default",
        cornerstone_guardian: Any = None,
        retrieval_id: Optional[str] = None,
    ) -> ContextBlock:
        """Assemble a formatted context block ready for system prompt injection.

        Cornerstones are ALWAYS injected first with a reserved token budget.
        Retrieval only uses the budget that remains after cornerstone reservation.

        Args:
            query: Natural language retrieval query.
            owner_id: Multi-tenancy scope key.
            cornerstone_guardian: Optional CornerstoneGuardian override.  When
                None, falls back to ``self.cornerstone_guardian`` (set at
                construction time via the plugin).
            retrieval_id: Optional retrieval UUID for cross-referencing.  When
                provided (e.g. from a prior ``retrieve()`` call), the same ID
                is carried into the returned ContextBlock.  When None, a fresh
                ID is generated.

        Returns:
            ContextBlock with parts ordered [cornerstones, ...retrieved items],
            total_tokens, and budget_remaining.
        """
        if retrieval_id is None:
            retrieval_id = _new_id()

        # Resolve which guardian to use (call-time arg wins over instance attr)
        guardian = cornerstone_guardian if cornerstone_guardian is not None else self.cornerstone_guardian

        # Determine total budget ceiling
        total_budget = getattr(self.config, "retrieval_token_budget", 3000)

        parts: List[Tuple[str, str, int]] = []
        total_tokens = 0

        # ------------------------------------------------------------------
        # Step 1: Inject cornerstones first with reserved budget
        # ------------------------------------------------------------------
        cornerstone_tokens = 0
        if guardian is not None:
            try:
                cs_text = await guardian.format_injection_block(owner_id=owner_id)
                if cs_text:
                    cornerstone_tokens = self._count_tokens(cs_text)
                    parts.append(("cornerstone", cs_text, cornerstone_tokens))
                    total_tokens += cornerstone_tokens
                    logger.debug(
                        "assemble_context: injected %d cornerstone tokens", cornerstone_tokens
                    )
            except Exception as exc:
                logger.warning("assemble_context: cornerstone injection failed: %s", exc)

        # ------------------------------------------------------------------
        # Step 2: Run retrieval with the remaining budget
        # ------------------------------------------------------------------
        available_for_retrieval = max(0, total_budget - cornerstone_tokens)

        if self.token_budget is not None:
            try:
                budget_obj = self.token_budget.create_budget(total=available_for_retrieval)
                available_for_retrieval = min(available_for_retrieval, budget_obj.remaining)
            except Exception as exc:
                logger.warning("Token budget creation failed, proceeding with defaults: %s", exc)

        result = await self.retrieve(
            query,
            constraints=RetrievalConstraints(token_budget=available_for_retrieval),
            owner_id=owner_id,
        )

        # ------------------------------------------------------------------
        # Step 3: Append retrieved items (cornerstones already guaranteed above)
        # ------------------------------------------------------------------
        for item in result.items:
            item_tokens = self._count_tokens(item.content)
            if total_tokens + item_tokens > total_budget:
                break
            parts.append((item.item_type, item.content, item_tokens))
            total_tokens += item_tokens

        return ContextBlock(
            parts=parts,
            total_tokens=total_tokens,
            budget_remaining=total_budget - total_tokens,
            retrieval_id=retrieval_id,
        )

    async def explain(self, retrieval_id: str) -> Dict[str, Any]:
        """Return provenance and scoring details for a retrieval (stub for Phase 1)."""
        return {"retrieval_id": retrieval_id, "note": "Full explain available in Phase 2"}

    # ------------------------------------------------------------------
    # Status penalty (#574)
    # ------------------------------------------------------------------

    # Default penalty multipliers for non-active claim statuses.
    _DEFAULT_STATUS_PENALTIES: Dict[str, float] = {
        "active": 1.0,
        "superseded": 0.1,
        "deprecated": 0.3,
        "cooling": 0.7,
    }

    def _apply_status_penalty(
        self,
        items: List[RetrievedItem],
    ) -> List[RetrievedItem]:
        """Apply score penalties to claims with non-active status.

        Claims that have been superseded, deprecated, or are in cooling
        should not compete equally with active claims. This multiplies
        each item's score by a penalty factor based on its status.

        Non-claim items (working_memory, event, etc.) are unaffected.
        Claims without metadata or without a status field default to 'active'.

        The penalty map is read from config (retrieval_status_penalties)
        with fallback to _DEFAULT_STATUS_PENALTIES.

        Args:
            items: List of RetrievedItems (may be mutated in place).

        Returns:
            The same list with scores adjusted.
        """
        penalties = getattr(
            self.config, "retrieval_status_penalties", None
        ) or self._DEFAULT_STATUS_PENALTIES

        for item in items:
            if item.item_type != "claim":
                continue
            if not item.metadata:
                continue
            status = item.metadata.get("status", "active")
            penalty = penalties.get(status, 0.5)  # unknown status → conservative penalty
            if penalty < 1.0:
                item.score *= penalty

        return items

    # ------------------------------------------------------------------
    # Working Memory fast-path (#371)
    # ------------------------------------------------------------------

    async def _working_memory_retrieve(
        self,
        query: str,
        constraints: RetrievalConstraints,
        owner_id: str,
    ) -> List[RetrievedItem]:
        """Check working memory cache before any DB queries.

        Returns a (possibly empty) list of RetrievedItems from the cache.
        When the list is non-empty, retrieve() returns immediately without
        touching the DB.

        Uses a simple substring query over the session's cached entries.
        Working memory items are assigned score=1.0 (authoritative).

        Args:
            query: Natural language query text.
            constraints: RetrievalConstraints (session_id extracted from owner_id
                         for now — future: expose session_id in constraints).
            owner_id: Used as session_id proxy when no explicit session available.

        Returns:
            List of RetrievedItem, possibly empty.
        """
        if self.working_memory is None or not self.working_memory.enabled:
            return []

        # Use owner_id as the session_id proxy.
        # Phase 2: RetrievalConstraints will carry an explicit session_id.
        session_id = owner_id

        try:
            # Audit fix: removed dead import of WorkingMemoryCategory
            matches = await self.working_memory.query(
                query,
                session_id=session_id,
            )
        except Exception as exc:
            logger.warning("Working memory query failed (non-fatal): %s", exc)
            return []

        if not matches:
            return []

        items: List[RetrievedItem] = []
        for entry in matches[: (constraints.top_k or 20)]:
            content = str(entry.value)
            items.append(
                RetrievedItem(
                    item_type="working_memory",
                    item_id=f"wm:{entry.session_id}:{entry.key}",
                    content=content,
                    score=1.0,
                    provenance=f"working_memory:{entry.category}",
                )
            )

        return items

    # ------------------------------------------------------------------
    # Query routing
    # ------------------------------------------------------------------

    def _route_query(
        self,
        query: str,
        constraints: RetrievalConstraints,
    ) -> str:
        """
        Simple heuristic router for Phase 1.
        - Short queries with no stopwords → lightweight (keyword-only would suffice)
        - Everything else → lightweight (hybrid)

        Phase 2 (AgenticRetrievalPipeline) overrides this to upgrade complex queries
        to 'agentic' when an LLM is available.
        """
        words = query.lower().split()
        has_semantic_cues = any(
            w in words for w in ("explain", "why", "how", "what", "when", "who", "describe")
        )
        return "lightweight"  # Phase 1 base: always lightweight; Phase 2 overrides

    # ------------------------------------------------------------------
    # Lightweight retrieval (BM25 + vector + RRF)
    # ------------------------------------------------------------------

    async def _lightweight_retrieve(
        self,
        query: str,
        constraints: RetrievalConstraints,
        owner_id: str,
    ) -> List[RetrievedItem]:
        """
        Parallel BM25 + vector search, fused with RRF.
        """
        top_k = constraints.top_k
        candidate_k = top_k * self.candidate_multiplier

        # Run BM25 and vector search in parallel
        bm25_task = asyncio.create_task(
            self._bm25_search(query, top_k=candidate_k, owner_id=owner_id)
        )
        vector_task = asyncio.create_task(
            self._vector_search(query, top_k=candidate_k, owner_id=owner_id)
        )

        bm25_results, vector_results = await asyncio.gather(bm25_task, vector_task)

        # Build ranked lists for RRF
        # BM25: FTS5 bm25() returns negative values (lower = better), normalise
        bm25_ranked = _normalise_bm25(bm25_results)
        vector_ranked = [(r.item_id, r.score) for r in vector_results]

        # RRF fusion
        fused = rrf_fusion([bm25_ranked, vector_ranked], k=self.rrf_k)

        # Build a content lookup from both result sets.
        # Prefer non-empty content: BM25 results carry full text while
        # vector results may have empty content (VectorResult has no content
        # field), so we only overwrite when the new entry has actual content.
        content_map: Dict[str, Tuple[str, str]] = {}  # item_id → (item_type, content)
        for r in bm25_results:
            content_map[r.item_id] = (r.item_type, r.content)
        for r in vector_results:
            new_content = _get_content(r)
            if r.item_id not in content_map or new_content:
                content_map[r.item_id] = (r.item_type, new_content)

        # Hydrate empty content from DB (vector-only results lack content).
        # Also cache fetched claims so we can attach metadata to RetrievedItems.
        claim_cache: Dict[str, Any] = {}  # item_id → SemanticClaim
        for item_id, (item_type, content) in list(content_map.items()):
            if not content and item_type == "claim":
                try:
                    claim = await self.storage.get_claim(item_id)
                    if claim:
                        claim_cache[item_id] = claim
                        content_map[item_id] = (
                            item_type,
                            _render_claim_content(claim),
                        )
                except Exception as exc:
                    logger.debug("Failed to hydrate claim %s: %s", item_id[:8], exc)

        # Fetch metadata for claim items that already had BM25/FTS content
        # (weren't fetched above) so we can attach memory_class/salience/status.
        claim_ids_needing_meta = [
            iid
            for iid, (itype, _) in content_map.items()
            if itype == "claim" and iid not in claim_cache
        ]
        for item_id in claim_ids_needing_meta:
            try:
                claim = await self.storage.get_claim(item_id)
                if claim:
                    claim_cache[item_id] = claim
            except Exception as exc:
                logger.debug("Failed to fetch claim meta %s: %s", item_id[:8], exc)

        # --- Sensitivity filtering (issue #272) ---
        claim_ids_in_map = [
            iid for iid, (itype, _) in content_map.items() if itype == "claim"
        ]
        sensitivity_map: Dict[str, str] = {}
        if claim_ids_in_map:
            try:
                sensitivity_map = await self.storage.get_claims_sensitivity_batch(
                    claim_ids_in_map
                )
            except Exception as exc:
                logger.warning(
                    "Sensitivity batch failed (%s); excluding all claims (fail-closed).",
                    exc,
                )

        max_level = _sensitivity_level(constraints.sensitivity_max)

        def _passes_sensitivity(item_id: str, item_type: str) -> bool:
            if item_type != "claim":
                return True
            sens = sensitivity_map.get(item_id)
            return sens is not None and _sensitivity_level(sens) <= max_level

        # ------------------------------------------------------------------
        # Temporal scoring (#513): parse query for date hints, boost claims
        # with nearby event_date, apply before/after date filters.
        # ------------------------------------------------------------------
        _temporal_boost_weight = getattr(
            self.config, "retrieval_temporal_boost_weight", 0.15
        )
        _has_date_filters = bool(constraints.before_date or constraints.after_date)

        _reference_date: Optional[datetime] = None
        if _temporal_boost_weight > 0:
            try:
                from cortex.core.temporal_scoring import parse_temporal_hints
                _reference_date = parse_temporal_hints(query)
            except Exception as _te:
                logger.debug("temporal hint parse failed (non-fatal): %s", _te)

        _date_filter_fn = None
        if _has_date_filters:
            try:
                from cortex.core.temporal_scoring import filter_by_date
                _before = constraints.before_date
                _after = constraints.after_date
                _date_filter_fn = lambda ed: filter_by_date(ed, _before, _after)  # noqa: E731
            except Exception as _te:
                logger.debug("temporal date filter import failed (non-fatal): %s", _te)

        _proximity_boost_fn = None
        if _reference_date is not None and _temporal_boost_weight > 0:
            try:
                from cortex.core.temporal_scoring import temporal_proximity_boost
                _ref = _reference_date
                _weight = _temporal_boost_weight
                _proximity_boost_fn = lambda ed: _weight * temporal_proximity_boost(ed, _ref)  # noqa: E731
            except Exception as _te:
                logger.debug("temporal proximity import failed (non-fatal): %s", _te)

        # Apply memory_types + sensitivity + temporal date filters
        items: List[RetrievedItem] = []
        for item_id, score in fused[:top_k]:
            if item_id not in content_map:
                continue
            item_type, content = content_map[item_id]
            if constraints.memory_types and item_type not in constraints.memory_types:
                continue
            if not _passes_sensitivity(item_id, item_type):
                continue

            # Attach LoCoMo classification metadata for claim items so the
            # context compiler can use memory_class/salience/status for
            # prioritisation and formatting without re-fetching from DB.
            item_metadata: Optional[Dict[str, Any]] = None
            _event_date: Optional[str] = None
            if item_type == "claim":
                cached = claim_cache.get(item_id)
                if cached is not None:
                    item_metadata = {
                        "memory_class": getattr(cached, "memory_class", "semantic"),
                        "salience": getattr(cached, "salience", "medium"),
                        "status": getattr(cached, "status", "active"),
                    }
                    _event_date = getattr(cached, "event_date", None)

            # Apply date range filter (only for claims that have event_date)
            if _date_filter_fn is not None and not _date_filter_fn(_event_date):
                continue

            # Apply temporal proximity boost to score
            final_score = score
            if _proximity_boost_fn is not None and _event_date:
                final_score = score + _proximity_boost_fn(_event_date)

            items.append(
                RetrievedItem(
                    item_type=item_type,
                    item_id=item_id,
                    content=content,
                    score=final_score,
                    metadata=item_metadata,
                )
            )

        # When date filters are active, sort chronologically by event_date.
        # Items without event_date go last (stable sort preserves relevance
        # ordering within the undated group).
        if _has_date_filters and items:
            def _sort_key(item: RetrievedItem) -> str:
                if item.metadata and item.item_type == "claim":
                    ed = (claim_cache.get(item.item_id) and
                          getattr(claim_cache.get(item.item_id), "event_date", None))
                    if ed:
                        return str(ed)
                return "9999-99-99"  # undated items sort last

            items.sort(key=_sort_key)

        return items

    # ------------------------------------------------------------------
    # BM25 search via FTS5
    # ------------------------------------------------------------------

    async def _bm25_search(self, query: str, top_k: int, owner_id: str) -> List[_FTSResult]:
        """
        Search both fts_claims and fts_episodes, merge, return top_k.
        """
        results: List[_FTSResult] = []
        for table in ("claims", "episodes"):
            try:
                raw = await self.storage.fts_search(query, table=table, top_k=top_k, owner_id=owner_id)
                for r in raw:
                    results.append(
                        _FTSResult(
                            item_type=_table_to_item_type(table),
                            item_id=_get_id(r),
                            content=_get_content(r),
                            rank=_get_rank(r),
                        )
                    )
            except Exception as exc:
                logger.warning("BM25 search failed for table=%s: %s", table, exc)

        # Sort by rank (FTS5 bm25() is negative — lower/more-negative = better match)
        results.sort(key=lambda r: r.rank)
        return results[:top_k]

    # ------------------------------------------------------------------
    # Vector search (sqlite-vec or cosine fallback)
    # ------------------------------------------------------------------

    async def _vector_search(
        self,
        query: str,
        top_k: int,
        owner_id: str,
    ) -> List[_VectorResult]:
        """
        Embed the query, then ask storage for nearest neighbours.
        Falls back to brute-force cosine if sqlite-vec unavailable.
        """
        embedding = await self._embed(query)
        if embedding is None:
            # No embedding available — return empty so BM25 carries the load
            logger.info("No embedding available; skipping vector search")
            return []

        try:
            raw = await self.storage.vector_search(
                embedding, top_k=top_k, owner_id=owner_id
            )
            return [
                _VectorResult(
                    item_type=_get_item_type(r),
                    item_id=_get_id(r),
                    content=_get_content(r),
                    score=_get_score(r),
                )
                for r in raw
            ]
        except NotImplementedError:
            # Storage doesn't support vector_search — try cosine fallback
            logger.warning(
                "sqlite-vec not available; falling back to brute-force cosine similarity. "
                "Install sqlite-vec for faster ANN vector search."
            )
            return await self._cosine_fallback(embedding, top_k)
        except Exception as exc:
            logger.warning("Vector search failed: %s", exc)
            return []

    async def _cosine_fallback(
        self,
        query_embedding: List[float],
        top_k: int,
    ) -> List[_VectorResult]:
        """
        Brute-force cosine similarity over embedding_index table.
        Used when sqlite-vec extension is unavailable.
        """
        try:
            raw_embeddings = await self.storage.get_all_embeddings()  # type: ignore[attr-defined]
        except Exception as exc:
            logger.warning("Cosine fallback: get_all_embeddings failed: %s", exc)
            return []

        scored: List[Tuple[float, Any]] = []
        for row in raw_embeddings:
            stored_vec = _decode_embedding(row)
            if stored_vec is None:
                continue
            try:
                sim = cosine_similarity(query_embedding, stored_vec)
            except ValueError:
                continue
            scored.append((sim, row))

        scored.sort(key=lambda x: x[0], reverse=True)
        results = []
        for sim, row in scored[:top_k]:
            results.append(
                _VectorResult(
                    item_type=_get_item_type(row),
                    item_id=_get_id(row),
                    content=_get_content(row),
                    score=sim,
                )
            )
        return results

    # ------------------------------------------------------------------
    # Embedding
    # ------------------------------------------------------------------

    async def _embed(self, text: str) -> Optional[List[float]]:
        """Produce an embedding for *text*, using the cache when available.

        Lookup order:
            1. EmbeddingCache (in-memory, ~0ms on hit).
            2. StorageAdapter.embed() — some adapters bundle their own model.
            3. LLMProvider.embed() — Voyage / OpenAI / etc. (~150ms).

        On a cache miss the result from step 2 or 3 is stored in the cache
        before returning, so subsequent calls for the same text are free.

        Args:
            text: Raw query or document text to embed.

        Returns:
            Float embedding vector, or ``None`` if no embedding source is
            available (vector search will be skipped gracefully).
        """
        # ------------------------------------------------------------------
        # 1. Cache lookup
        # ------------------------------------------------------------------
        if self.embedding_cache is not None:
            cached = self.embedding_cache.get(text)
            if cached is not None:
                return cached

        # ------------------------------------------------------------------
        # 2. Storage adapter embed (wraps embedding model in some deployments)
        # ------------------------------------------------------------------
        if hasattr(self.storage, "embed"):
            try:
                result = await self.storage.embed(text)  # type: ignore[attr-defined]
                if result is not None and self.embedding_cache is not None:
                    self.embedding_cache.set(text, result)
                return result
            except Exception as exc:
                logger.debug("storage.embed failed: %s", exc)

        # ------------------------------------------------------------------
        # 3. LLM provider embed (Voyage, OpenAI, etc.)
        # ------------------------------------------------------------------
        if self.llm is not None and hasattr(self.llm, "embed"):
            try:
                _embed_resp = await self.llm.embed(texts=[text])
                result = (
                    _embed_resp.embeddings[0]
                    if hasattr(_embed_resp, "embeddings")
                    else _embed_resp
                )
                if result is not None and self.embedding_cache is not None:
                    self.embedding_cache.set(text, result)
                return result
            except Exception as exc:
                logger.debug("llm.embed failed: %s", exc)

        return None

    # ------------------------------------------------------------------
    # Token budget enforcement
    # ------------------------------------------------------------------

    async def _apply_budget(
        self,
        items: List[RetrievedItem],
        constraints: RetrievalConstraints,
    ) -> Tuple[int, List[RetrievedItem]]:
        """
        Trim items list so total token count stays within budget.
        Returns (tokens_used, trimmed_items).
        """
        # Determine ceiling
        if constraints.token_budget is not None:
            budget = constraints.token_budget
        else:
            budget = getattr(self.config, "retrieval_token_budget", 3000)

        trimmed: List[RetrievedItem] = []
        tokens_used = 0

        for item in items:
            item_tokens = self._count_tokens(item.content)
            if tokens_used + item_tokens > budget:
                logger.debug(
                    "Budget exhausted at %d/%d tokens — dropping remaining %d items",
                    tokens_used, budget, len(items) - len(trimmed),
                )
                break
            trimmed.append(item)
            tokens_used += item_tokens

        return tokens_used, trimmed

    async def _rerank_results(
        self,
        query: str,
        items: List[RetrievedItem],
    ) -> List[RetrievedItem]:
        """Rerank retrieved items using Voyage rerank-2.5-lite (#502).

        Calls the Voyage rerank API with the retrieved items' content, maps
        results back to the original RetrievedItem objects, and returns the
        top-K by reranked relevance score.

        Requires VOYAGE_API_KEY env var (same key used for embeddings).

        On any failure, logs a warning and returns items unchanged so the
        caller always gets results (graceful degradation).

        Args:
            query: The original retrieval query string.
            items: Candidate items to rerank (already merged WM + DB results).

        Returns:
            Reranked list trimmed to retrieval_rerank_top_k, or the original
            list if reranking fails.
        """
        rerank_model = getattr(self.config, "retrieval_rerank_model", "rerank-2.5-lite")
        rerank_top_k = getattr(self.config, "retrieval_rerank_top_k", 5)

        # Extract document texts for the rerank API.
        # Use item content; skip items with empty content (they can't be ranked).
        documents = [item.content for item in items]
        non_empty_mask = [bool(doc.strip()) for doc in documents]

        if not any(non_empty_mask):
            logger.debug("_rerank_results: all items have empty content, skipping rerank")
            return items[:rerank_top_k]

        t0 = time.monotonic()
        try:
            if voyageai is None:
                logger.warning(
                    "_rerank_results: voyageai package not installed; skipping rerank. "
                    "Install with: pip install voyageai"
                )
                return items[:rerank_top_k]

            api_key = os.environ.get("VOYAGE_API_KEY")
            if not api_key:
                logger.warning(
                    "_rerank_results: VOYAGE_API_KEY not set, skipping rerank"
                )
                return items[:rerank_top_k]

            client = voyageai.AsyncClient(api_key=api_key)
            rerank_result = await client.rerank(
                query=query,
                documents=documents,
                model=rerank_model,
                top_k=min(rerank_top_k, len(documents)),
            )

            latency_ms = int((time.monotonic() - t0) * 1000)
            logger.debug(
                "_rerank_results: reranked %d → %d items with %s in %dms",
                len(items),
                len(rerank_result.results),
                rerank_model,
                latency_ms,
            )

            # Map reranked results back to RetrievedItem objects.
            # rerank_result.results is sorted best-first by the API;
            # we also cap to rerank_top_k as a safety net.
            reranked: List[RetrievedItem] = []
            for rr in rerank_result.results[:rerank_top_k]:
                original_item = items[rr.index]
                reranked.append(
                    RetrievedItem(
                        item_type=original_item.item_type,
                        item_id=original_item.item_id,
                        content=original_item.content,
                        score=float(rr.relevance_score),
                        provenance=f"rerank:{rerank_model}",
                    )
                )

            return reranked

        except Exception as exc:
            latency_ms = int((time.monotonic() - t0) * 1000)
            logger.warning(
                "_rerank_results: rerank failed after %dms (%s: %s); "
                "returning unranked results",
                latency_ms,
                type(exc).__name__,
                exc,
            )
            # Graceful fallback: return original items trimmed to top_k
            return items[:rerank_top_k]

    def _count_tokens(self, text: str) -> int:
        """Count tokens — delegate to TokenBudgetManager if available, else word-approx."""
        if self.token_budget is not None and hasattr(self.token_budget, "count_tokens"):
            try:
                return self.token_budget.count_tokens(text)
            except Exception as e:
                logger.warning("cortex_core_retrieval: unhandled exception: %s", e)
        # Fallback: ~1.3 tokens per word (rough BPE approximation)
        return max(1, int(len(text.split()) * 1.3))

    # ------------------------------------------------------------------
    # Retrieval logging
    # ------------------------------------------------------------------

    async def _log_retrieval(
        self,
        *,
        retrieval_id: str,
        owner_id: str,
        query: str,
        mode: str,
        constraints: RetrievalConstraints,
        items: List[RetrievedItem],
        tokens_used: int,
        latency_ms: int,
    ) -> None:
        """Log retrieval metadata to retrieval_log table."""
        import json

        selected = [
            {"item_id": it.item_id, "item_type": it.item_type, "score": it.score}
            for it in items
        ]
        context_hash = hashlib.sha256(
            json.dumps(selected, sort_keys=True).encode()
        ).hexdigest()[:16]

        try:
            await self.storage.log_retrieval(
                id=retrieval_id,
                owner_id=owner_id,
                query_text=query,
                mode=mode,
                token_budget=constraints.token_budget or getattr(
                    self.config, "retrieval_token_budget", 3000
                ),
                tokens_used=tokens_used,
                selected_json=json.dumps(selected),
                final_context_hash=context_hash,
                latency_ms=latency_ms,
                created_at=datetime.now(timezone.utc).isoformat(),
            )
        except Exception as exc:
            logger.warning("Failed to log retrieval %s: %s", retrieval_id, exc)


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _new_id() -> str:
    return uuid.uuid4().hex


def _normalise_bm25(
    results: List[_FTSResult],
) -> List[Tuple[str, float]]:
    """Convert FTS5 bm25() scores to [0,1] similarity scores for RRF input.

    SQLite FTS5's bm25() returns negative values where more negative = better match.
    We negate to make larger = better, then min-max normalise to [0, 1] so the
    scores are compatible with vector similarity scores in RRF fusion.

    Args:
        results: List of FTS results with .rank (negative bm25 scores).

    Returns:
        List of (item_id, normalised_score) tuples, higher = better match.
    """
    if not results:
        return []
    # bm25() values are ≤ 0; negate so larger = better, then min-max normalise
    ranks = [-r.rank for r in results]  # now all ≥ 0
    min_r, max_r = min(ranks), max(ranks)
    span = max_r - min_r or 1.0
    normalised = [(r.item_id, (rank - min_r) / span) for r, rank in zip(results, ranks)]
    return normalised


def _table_to_item_type(table: str) -> str:
    return {"claims": "claim", "episodes": "event"}.get(table, table)


# The following helpers handle both dataclass objects and plain dicts/namedtuples
# returned by different StorageAdapter implementations.

def _get_id(obj: Any) -> str:
    if hasattr(obj, "item_id"):
        return obj.item_id
    if hasattr(obj, "claim_id"):
        return obj.claim_id
    if hasattr(obj, "event_id"):
        return obj.event_id
    if isinstance(obj, dict):
        return obj.get("item_id") or obj.get("claim_id") or obj.get("event_id") or ""
    return str(getattr(obj, "id", ""))


def _get_content(obj: Any) -> str:
    if hasattr(obj, "content"):
        return obj.content
    if isinstance(obj, dict):
        return obj.get("content", "")
    return ""


def _get_rank(obj: Any) -> float:
    for attr in ("rank", "bm25_rank", "score"):
        if hasattr(obj, attr):
            return float(getattr(obj, attr))
    if isinstance(obj, dict):
        return float(obj.get("rank", obj.get("score", 0.0)))
    return 0.0


def _get_score(obj: Any) -> float:
    if hasattr(obj, "score"):
        return float(obj.score)
    if isinstance(obj, dict):
        return float(obj.get("score", 0.0))
    return 0.0


def _get_item_type(obj: Any) -> str:
    if hasattr(obj, "item_type"):
        return obj.item_type
    if isinstance(obj, dict):
        return obj.get("item_type", "claim")
    return "claim"


def _decode_embedding(row: Any) -> Optional[List[float]]:
    """Decode a stored embedding blob or list back to a Python float list."""
    raw = getattr(row, "embedding", None) or (
        row.get("embedding") if isinstance(row, dict) else None
    )
    if raw is None:
        return None
    if isinstance(raw, (list, tuple)):
        return [float(x) for x in raw]
    if isinstance(raw, (bytes, bytearray)):
        import struct
        n = len(raw) // 4
        return list(struct.unpack(f"{n}f", raw[:n * 4]))
    return None
