"""Hybrid retrieval combining BM25, semantic similarity, and activation.

This module implements tri-hybrid retrieval with staged architecture:
- Stage 1: BM25 filtering (keyword exact match, top_k=100)
- Stage 2: Re-ranking with tri-hybrid scoring:
  * BM25 keyword matching (30% weight by default)
  * Semantic similarity (40% weight by default)
  * Activation-based ranking (30% weight by default)

Performance optimizations (Epic 1 + Epic 2):
- Lazy BM25 index loading: Deferred until first retrieve() call (99.9% faster creation)
- Query embedding cache (LRU, configurable size)
- Persistent BM25 index (load once, rebuild on reindex)
- Activation score caching via CacheManager
- Dual-hybrid fallback: BM25+Activation when embeddings unavailable (85% quality vs 95% tri-hybrid)

Classes:
    HybridConfig: Configuration for hybrid retrieval weights
    HybridRetriever: Main hybrid retrieval implementation with BM25
"""

import hashlib
import json
import logging
import os
import pickle
import threading
import time
from collections import OrderedDict
from dataclasses import dataclass
from pathlib import Path
from typing import Any

import numpy as np
import numpy.typing as npt

logger = logging.getLogger(__name__)


# Module-level cache for HybridRetriever instances
# Cache stores (HybridRetriever, timestamp) tuples keyed by (db_path, config_hash)
_retriever_cache: dict[tuple[str, str], tuple["HybridRetriever", float]] = {}
_retriever_cache_lock = threading.Lock()
_retriever_cache_stats = {"hits": 0, "misses": 0}

# Cache configuration from environment variables
_RETRIEVER_CACHE_SIZE = int(os.environ.get("AURORA_RETRIEVER_CACHE_SIZE", "10"))
_RETRIEVER_CACHE_TTL = int(os.environ.get("AURORA_RETRIEVER_CACHE_TTL", "1800"))  # 30 minutes


@dataclass
class HybridConfig:
    """Configuration for tri-hybrid retrieval.

    Supports two modes:
    1. Dual-hybrid (legacy): activation + semantic (weights sum to 1.0)
    2. Tri-hybrid (default): BM25 + activation + semantic (weights sum to 1.0)

    Attributes:
        bm25_weight: Weight for BM25 keyword score (default 0.3, use 0.0 for dual-hybrid)
        activation_weight: Weight for activation score (default 0.3, or 0.6 for dual-hybrid)
        semantic_weight: Weight for semantic similarity (default 0.4)
        activation_top_k: Number of top chunks to retrieve by activation (default 100)
        stage1_top_k: Number of candidates to pass from Stage 1 BM25 filter (default 100)
        fallback_to_activation: If True, fall back to activation-only if embeddings unavailable
        use_staged_retrieval: Enable staged retrieval (BM25 filter → re-rank)
        bm25_index_path: Path to persist BM25 index (default: .aurora/indexes/bm25_index.pkl)
        mmr_lambda: MMR diversity parameter (0.0=pure diversity, 1.0=pure relevance, default 0.5)

    Example (tri-hybrid):
        >>> config = HybridConfig(bm25_weight=0.3, activation_weight=0.3, semantic_weight=0.4)
        >>> retriever = HybridRetriever(store, engine, provider, config)

    Example (dual-hybrid, legacy):
        >>> config = HybridConfig(bm25_weight=0.0, activation_weight=0.6, semantic_weight=0.4)
        >>> retriever = HybridRetriever(store, engine, provider, config)

    Example (with diversity):
        >>> results = retriever.retrieve("auth", top_k=10, diverse=True, mmr_lambda=0.7)

    """

    bm25_weight: float = 0.3
    activation_weight: float = 0.3
    semantic_weight: float = 0.4
    activation_top_k: int = 500  # Increased from 100 to improve recall on large repos
    stage1_top_k: int = 100
    fallback_to_activation: bool = True
    use_staged_retrieval: bool = True
    # Caching configuration
    enable_query_cache: bool = True
    query_cache_size: int = 100
    query_cache_ttl_seconds: int = 1800  # 30 minutes
    # BM25 index persistence (path relative to project root or absolute)
    bm25_index_path: str | None = None
    # Enable persistent BM25 index (load from disk if available)
    enable_bm25_persistence: bool = True
    # Code-first ordering: boost score for code type chunks (0.0-0.5, default 0.0)
    # Disabled by default - type-aware retrieval handles code/kb separation
    code_type_boost: float = 0.0
    # MMR (Maximal Marginal Relevance) configuration
    # Default lambda=0.5 balances relevance and diversity
    mmr_lambda: float = 0.5

    def __post_init__(self) -> None:
        """Validate configuration."""
        if not (0.0 <= self.bm25_weight <= 1.0):
            raise ValueError(f"bm25_weight must be in [0, 1], got {self.bm25_weight}")
        if not (0.0 <= self.activation_weight <= 1.0):
            raise ValueError(f"activation_weight must be in [0, 1], got {self.activation_weight}")
        if not (0.0 <= self.semantic_weight <= 1.0):
            raise ValueError(f"semantic_weight must be in [0, 1], got {self.semantic_weight}")

        total_weight = self.bm25_weight + self.activation_weight + self.semantic_weight
        if abs(total_weight - 1.0) > 1e-6:
            raise ValueError(
                f"Weights must sum to 1.0, got {total_weight} "
                f"(bm25={self.bm25_weight}, activation={self.activation_weight}, semantic={self.semantic_weight})",
            )

        if self.activation_top_k < 1:
            raise ValueError(f"activation_top_k must be >= 1, got {self.activation_top_k}")
        if self.stage1_top_k < 1:
            raise ValueError(f"stage1_top_k must be >= 1, got {self.stage1_top_k}")
        if self.query_cache_size < 1:
            raise ValueError(f"query_cache_size must be >= 1, got {self.query_cache_size}")
        if self.query_cache_ttl_seconds < 0:
            raise ValueError(
                f"query_cache_ttl_seconds must be >= 0, got {self.query_cache_ttl_seconds}",
            )
        if not (0.0 <= self.code_type_boost <= 0.5):
            raise ValueError(
                f"code_type_boost must be in [0, 0.5], got {self.code_type_boost}",
            )
        if not (0.0 <= self.mmr_lambda <= 1.0):
            raise ValueError(
                f"mmr_lambda must be in [0, 1], got {self.mmr_lambda}",
            )


@dataclass
class CacheStats:
    """Statistics for query embedding cache."""

    hits: int = 0
    misses: int = 0
    evictions: int = 0

    @property
    def hit_rate(self) -> float:
        """Calculate cache hit rate."""
        total = self.hits + self.misses
        return self.hits / total if total > 0 else 0.0


class QueryEmbeddingCache:
    """LRU cache for query embeddings with TTL support.

    Caches query embeddings to avoid repeated embedding generation for
    identical or similar queries. Uses normalized query as key.

    Attributes:
        capacity: Maximum number of cached embeddings
        ttl_seconds: Time-to-live for cached entries
        stats: Cache statistics (hits, misses, evictions)

    """

    def __init__(self, capacity: int = 100, ttl_seconds: int = 1800):
        """Initialize query embedding cache.

        Args:
            capacity: Maximum cached embeddings (default 100)
            ttl_seconds: TTL in seconds (default 1800 = 30 min)

        """
        self.capacity = capacity
        self.ttl_seconds = ttl_seconds
        self._cache: OrderedDict[str, tuple[npt.NDArray[np.float32], float]] = OrderedDict()
        self.stats = CacheStats()

    def _normalize_query(self, query: str) -> str:
        """Normalize query for cache key.

        Args:
            query: Raw query string

        Returns:
            Normalized query (lowercase, stripped, single spaces)

        """
        return " ".join(query.lower().split())

    def _make_key(self, query: str) -> str:
        """Create cache key from query.

        Args:
            query: Query string

        Returns:
            Hash-based cache key

        """
        normalized = self._normalize_query(query)
        return hashlib.md5(normalized.encode(), usedforsecurity=False).hexdigest()

    def get(self, query: str) -> npt.NDArray[np.float32] | None:
        """Get cached embedding for query.

        Args:
            query: Query string

        Returns:
            Cached embedding if found and not expired, None otherwise

        """
        key = self._make_key(query)

        if key not in self._cache:
            self.stats.misses += 1
            return None

        embedding, timestamp = self._cache[key]

        # Check TTL
        if time.time() - timestamp > self.ttl_seconds:
            del self._cache[key]
            self.stats.misses += 1
            return None

        # Move to end (most recently used)
        self._cache.move_to_end(key)
        self.stats.hits += 1
        return embedding

    def set(self, query: str, embedding: npt.NDArray[np.float32]) -> None:
        """Cache embedding for query.

        Args:
            query: Query string
            embedding: Query embedding to cache

        """
        key = self._make_key(query)

        # Remove if exists (will re-add at end)
        if key in self._cache:
            del self._cache[key]
        # Evict LRU if at capacity
        elif len(self._cache) >= self.capacity:
            self._cache.popitem(last=False)
            self.stats.evictions += 1

        self._cache[key] = (embedding, time.time())

    def clear(self) -> None:
        """Clear all cached embeddings."""
        self._cache.clear()
        self.stats = CacheStats()

    def size(self) -> int:
        """Get current cache size."""
        return len(self._cache)


# Module-level shared query embedding cache
_shared_query_cache: QueryEmbeddingCache | None = None
_shared_query_cache_lock = threading.Lock()


def get_shared_query_cache(capacity: int = 100, ttl_seconds: int = 1800) -> QueryEmbeddingCache:
    """Get or create shared QueryEmbeddingCache instance.

    Returns a singleton QueryEmbeddingCache that is shared across all
    HybridRetriever instances. This allows query embeddings to be reused
    even when different retrievers are created (e.g., in SOAR phases).

    Note: The first call to this function sets the capacity and TTL.
    Subsequent calls with different parameters will return the existing cache
    with its original settings (capacity/TTL cannot be changed after creation).

    Args:
        capacity: Maximum cached embeddings (default 100)
        ttl_seconds: TTL in seconds (default 1800 = 30 min)

    Returns:
        Shared QueryEmbeddingCache singleton

    """
    global _shared_query_cache

    with _shared_query_cache_lock:
        if _shared_query_cache is None:
            logger.debug(
                f"Creating shared QueryEmbeddingCache (capacity={capacity}, ttl={ttl_seconds}s)"
            )
            _shared_query_cache = QueryEmbeddingCache(capacity=capacity, ttl_seconds=ttl_seconds)
        elif (
            _shared_query_cache.capacity != capacity
            or _shared_query_cache.ttl_seconds != ttl_seconds
        ):
            # Warn if requesting different settings than existing cache
            logger.debug(
                f"Shared QueryEmbeddingCache already exists "
                f"(capacity={_shared_query_cache.capacity}, "
                f"ttl={_shared_query_cache.ttl_seconds}s), "
                f"ignoring requested capacity={capacity}, ttl={ttl_seconds}s"
            )
        return _shared_query_cache


def clear_shared_query_cache() -> None:
    """Clear the shared QueryEmbeddingCache singleton.

    This is primarily for testing purposes, to reset the cache between tests.
    """
    global _shared_query_cache

    with _shared_query_cache_lock:
        _shared_query_cache = None
        logger.debug("Cleared shared QueryEmbeddingCache")


def _compute_config_hash(config: "HybridConfig") -> str:
    """Compute MD5 hash of config for cache key.

    Args:
        config: HybridConfig instance

    Returns:
        MD5 hash of config as hex string

    """
    # Convert config to dict and sort keys for deterministic hashing
    config_dict = {
        "bm25_weight": config.bm25_weight,
        "activation_weight": config.activation_weight,
        "semantic_weight": config.semantic_weight,
        "activation_top_k": config.activation_top_k,
        "stage1_top_k": config.stage1_top_k,
        "fallback_to_activation": config.fallback_to_activation,
        "use_staged_retrieval": config.use_staged_retrieval,
        "enable_query_cache": config.enable_query_cache,
        "query_cache_size": config.query_cache_size,
        "query_cache_ttl_seconds": config.query_cache_ttl_seconds,
        "enable_bm25_persistence": config.enable_bm25_persistence,
    }
    config_json = json.dumps(config_dict, sort_keys=True)
    return hashlib.md5(config_json.encode(), usedforsecurity=False).hexdigest()


def get_cached_retriever(
    store: Any,
    activation_engine: Any,
    embedding_provider: Any,
    config: "HybridConfig | None" = None,
    aurora_config: Any | None = None,
) -> "HybridRetriever":
    """Get or create cached HybridRetriever instance.

    Returns cached retriever if one exists for the given db_path and config,
    otherwise creates a new one and caches it. Thread-safe with LRU eviction.

    Args:
        store: Storage backend (must have db_path attribute)
        activation_engine: ACT-R activation engine
        embedding_provider: Embedding provider
        config: Hybrid configuration (optional)
        aurora_config: Global AURORA Config object (optional)

    Returns:
        Cached or new HybridRetriever instance

    """
    # Get db_path from store
    db_path = getattr(store, "db_path", ":memory:")

    # Use default config if none provided
    if config is None:
        config = HybridConfig()

    # Compute config hash for cache key
    config_hash = _compute_config_hash(config)
    cache_key = (db_path, config_hash)

    with _retriever_cache_lock:
        # Check cache
        if cache_key in _retriever_cache:
            entry = _retriever_cache[cache_key]
            retriever, timestamp = entry

            # Check TTL
            if time.time() - timestamp <= _RETRIEVER_CACHE_TTL:
                _retriever_cache_stats["hits"] += 1
                logger.debug(
                    f"Reusing cached HybridRetriever for db_path={db_path} "
                    f"(hit_rate={_get_hit_rate():.1%})",
                )
                return retriever

            # TTL expired, remove from cache
            logger.debug(
                f"Cached HybridRetriever expired for db_path={db_path} (TTL={_RETRIEVER_CACHE_TTL}s)"
            )
            del _retriever_cache[cache_key]

        # Cache miss - create new retriever
        _retriever_cache_stats["misses"] += 1
        logger.debug(
            f"Creating new HybridRetriever for db_path={db_path} (hit_rate={_get_hit_rate():.1%})",
        )

        # Apply LRU eviction if at capacity
        if len(_retriever_cache) >= _RETRIEVER_CACHE_SIZE:
            # Evict oldest entry (first item in dict)
            oldest_key = next(iter(_retriever_cache))
            del _retriever_cache[oldest_key]
            logger.debug(
                f"Evicted oldest HybridRetriever from cache (size={_RETRIEVER_CACHE_SIZE})"
            )

        # Create new retriever
        retriever = HybridRetriever(
            store=store,
            activation_engine=activation_engine,
            embedding_provider=embedding_provider,
            config=config,
            aurora_config=aurora_config,
        )

        # Cache with timestamp
        _retriever_cache[cache_key] = (retriever, time.time())

        return retriever


def _get_hit_rate() -> float:
    """Calculate cache hit rate."""
    total = _retriever_cache_stats["hits"] + _retriever_cache_stats["misses"]
    return _retriever_cache_stats["hits"] / total if total > 0 else 0.0


def get_cache_stats() -> dict[str, Any]:
    """Get HybridRetriever cache statistics.

    Returns:
        Dict with keys:
        - total_hits: Number of cache hits
        - total_misses: Number of cache misses
        - hit_rate: Cache hit rate (0.0-1.0)
        - cache_size: Current number of cached retrievers

    """
    with _retriever_cache_lock:
        return {
            "total_hits": _retriever_cache_stats["hits"],
            "total_misses": _retriever_cache_stats["misses"],
            "hit_rate": _get_hit_rate(),
            "cache_size": len(_retriever_cache),
        }


def clear_retriever_cache() -> None:
    """Clear all cached HybridRetriever instances and reset statistics."""
    with _retriever_cache_lock:
        _retriever_cache.clear()
        _retriever_cache_stats["hits"] = 0
        _retriever_cache_stats["misses"] = 0
        logger.debug("Cleared HybridRetriever cache")


class HybridRetriever:
    """Tri-hybrid retrieval combining BM25, semantic similarity, and activation.

    Retrieval process (staged architecture):
    1. Stage 1: BM25 Filtering
       - Retrieve top-K chunks by activation (default K=100)
       - Build BM25 index from candidates
       - Score candidates with BM25 keyword matching
       - Select top stage1_top_k candidates (default 100)
    2. Stage 2: Tri-hybrid Re-ranking
       - Calculate semantic similarity for Stage 1 candidates
       - Normalize BM25, semantic, and activation scores independently
       - Combine scores: 30% BM25 + 40% semantic + 30% activation (configurable)
       - Return top-N results by tri-hybrid score

    Attributes:
        store: Storage backend for chunks
        activation_engine: ACT-R activation engine
        embedding_provider: Provider for generating embeddings
        config: Hybrid retrieval configuration
        bm25_scorer: BM25 scorer for keyword matching (lazy-initialized)

    Example (tri-hybrid):
        >>> from aurora_core.store import SQLiteStore
        >>> from aurora_core.activation import ActivationEngine
        >>> from aurora_context_code.semantic import EmbeddingProvider, HybridRetriever
        >>>
        >>> store = SQLiteStore(":memory:")
        >>> engine = ActivationEngine(store)
        >>> provider = EmbeddingProvider()
        >>> retriever = HybridRetriever(store, engine, provider)
        >>>
        >>> results = retriever.retrieve("SoarOrchestrator", top_k=5)
        >>> # Results will favor exact keyword matches with tri-hybrid scoring

    """

    def __init__(
        self,
        store: Any,  # aurora_core.store.Store
        activation_engine: Any,  # aurora_core.activation.ActivationEngine
        embedding_provider: Any,  # EmbeddingProvider
        config: HybridConfig | None = None,
        aurora_config: Any | None = None,  # aurora_core.config.Config
    ):
        """Initialize tri-hybrid retriever with lazy BM25 loading (Epic 2).

        BM25 index is loaded lazily on first retrieve() call, reducing creation time
        from 150-250ms to ~0.0ms (99.9% improvement). Thread-safe double-checked locking
        ensures the index is loaded exactly once even with concurrent retrieve() calls.

        Args:
            store: Storage backend
            activation_engine: ACT-R activation engine
            embedding_provider: Embedding provider (None triggers dual-hybrid fallback)
            config: Hybrid configuration (takes precedence if provided)
            aurora_config: Global AURORA Config object (loads hybrid_weights from context.code.hybrid_weights)

        Note:
            If both config and aurora_config are provided, config takes precedence.
            If neither is provided, uses default HybridConfig values (tri-hybrid: 30/40/30).
            If embedding_provider is None, dual-hybrid fallback (BM25+Activation) is used.

        """
        # Type annotations for instance variables
        self._query_cache: QueryEmbeddingCache | None

        self.store = store
        self.activation_engine = activation_engine
        self.embedding_provider = embedding_provider

        # Load configuration with precedence: explicit config > aurora_config > defaults
        if config is not None:
            self.config = config
        elif aurora_config is not None:
            self.config = self._load_from_aurora_config(aurora_config)
        else:
            self.config = HybridConfig()

        # BM25 scorer (lazy-initialized in retrieve() or loaded from persistent index)
        self.bm25_scorer: Any = None  # BM25Scorer from aurora_context_code.semantic.bm25_scorer
        self._bm25_index_loaded = False  # Track if we've loaded the persistent index
        self._bm25_lock = threading.Lock()  # Thread-safety for lazy loading

        # Query embedding cache (shared across all retrievers - Task 4.0)
        if self.config.enable_query_cache:
            self._query_cache = get_shared_query_cache(
                capacity=self.config.query_cache_size,
                ttl_seconds=self.config.query_cache_ttl_seconds,
            )
            logger.debug(
                f"Using shared query cache: size={self.config.query_cache_size}, "
                f"ttl={self.config.query_cache_ttl_seconds}s",
            )
        else:
            self._query_cache = None

    def retrieve(
        self,
        query: str,
        top_k: int = 10,
        _context_keywords: list[str] | None = None,
        min_semantic_score: float | None = None,
        chunk_type: str | None = None,
        diverse: bool = False,
        mmr_lambda: float | None = None,
    ) -> list[dict[str, Any]]:
        """Retrieve chunks using tri-hybrid scoring with staged architecture.

        Args:
            query: User query string
            top_k: Number of results to return
            context_keywords: Optional keywords for context boost (not yet implemented)
            min_semantic_score: Minimum semantic score threshold (0.0-1.0). Results below this will be filtered out.
            chunk_type: Optional filter by chunk type ('code' or 'kb').
            diverse: If True, apply MMR reranking for diverse results (default False)
            mmr_lambda: MMR diversity parameter (0.0=diversity, 1.0=relevance). Defaults to config.mmr_lambda

        Returns:
            List of dicts with keys:
            - chunk_id: Chunk identifier
            - content: Chunk content
            - bm25_score: BM25 keyword component (0-1 normalized)
            - activation_score: Activation component (0-1 normalized)
            - semantic_score: Semantic similarity component (0-1 normalized)
            - hybrid_score: Combined tri-hybrid score (0-1 range)
            - metadata: Additional chunk metadata

        Raises:
            ValueError: If query is empty or top_k < 1

        Example:
            >>> results = retriever.retrieve("SoarOrchestrator", top_k=5)
            >>> for result in results:
            ...     print(f"{result['chunk_id']}: {result['hybrid_score']:.3f}")
            ...     print(f"  BM25: {result['bm25_score']:.3f}")
            ...     print(f"  Semantic: {result['semantic_score']:.3f}")
            ...     print(f"  Activation: {result['activation_score']:.3f}")

        Example (with diversity):
            >>> # Get diverse results covering different aspects of auth
            >>> results = retriever.retrieve("authentication", top_k=10, diverse=True)

        """
        # Validate inputs
        if not query or not query.strip():
            raise ValueError("Query cannot be empty")
        if top_k < 1:
            raise ValueError(f"top_k must be >= 1, got {top_k}")

        # ========== TWO-PHASE RETRIEVAL OPTIMIZATION ==========
        # Phase 1: Retrieve chunks WITHOUT embeddings (fast, ~1.5KB saved per chunk)
        # Phase 2: Fetch embeddings ONLY for top candidates after BM25 filtering
        # This optimization reduces I/O significantly for large result sets

        # Step 1: Retrieve top-K chunks by activation WITHOUT embeddings
        # Embeddings will be fetched later only for BM25-filtered candidates
        use_two_phase = (
            self.config.use_staged_retrieval
            and self.config.bm25_weight > 0
            and hasattr(self.store, "fetch_embeddings_for_chunks")
        )

        activation_candidates = self.store.retrieve_by_activation(
            min_activation=0.0,  # Get all chunks, we'll filter by score
            limit=self.config.activation_top_k,
            include_embeddings=not use_two_phase,  # Skip embeddings if two-phase enabled
            chunk_type=chunk_type,  # Optional filter by type ('code' or 'kb')
        )

        # If no chunks available, return empty list
        if not activation_candidates:
            return []

        # Step 2: Generate query embedding for semantic similarity (with caching)
        query_embedding = None

        # Try cache first
        if self._query_cache is not None:
            query_embedding = self._query_cache.get(query)
            if query_embedding is not None:
                logger.debug(f"Query cache hit for: {query[:50]}...")

        # Generate embedding if not cached
        if query_embedding is None:
            # If no embedding provider, fall back to BM25+Activation dual-hybrid
            if self.embedding_provider is None:
                logger.debug("No embedding provider - using BM25+Activation fallback")
                return self._fallback_to_dual_hybrid(activation_candidates, query, top_k)

            try:
                query_embedding = self.embedding_provider.embed_query(query)
                # Cache the embedding
                if self._query_cache is not None:
                    self._query_cache.set(query, query_embedding)
                    logger.debug(f"Cached embedding for: {query[:50]}...")
            except Exception as e:
                # If embedding fails and fallback is enabled, use BM25+Activation dual-hybrid
                if self.config.fallback_to_activation:
                    return self._fallback_to_dual_hybrid(activation_candidates, query, top_k)
                raise ValueError(f"Failed to generate query embedding: {e}") from e

        # ========== STAGE 1: BM25 FILTERING ==========
        if self.config.use_staged_retrieval and self.config.bm25_weight > 0:
            stage1_candidates = self._stage1_bm25_filter(query, activation_candidates)
        else:
            # Skip Stage 1 if staged retrieval disabled or BM25 weight is 0
            stage1_candidates = activation_candidates

        # ========== PHASE 2: FETCH EMBEDDINGS FOR TOP CANDIDATES ==========
        # Only fetch embeddings for chunks that passed BM25 filtering
        if use_two_phase and stage1_candidates:
            candidate_ids = [chunk.id for chunk in stage1_candidates]
            embeddings_map = self.store.fetch_embeddings_for_chunks(candidate_ids)
            logger.debug(
                f"Fetched embeddings for {len(embeddings_map)}/{len(candidate_ids)} chunks"
            )

            # Attach embeddings to chunks
            for chunk in stage1_candidates:
                if chunk.id in embeddings_map:
                    chunk.embeddings = embeddings_map[chunk.id]

        # ========== STAGE 2: TRI-HYBRID RE-RANKING ==========
        results = []

        for chunk in stage1_candidates:
            # Get activation score (from chunk's activation attribute)
            activation_score = getattr(chunk, "activation", 0.0)

            # Calculate semantic similarity
            chunk_embedding = getattr(chunk, "embeddings", None)
            if chunk_embedding is not None:
                from aurora_context_code.semantic.embedding_provider import cosine_similarity

                # Convert embedding bytes to numpy array if needed
                if isinstance(chunk_embedding, bytes):
                    chunk_embedding = np.frombuffer(chunk_embedding, dtype=np.float32)

                semantic_score = cosine_similarity(query_embedding, chunk_embedding)
                # Cosine similarity is in [-1, 1], normalize to [0, 1]
                semantic_score = (semantic_score + 1.0) / 2.0
            # No embedding available, use 0 or fallback
            elif self.config.fallback_to_activation:
                semantic_score = 0.0
            else:
                continue  # Skip chunks without embeddings

            # Calculate BM25 score (if enabled)
            if self.config.bm25_weight > 0 and self.bm25_scorer is not None:
                # Get chunk content for BM25 scoring
                chunk_content = self._get_chunk_content_for_bm25(chunk)
                bm25_score = self.bm25_scorer.score(query, chunk_content)
            else:
                bm25_score = 0.0

            # Store for later normalization
            results.append(
                {
                    "chunk": chunk,
                    "raw_activation": activation_score,
                    "raw_semantic": semantic_score,
                    "raw_bm25": bm25_score,
                },
            )

        # If no valid results, return empty
        if not results:
            return []

        # NOTE: Semantic threshold filtering is disabled when BM25 is enabled (tri-hybrid mode)
        # to allow keyword matches with low semantic similarity to be retrieved.
        # In tri-hybrid mode, the hybrid score (BM25 + semantic + activation) determines relevance.
        # Only filter by semantic score in dual-hybrid mode (when bm25_weight == 0)
        if min_semantic_score is not None and self.config.bm25_weight == 0.0:
            results = [r for r in results if r["raw_semantic"] >= min_semantic_score]
            if not results:
                return []  # All results below threshold

        # Normalize scores independently to [0, 1] range
        activation_scores_normalized = self._normalize_scores(
            [r["raw_activation"] for r in results],
        )
        semantic_scores_normalized = self._normalize_scores([r["raw_semantic"] for r in results])
        bm25_scores_normalized = self._normalize_scores([r["raw_bm25"] for r in results])

        # ========== BATCH FETCH ACCESS STATS (N+1 QUERY OPTIMIZATION) ==========
        # Pre-fetch access stats for all result chunks in a single query
        chunk_ids = [r["chunk"].id for r in results]
        access_stats_cache: dict[str, dict[str, Any]] = {}
        if hasattr(self.store, "get_access_stats_batch"):
            try:
                access_stats_cache = self.store.get_access_stats_batch(chunk_ids)
                logger.debug(f"Batch fetched access stats for {len(access_stats_cache)} chunks")
            except Exception as e:
                logger.debug(f"Batch access stats failed, falling back to per-chunk: {e}")

        # Calculate tri-hybrid scores and prepare output
        final_results = []
        for i, result_data in enumerate(results):
            chunk = result_data["chunk"]
            activation_norm = activation_scores_normalized[i]
            semantic_norm = semantic_scores_normalized[i]
            bm25_norm = bm25_scores_normalized[i]

            # Tri-hybrid scoring formula: weights × normalized scores
            hybrid_score = (
                self.config.bm25_weight * bm25_norm
                + self.config.activation_weight * activation_norm
                + self.config.semantic_weight * semantic_norm
            )

            # Apply code-first ordering boost for code type chunks
            # This ensures code results rank higher than KB/markdown in search results
            chunk_type = getattr(chunk, "type", "unknown")
            if chunk_type == "code" and self.config.code_type_boost > 0:
                hybrid_score += self.config.code_type_boost
                # Clamp to [0, 1] range to satisfy validation constraints
                hybrid_score = min(hybrid_score, 1.0)

            # Extract content and metadata from chunk (using cached access stats)
            content, metadata = self._extract_chunk_content_metadata(
                chunk,
                access_stats_cache=access_stats_cache,
            )

            final_results.append(
                {
                    "chunk_id": chunk.id,
                    "content": content,
                    "bm25_score": bm25_norm,
                    "activation_score": activation_norm,
                    "semantic_score": semantic_norm,
                    "hybrid_score": hybrid_score,
                    "metadata": metadata,
                },
            )

        # Sort by hybrid score (descending)
        final_results.sort(key=lambda x: x["hybrid_score"], reverse=True)

        # Apply MMR reranking for diversity if requested
        if diverse and len(final_results) > 1:
            lambda_val = mmr_lambda if mmr_lambda is not None else self.config.mmr_lambda
            final_results = self._apply_mmr_reranking(
                results=final_results,
                stage1_candidates=stage1_candidates,
                top_k=top_k,
                mmr_lambda=lambda_val,
            )
            return final_results

        # Return top K results
        return final_results[:top_k]

    def _apply_mmr_reranking(
        self,
        results: list[dict[str, Any]],
        stage1_candidates: list[Any],
        top_k: int,
        mmr_lambda: float,
    ) -> list[dict[str, Any]]:
        """Apply Maximal Marginal Relevance reranking for diverse results.

        MMR balances relevance and diversity by penalizing candidates that are
        too similar to already-selected results. This prevents the "echo chamber"
        effect where all results are about the same aspect of a topic.

        Formula:
            MMR(d) = λ × relevance(d) - (1-λ) × max_similarity(d, selected)

        Where:
            - λ=1.0: Pure relevance (no diversity)
            - λ=0.5: Balanced (default)
            - λ=0.0: Pure diversity (least similar to selected)

        Args:
            results: Sorted list of result dicts (by hybrid_score descending)
            stage1_candidates: Original chunk objects (for embedding access)
            top_k: Number of results to return
            mmr_lambda: Balance parameter (0.0=diversity, 1.0=relevance)

        Returns:
            Reranked list of results with diverse coverage

        """
        if len(results) <= 1:
            return results[:top_k]

        # Build chunk_id -> embedding lookup from stage1_candidates
        embedding_map: dict[str, npt.NDArray[np.float32] | None] = {}
        for chunk in stage1_candidates:
            chunk_embedding = getattr(chunk, "embeddings", None)
            if chunk_embedding is not None:
                if isinstance(chunk_embedding, bytes):
                    chunk_embedding = np.frombuffer(chunk_embedding, dtype=np.float32)
                embedding_map[chunk.id] = chunk_embedding
            else:
                embedding_map[chunk.id] = None

        # Initialize with top result (always selected first)
        selected: list[dict[str, Any]] = [results[0]]
        remaining = list(results[1:])

        while len(selected) < top_k and remaining:
            best_mmr_score = -float("inf")
            best_idx = 0

            for i, candidate in enumerate(remaining):
                # Relevance component: normalized hybrid score
                relevance = candidate["hybrid_score"]

                # Diversity component: 1 - max similarity to selected results
                candidate_embedding = embedding_map.get(candidate["chunk_id"])
                if candidate_embedding is None:
                    # No embedding, fall back to pure relevance
                    diversity = 0.0
                else:
                    max_similarity = 0.0
                    for selected_result in selected:
                        selected_embedding = embedding_map.get(selected_result["chunk_id"])
                        if selected_embedding is not None:
                            # Cosine similarity between candidate and selected
                            similarity = self._cosine_similarity(
                                candidate_embedding, selected_embedding
                            )
                            # Normalize from [-1, 1] to [0, 1]
                            similarity = (similarity + 1.0) / 2.0
                            max_similarity = max(max_similarity, similarity)
                    diversity = 1.0 - max_similarity

                # MMR score: balance relevance and diversity
                mmr_score = mmr_lambda * relevance + (1.0 - mmr_lambda) * diversity

                if mmr_score > best_mmr_score:
                    best_mmr_score = mmr_score
                    best_idx = i

            # Add best candidate to selected
            selected.append(remaining.pop(best_idx))

        logger.debug(
            f"MMR reranking: selected {len(selected)} diverse results (lambda={mmr_lambda:.2f})"
        )
        return selected

    def _cosine_similarity(
        self,
        vec1: npt.NDArray[np.float32],
        vec2: npt.NDArray[np.float32],
    ) -> float:
        """Calculate cosine similarity between two vectors.

        Args:
            vec1: First vector
            vec2: Second vector

        Returns:
            Cosine similarity in range [-1, 1]

        """
        norm1 = np.linalg.norm(vec1)
        norm2 = np.linalg.norm(vec2)
        if norm1 < 1e-9 or norm2 < 1e-9:
            return 0.0
        return float(np.dot(vec1, vec2) / (norm1 * norm2))

    def _stage1_bm25_filter(self, query: str, candidates: list[Any]) -> list[Any]:
        """Stage 1: Filter candidates using BM25 keyword matching with lazy loading (Epic 2).

        BM25 index is loaded lazily on first call using thread-safe double-checked locking.
        This defers the 150-250ms index load cost from __init__() to first retrieve(),
        improving retriever creation time by 99.9% (0.0ms vs 150-250ms).

        Uses persistent BM25 index if available, otherwise builds from candidates.
        The persistent index is built during indexing and loaded on first retrieve(),
        eliminating the O(n) index build on each query (51% of search time savings).

        Args:
            query: User query string
            candidates: Chunks retrieved by activation

        Returns:
            Top stage1_top_k candidates by BM25 score

        """
        from aurora_context_code.semantic.bm25_scorer import BM25Scorer

        # Lazy load BM25 index on first retrieve() call (thread-safe)
        if not self._bm25_index_loaded and self.config.enable_bm25_persistence:
            with self._bm25_lock:
                # Double-check pattern (another thread may have loaded while we waited)
                if not self._bm25_index_loaded:
                    self._try_load_bm25_index()

        # Use persistent index if loaded, otherwise build from candidates
        if self._bm25_index_loaded and self.bm25_scorer is not None:
            logger.debug("Using persistent BM25 index")
        else:
            # Build BM25 index from candidates (fallback if no persistent index)
            logger.debug("Building BM25 index from candidates (no persistent index)")
            self.bm25_scorer = BM25Scorer(k1=1.5, b=0.75)

            # Prepare documents for BM25 indexing
            documents = []
            for chunk in candidates:
                chunk_content = self._get_chunk_content_for_bm25(chunk)
                documents.append((chunk.id, chunk_content))

            # Build BM25 index
            self.bm25_scorer.build_index(documents)

        # Score all candidates with BM25
        scored_candidates = []
        for chunk in candidates:
            chunk_content = self._get_chunk_content_for_bm25(chunk)
            bm25_score = self.bm25_scorer.score(query, chunk_content)
            scored_candidates.append((bm25_score, chunk))

        # Sort by BM25 score (descending) and take top stage1_top_k
        scored_candidates.sort(key=lambda x: x[0], reverse=True)
        top_candidates = [chunk for _, chunk in scored_candidates[: self.config.stage1_top_k]]

        return top_candidates

    def _get_chunk_content_for_bm25(self, chunk: Any) -> str:
        """Get chunk content suitable for BM25 tokenization.

        Args:
            chunk: Chunk object

        Returns:
            Content string (signature + docstring + name for CodeChunk)

        """
        # For CodeChunk: combine signature, docstring, and name
        if hasattr(chunk, "signature"):
            parts = []
            if getattr(chunk, "name", None):
                parts.append(chunk.name)
            if getattr(chunk, "signature", None):
                parts.append(chunk.signature)
            if getattr(chunk, "docstring", None):
                parts.append(chunk.docstring)
            return " ".join(parts) if parts else ""
        # For other chunk types, use to_json() content
        chunk_json = chunk.to_json() if hasattr(chunk, "to_json") else {}
        return str(chunk_json.get("content", ""))

    def _extract_chunk_content_metadata(
        self,
        chunk: Any,
        access_stats_cache: dict[str, dict[str, Any]] | None = None,
    ) -> tuple[str, dict[str, Any]]:
        """Extract content and metadata from chunk.

        Args:
            chunk: Chunk object
            access_stats_cache: Optional pre-fetched access stats (for N+1 optimization)

        Returns:
            Tuple of (content, metadata)

        """
        # For CodeChunk: content is signature + docstring
        if hasattr(chunk, "signature") and hasattr(chunk, "docstring"):
            content_parts = []
            if getattr(chunk, "signature", None):
                content_parts.append(chunk.signature)
            if getattr(chunk, "docstring", None):
                content_parts.append(chunk.docstring)
            content = "\n".join(content_parts) if content_parts else ""

            metadata = {
                "type": getattr(chunk, "type", "unknown"),
                "name": getattr(chunk, "name", ""),
                "file_path": getattr(chunk, "file_path", ""),
                "line_start": getattr(chunk, "line_start", 0),
                "line_end": getattr(chunk, "line_end", 0),
            }

            # Include access count from activation stats (use cache if available)
            if access_stats_cache and chunk.id in access_stats_cache:
                metadata["access_count"] = access_stats_cache[chunk.id].get("access_count", 0)
            else:
                try:
                    access_stats = self.store.get_access_stats(chunk.id)
                    metadata["access_count"] = access_stats.get("access_count", 0)
                except Exception:
                    # If access stats unavailable, default to 0
                    metadata["access_count"] = 0

            # Include git metadata if available
            if hasattr(chunk, "metadata") and chunk.metadata:
                if "commit_count" in chunk.metadata:
                    metadata["commit_count"] = chunk.metadata["commit_count"]
                if "last_modified" in chunk.metadata:
                    metadata["last_modified"] = chunk.metadata["last_modified"]
                if "git_hash" in chunk.metadata:
                    metadata["git_hash"] = chunk.metadata["git_hash"]
        else:
            # Other chunk types - use to_json() to get content
            chunk_json = chunk.to_json() if hasattr(chunk, "to_json") else {}
            content = str(chunk_json.get("content", ""))
            metadata = {
                "type": getattr(chunk, "type", "unknown"),
                "name": getattr(chunk, "name", ""),
                "file_path": getattr(chunk, "file_path", ""),
            }

            # Include access count from activation stats (use cache if available)
            if access_stats_cache and chunk.id in access_stats_cache:
                metadata["access_count"] = access_stats_cache[chunk.id].get("access_count", 0)
            else:
                try:
                    access_stats = self.store.get_access_stats(chunk.id)
                    metadata["access_count"] = access_stats.get("access_count", 0)
                except Exception:
                    # If access stats unavailable, default to 0
                    metadata["access_count"] = 0

            # Include git metadata if available
            if hasattr(chunk, "metadata") and chunk.metadata:
                if "commit_count" in chunk.metadata:
                    metadata["commit_count"] = chunk.metadata["commit_count"]
                if "last_modified" in chunk.metadata:
                    metadata["last_modified"] = chunk.metadata["last_modified"]
                if "git_hash" in chunk.metadata:
                    metadata["git_hash"] = chunk.metadata["git_hash"]

        return content, metadata

    def _fallback_to_dual_hybrid(
        self, activation_candidates: list[Any], query: str, top_k: int
    ) -> list[dict[str, Any]]:
        """Fallback to BM25+Activation dual-hybrid when embeddings unavailable (Epic 2).

        This fallback provides significantly better search quality (~85-100%) than the
        old activation-only fallback (~60%) by leveraging keyword matching (BM25)
        alongside access patterns (activation). Qualitative testing showed 100% overlap
        with tri-hybrid results on the Aurora codebase.

        Weight normalization: Redistributes semantic_weight proportionally to BM25 and
        activation, ensuring weights sum to 1.0. For default tri-hybrid (30/40/30), the
        dual-hybrid weights become (43/57/0) - preserving the BM25:activation ratio.

        Args:
            activation_candidates: Chunks retrieved by activation
            query: User query string
            top_k: Number of results to return

        Returns:
            List of results with BM25+Activation dual-hybrid scores (semantic=0)

        """
        logger.warning(
            "ML embeddings unavailable - using keyword + recency fallback (no semantic matching). "
            "To enable: pip install sentence-transformers torch"
        )

        # Run Stage 1 BM25 filter to get keyword scores
        stage1_candidates = self._stage1_bm25_filter(query, activation_candidates)

        # Normalize weights (redistribute semantic_weight to bm25 and activation)
        total_weight = self.config.bm25_weight + self.config.activation_weight
        if total_weight < 1e-6:
            # Edge case: both weights are 0, fall back to activation-only
            logger.warning("Both BM25 and activation weights are 0 - using activation-only")
            bm25_dual = 0.0
            activation_dual = 1.0
        else:
            bm25_dual = self.config.bm25_weight / total_weight
            activation_dual = self.config.activation_weight / total_weight

        # Build results with dual-hybrid scoring
        results = []
        for chunk in stage1_candidates:
            activation_score = getattr(chunk, "activation", 0.0)

            # Get BM25 score (already calculated in _stage1_bm25_filter)
            if self.config.bm25_weight > 0 and self.bm25_scorer is not None:
                chunk_content = self._get_chunk_content_for_bm25(chunk)
                bm25_score = self.bm25_scorer.score(query, chunk_content)
            else:
                bm25_score = 0.0

            results.append(
                {
                    "chunk": chunk,
                    "raw_activation": activation_score,
                    "raw_semantic": 0.0,  # No embeddings available
                    "raw_bm25": bm25_score,
                }
            )

        # Normalize scores independently
        activation_scores_normalized = self._normalize_scores(
            [r["raw_activation"] for r in results]
        )
        bm25_scores_normalized = self._normalize_scores([r["raw_bm25"] for r in results])

        # Batch fetch access stats (N+1 query optimization)
        chunk_ids = [r["chunk"].id for r in results]
        access_stats_cache: dict[str, dict[str, Any]] = {}
        if hasattr(self.store, "get_access_stats_batch"):
            try:
                access_stats_cache = self.store.get_access_stats_batch(chunk_ids)
            except Exception as e:
                logger.debug(f"Batch access stats failed: {e}")

        # Calculate dual-hybrid scores
        final_results = []
        for i, result_data in enumerate(results):
            chunk = result_data["chunk"]
            activation_norm = activation_scores_normalized[i]
            bm25_norm = bm25_scores_normalized[i]

            # Dual-hybrid scoring: weighted BM25 + activation (no semantic)
            hybrid_score = bm25_dual * bm25_norm + activation_dual * activation_norm

            # Apply code-first ordering boost for code type chunks
            chunk_type = getattr(chunk, "type", "unknown")
            if chunk_type == "code" and self.config.code_type_boost > 0:
                hybrid_score += self.config.code_type_boost
                # Clamp to [0, 1] range to satisfy validation constraints
                hybrid_score = min(hybrid_score, 1.0)

            content, metadata = self._extract_chunk_content_metadata(
                chunk,
                access_stats_cache=access_stats_cache,
            )

            final_results.append(
                {
                    "chunk_id": chunk.id,
                    "content": content,
                    "bm25_score": bm25_norm,
                    "activation_score": activation_norm,
                    "semantic_score": 0.0,  # Embeddings unavailable
                    "hybrid_score": hybrid_score,
                    "metadata": metadata,
                }
            )

        # Sort by hybrid score (descending)
        final_results.sort(key=lambda x: x["hybrid_score"], reverse=True)

        # Return top K results
        return final_results[:top_k]

    def _normalize_scores(self, scores: list[float]) -> list[float]:
        """Normalize scores to [0, 1] range using min-max scaling.

        Args:
            scores: Raw scores to normalize

        Returns:
            Normalized scores in [0, 1] range

        Note:
            When all scores are equal, returns original scores unchanged
            to preserve meaningful zero values rather than inflating to 1.0.

        """
        if not scores:
            return []

        min_score = min(scores)
        max_score = max(scores)

        if max_score - min_score < 1e-9:
            # All scores equal - preserve original values
            # This prevents [0.0, 0.0, 0.0] from becoming [1.0, 1.0, 1.0]
            return list(scores)

        return [(s - min_score) / (max_score - min_score) for s in scores]

    def _load_from_aurora_config(self, aurora_config: Any) -> HybridConfig:
        """Load tri-hybrid configuration from global AURORA Config.

        Args:
            aurora_config: AURORA Config object with context.code.hybrid_weights

        Returns:
            HybridConfig loaded from config

        Raises:
            ValueError: If config values are invalid

        """
        # Load from context.code.hybrid_weights section
        weights = aurora_config.get("context.code.hybrid_weights", {})

        # Extract values with fallback to tri-hybrid defaults
        bm25_weight = weights.get("bm25", 0.3)
        activation_weight = weights.get("activation", 0.3)
        semantic_weight = weights.get("semantic", 0.4)
        activation_top_k = weights.get("top_k", 500)  # Match HybridConfig default
        stage1_top_k = weights.get("stage1_top_k", 100)
        fallback_to_activation = weights.get("fallback_to_activation", True)
        use_staged_retrieval = weights.get("use_staged_retrieval", True)

        # Create and validate HybridConfig (validation happens in __post_init__)
        return HybridConfig(
            bm25_weight=bm25_weight,
            activation_weight=activation_weight,
            semantic_weight=semantic_weight,
            activation_top_k=activation_top_k,
            stage1_top_k=stage1_top_k,
            fallback_to_activation=fallback_to_activation,
            use_staged_retrieval=use_staged_retrieval,
        )

    def get_cache_stats(self) -> dict[str, Any]:
        """Get query embedding cache statistics.

        Returns:
            Dictionary with cache stats:
            - enabled: Whether cache is enabled
            - size: Current number of cached embeddings
            - capacity: Maximum cache capacity
            - hits: Number of cache hits
            - misses: Number of cache misses
            - hit_rate: Cache hit rate (0.0-1.0)
            - evictions: Number of LRU evictions

        """
        if self._query_cache is None:
            return {"enabled": False}

        return {
            "enabled": True,
            "size": self._query_cache.size(),
            "capacity": self._query_cache.capacity,
            "hits": self._query_cache.stats.hits,
            "misses": self._query_cache.stats.misses,
            "hit_rate": self._query_cache.stats.hit_rate,
            "evictions": self._query_cache.stats.evictions,
        }

    def clear_cache(self) -> None:
        """Clear the query embedding cache."""
        if self._query_cache is not None:
            self._query_cache.clear()
            logger.debug("Query embedding cache cleared")

    def _get_bm25_index_path(self) -> Path | None:
        """Get the path for the BM25 index file.

        Returns:
            Path to BM25 index file, or None if not configured

        """
        if self.config.bm25_index_path:
            return Path(self.config.bm25_index_path).expanduser()

        # Default: try to find .aurora directory relative to store's db_path
        if hasattr(self.store, "db_path") and self.store.db_path != ":memory:":
            db_path = Path(self.store.db_path)
            return db_path.parent / "indexes" / "bm25_index.pkl"

        return None

    def _try_load_bm25_index(self) -> bool:
        """Try to load a persistent BM25 index from disk.

        Returns:
            True if index was loaded successfully, False otherwise

        """
        index_path = self._get_bm25_index_path()
        if index_path is None or not index_path.exists():
            # Changed from DEBUG to INFO for better visibility (Task 3.6)
            logger.info(f"No persistent BM25 index found at {index_path}")
            return False

        try:
            from aurora_context_code.semantic.bm25_scorer import BM25Scorer

            self.bm25_scorer = BM25Scorer(k1=1.5, b=0.75)
            self.bm25_scorer.load_index(index_path)
            self._bm25_index_loaded = True

            # Enhanced logging with corpus size and file size (Task 3.6)
            file_size_mb = index_path.stat().st_size / (1024 * 1024)
            corpus_size = self.bm25_scorer.corpus_size

            # Validate loaded index has documents (Task 3.7)
            if corpus_size == 0:
                logger.warning(
                    f"✗ Loaded BM25 index from {index_path} but corpus_size is 0 (empty index)"
                )
                self.bm25_scorer = None
                self._bm25_index_loaded = False
                return False

            logger.info(
                f"✓ Loaded BM25 index from {index_path} "
                f"({corpus_size} documents, {file_size_mb:.2f} MB)"
            )
            return True
        except (pickle.UnpicklingError, ModuleNotFoundError, EOFError) as e:
            # Improved error handling for pickle format mismatches (Task 3.8)
            error_type = type(e).__name__
            logger.warning(f"✗ Failed to load BM25 index from {index_path} ({error_type}): {e}")
            self.bm25_scorer = None
            self._bm25_index_loaded = False
            return False
        except Exception as e:
            # Catch-all for other errors
            error_type = type(e).__name__
            logger.warning(f"✗ Failed to load BM25 index from {index_path} ({error_type}): {e}")
            self.bm25_scorer = None
            self._bm25_index_loaded = False
            return False

    def build_and_save_bm25_index(self, documents: list[tuple[str, str]] | None = None) -> bool:
        """Build BM25 index from documents and save to disk.

        This method is called during indexing to build the persistent BM25 index.
        If documents are not provided, it retrieves all chunks from the store.

        Args:
            documents: List of (doc_id, doc_content) tuples, or None to load from store

        Returns:
            True if index was built and saved successfully, False otherwise

        """
        index_path = self._get_bm25_index_path()
        if index_path is None:
            logger.warning("Cannot save BM25 index: no index path configured")
            return False

        try:
            from aurora_context_code.semantic.bm25_scorer import BM25Scorer

            # If no documents provided, load from store
            if documents is None:
                documents = self._load_all_chunks_for_bm25()

            if not documents:
                logger.warning("No documents to build BM25 index from")
                return False

            # Build the index
            self.bm25_scorer = BM25Scorer(k1=1.5, b=0.75)
            self.bm25_scorer.build_index(documents)

            # Save to disk
            self.bm25_scorer.save_index(index_path)
            self._bm25_index_loaded = True
            logger.info(
                f"Built and saved BM25 index to {index_path} ({len(documents)} documents)",
            )
            return True

        except Exception as e:
            logger.error(f"Failed to build/save BM25 index: {e}")
            return False

    def _load_all_chunks_for_bm25(self) -> list[tuple[str, str]]:
        """Load all chunks from store for BM25 indexing.

        Returns:
            List of (chunk_id, content) tuples

        """
        documents = []
        try:
            # Retrieve all chunks (high limit to get all)
            chunks = self.store.retrieve_by_activation(min_activation=0.0, limit=100000)
            for chunk in chunks:
                chunk_content = self._get_chunk_content_for_bm25(chunk)
                if chunk_content:
                    documents.append((chunk.id, chunk_content))
            logger.debug(f"Loaded {len(documents)} chunks for BM25 indexing")
        except Exception as e:
            logger.warning(f"Failed to load chunks for BM25 index: {e}")
        return documents

    def invalidate_bm25_index(self) -> None:
        """Invalidate the current BM25 index (call after reindexing).

        This clears the in-memory index and removes the persistent file,
        forcing a rebuild on next search or explicit build_and_save_bm25_index call.
        """
        self.bm25_scorer = None
        self._bm25_index_loaded = False

        index_path = self._get_bm25_index_path()
        if index_path and index_path.exists():
            try:
                index_path.unlink()
                logger.debug(f"Removed stale BM25 index at {index_path}")
            except OSError as e:
                logger.warning(f"Failed to remove BM25 index: {e}")
