"""Vectors backend for Phase 2 storage (post-embedding).

This module manages the vectors.lance table which stores embedded vectors
for semantic search. Enables:
- Vector similarity search
- Resumable embedding (picks up from chunks.lance)
- Search without JOINs (denormalized data)
- Efficient vector indexing with IVF_SQ
"""

import platform
import shutil
import sys
from datetime import datetime
from pathlib import Path
from typing import Any

import lancedb
import pyarrow as pa
from loguru import logger

from .exceptions import (
    DatabaseError,
    DatabaseInitializationError,
    DatabaseNotInitializedError,
    SearchError,
)

# Maximum number of file paths per SQL IN clause.
# DataFusion's recursive descent parser stack-overflows on thousands of OR clauses,
# causing SEGV (signal 139) on Linux. Bounded IN batches keep the parse tree shallow.
DELETE_BATCH_LIMIT = 500


# PyArrow schema for vectors table (dynamic dimension)
def _create_vectors_schema(vector_dim: int) -> pa.Schema:
    """Create vectors schema with dynamic vector dimension.

    Args:
        vector_dim: Embedding vector dimension (e.g., 384 for MiniLM, 768 for CodeBERT)

    Returns:
        PyArrow schema for vectors table
    """
    return pa.schema(
        [
            # Identity (links to chunks table)
            pa.field("chunk_id", pa.string()),
            # Vector embedding (dimension varies by model)
            pa.field("vector", pa.list_(pa.float32(), vector_dim)),
            # Denormalized fields for search (avoid JOINs)
            pa.field("file_path", pa.string()),
            pa.field("content", pa.string()),
            pa.field("language", pa.string()),
            pa.field("start_line", pa.int32()),
            pa.field("end_line", pa.int32()),
            pa.field("chunk_type", pa.string()),
            pa.field("name", pa.string()),
            pa.field("function_name", pa.string()),  # Separate function name field
            pa.field("class_name", pa.string()),  # Separate class name field
            pa.field(
                "project_name", pa.string()
            ),  # Monorepo subproject name (empty for single projects)
            pa.field("hierarchy_path", pa.string()),
            # Metadata
            pa.field("embedded_at", pa.string()),  # ISO timestamp
            pa.field("model_version", pa.string()),  # Embedding model used
        ]
    )


# Default schema for backward compatibility (768-dimensional for GraphCodeBERT)
# NOTE: This is rarely used due to dimension auto-detection in VectorsBackend.__init__
VECTORS_SCHEMA = _create_vectors_schema(768)


class VectorsBackend:
    """Manages vectors.lance table for Phase 2 storage and search.

    This backend stores embedded vectors with denormalized metadata for fast
    semantic search without requiring JOINs to the chunks table.

    Features:
    - Vector similarity search with IVF_SQ index
    - Incremental embedding (detect which chunks need vectors)
    - Metadata filtering (language, file_path, chunk_type)
    - Resumable embedding workflow
    - Index rebuilding after batch inserts

    Example:
        backend = VectorsBackend(db_path)
        await backend.initialize()

        # Add vectors for chunks
        chunks_with_vectors = [
            {"chunk_id": "abc", "vector": [...], "content": "...", ...}
        ]
        count = await backend.add_vectors(chunks_with_vectors)

        # Search by vector similarity
        results = await backend.search(query_vector, limit=10)

        # Find unembedded chunks
        all_chunk_ids = ["abc", "def", "ghi"]
        pending = await backend.get_unembedded_chunk_ids(all_chunk_ids)
    """

    TABLE_NAME = "vectors"
    DEFAULT_VECTOR_DIMENSION = 768  # GraphCodeBERT (default model)

    def __init__(
        self, db_path: Path, vector_dim: int | None = None, table_name: str = "vectors"
    ) -> None:
        """Initialize vectors backend.

        Args:
            db_path: Directory for LanceDB database (same as chunks backend)
            vector_dim: Expected vector dimension (auto-detected if not provided)
            table_name: Name of the vectors table (default: "vectors")
        """
        self.db_path = Path(db_path) if isinstance(db_path, str) else db_path
        self.table_name = table_name
        self._db = None
        self._table = None
        # Vector dimension is auto-detected from first batch or set explicitly
        self.vector_dim = vector_dim
        self._append_count = 0  # Track appends (row count) for periodic compaction
        # Fix 5: schema verification cache — avoids per-call set-diff on stable schema
        self._schema_verified: bool = False

    # ------------------------------------------------------------------
    # Helpers: idempotent table operations
    # ------------------------------------------------------------------

    def _list_table_names(self) -> list[str]:
        """Return table names from LanceDB, handling API variations."""
        tables_response = self._db.list_tables()
        if hasattr(tables_response, "tables"):
            return tables_response.tables
        return tables_response

    def _idempotent_create_table(
        self,
        name: str,
        data: Any,
        schema: pa.Schema | None = None,
        mode: str | None = None,
    ) -> Any:
        """Create a LanceDB table idempotently.

        By default uses ``exist_ok=True`` so that creating a table that
        already exists simply opens the existing table instead of raising.
        Pass ``mode="overwrite"`` to force-replace an existing table.

        Args:
            name: Table name
            data: Initial data (list of dicts or PyArrow table)
            schema: Optional PyArrow schema
            mode: Optional LanceDB write mode ("overwrite", "append").
                  When None, ``exist_ok=True`` is used instead.

        Returns:
            LanceDB table handle
        """
        kwargs: dict[str, Any] = {}
        if schema is not None:
            kwargs["schema"] = schema
        if mode is not None:
            kwargs["mode"] = mode
        else:
            kwargs["exist_ok"] = True
        return self._db.create_table(name, data, **kwargs)

    def _evolve_schema_if_needed(
        self,
        existing_schema: pa.Schema,
        target_schema: pa.Schema,
    ) -> None:
        """Add missing columns to an existing table for forward-compatible schema evolution.

        Compares the target schema against the existing table schema and adds any
        missing columns with null defaults.  After adding columns the internal
        ``_table`` handle is reopened to pick up the new schema.

        Args:
            existing_schema: Current table schema
            target_schema: Desired schema (superset of existing)
        """
        existing_field_names = set(existing_schema.names)
        new_field_names = set(target_schema.names)
        missing_columns = new_field_names - existing_field_names

        if missing_columns:
            logger.info(
                f"Schema evolution: adding missing columns {missing_columns} "
                f"to vectors table"
            )
            missing_fields = [
                target_schema.field(col_name) for col_name in missing_columns
            ]
            self._table.add_columns(missing_fields)
            # Reopen to pick up new schema
            self._table = self._db.open_table(self.table_name)
            logger.info(
                f"Added {len(missing_columns)} new columns "
                f"(preserving existing embeddings)"
            )

    def _is_corruption_error(self, error: Exception) -> bool:
        """Check if error indicates corrupted LanceDB data fragments.

        Args:
            error: Exception to check

        Returns:
            True if error indicates corruption (missing data fragments), False otherwise

        Note:
            This checks for ACTUAL corruption (missing data fragment files),
            NOT schema mismatches or other operational errors. Schema mismatches
            should be handled by the caller, not treated as corruption.
        """
        error_msg = str(error).lower()

        # Check for genuine corruption: missing data fragment files
        # These errors mention specific fragment files like "data/abc123.lance"
        # Example: "NotFound: data fragment 'data/abc123.lance' not found"
        is_fragment_error = (
            "not found" in error_msg or "no such file" in error_msg
        ) and ("fragment" in error_msg or "data/" in error_msg)

        # Schema errors are NOT corruption - they indicate wrong collection_name
        is_schema_error = (
            "schema" in error_msg
            or "field" in error_msg
            or "column" in error_msg
            or "type mismatch" in error_msg
        )

        # Only treat as corruption if it's a fragment error AND NOT a schema error
        return is_fragment_error and not is_schema_error

    def _handle_corrupt_table(self, error: Exception, table_name: str) -> bool:
        """Handle corrupted LanceDB table by deleting and resetting.

        Args:
            error: The corruption error
            table_name: Name of the corrupted table

        Returns:
            True if recovery successful, False if unrecoverable
        """
        if not self._is_corruption_error(error):
            return False

        try:
            table_path = self.db_path / f"{table_name}.lance"
            if table_path.exists():
                logger.warning(
                    f"Detected corrupted {table_name} table (missing data fragment). "
                    f"Auto-recovering by deleting: {table_path}"
                )
                shutil.rmtree(table_path)
                logger.info(f"Deleted corrupted table: {table_path}")

            # Reset internal table reference
            self._table = None
            return True

        except Exception as e:
            logger.error(f"Failed to recover from corruption: {e}")
            return False

    def _delete_chunk_ids(self, chunk_ids: list[str]) -> int:
        """Delete existing vectors with specified chunk_ids to prevent duplicates.

        This method is called before appending new vectors to ensure no duplicates
        exist in the vectors table. Uses batched deletes to avoid SQL length limits.

        Args:
            chunk_ids: List of chunk_ids to delete

        Returns:
            Number of chunks deleted (estimated, not exact)

        Note:
            Deletion failures are logged but non-fatal. If deletion fails, append
            will still proceed, which may result in duplicates. This is acceptable
            as search will still work (just with some redundancy).
        """
        if self._table is None or not chunk_ids:
            return 0

        deleted_count = 0
        batch_size = 500  # Limit SQL query length

        try:
            for i in range(0, len(chunk_ids), batch_size):
                batch = chunk_ids[i : i + batch_size]
                # Build SQL IN clause with proper escaping
                id_list = ", ".join(
                    f"'{cid.replace(chr(39), chr(39) * 2)}'" for cid in batch
                )

                try:
                    # LanceDB delete returns None, so we can't get exact count
                    self._table.delete(f"chunk_id IN ({id_list})")
                    deleted_count += len(batch)
                except Exception as e:
                    # Log but don't fail - dedup is best-effort
                    logger.debug(
                        f"Dedup delete failed for batch {i // batch_size + 1} "
                        f"(non-fatal, may result in duplicates): {e}"
                    )

            if deleted_count > 0:
                logger.debug(
                    f"Deleted {deleted_count} existing vectors to prevent duplicates"
                )

        except Exception as e:
            logger.warning(f"Dedup deletion failed (non-fatal): {e}")

        return deleted_count

    async def initialize(self, force: bool = False) -> None:
        """Create table if not exists with proper schema and vector index.

        Args:
            force: If True, drop and recreate existing table (destructive).
                   Use for full re-index operations. Default: False (idempotent open).

        Raises:
            DatabaseInitializationError: If initialization fails
        """
        try:
            # Ensure directory exists
            self.db_path.mkdir(parents=True, exist_ok=True)

            # Connect to LanceDB
            logger.info(f"[VectorsBackend] Connecting to LanceDB at: {self.db_path}")
            self._db = lancedb.connect(str(self.db_path))

            # Check if table exists
            table_names = self._list_table_names()

            if self.table_name in table_names:
                if force:
                    # force=True: drop and recreate for full re-index
                    logger.info(
                        f"force=True: dropping vectors table '{self.table_name}' for recreation."
                    )
                    try:
                        self._db.drop_table(self.table_name)
                    except Exception as drop_err:
                        logger.warning(f"Failed to drop table (non-fatal): {drop_err}")
                    self._table = None
                    # Fix 5: reset schema cache so it is re-verified after recreation
                    self._schema_verified = False
                    logger.debug("Vectors table will be recreated on first write")
                else:
                    try:
                        self._table = self._db.open_table(self.table_name)
                        logger.debug(
                            f"Opened existing vectors table '{self.table_name}' at {self.db_path}"
                        )
                    except Exception as e:
                        # Check for stale table entry (listed but not actually openable)
                        if (
                            "not found" in str(e).lower()
                            and "fragment" not in str(e).lower()
                        ):
                            logger.warning(
                                f"Stale table entry '{self.table_name}' detected "
                                f"(listed but not openable: {e}). "
                                f"Cleaning up for fresh creation."
                            )
                            try:
                                self._db.drop_table(self.table_name)
                            except Exception:
                                pass
                            self._table = None
                        # Check for corruption and auto-recover
                        elif self._handle_corrupt_table(e, self.table_name):
                            self._table = None
                            logger.info(
                                f"Vectors table '{self.table_name}' corrupted and deleted. Will be recreated on next index."
                            )
                        else:
                            raise
            else:
                # Table will be created on first add_vectors with schema
                self._table = None
                logger.debug("Vectors table will be created on first write")

        except Exception as e:
            logger.error(f"Failed to initialize vectors backend: {e}")
            raise DatabaseInitializationError(
                f"Vectors backend initialization failed: {e}"
            ) from e

    async def check_dimension_mismatch(self, expected_dim: int) -> bool:
        """Check if vectors table has mismatched dimensions.

        Args:
            expected_dim: Expected vector dimension from embedding model

        Returns:
            True if dimension mismatch detected, False otherwise
        """
        if self._table is None:
            # No table exists yet - no mismatch
            return False

        try:
            # Get schema from existing table
            schema = self._table.schema
            vector_field = schema.field("vector")

            # Extract dimension from list type (e.g., list<item: float>[384])
            # The list_size property gives the fixed dimension
            if hasattr(vector_field.type, "list_size"):
                actual_dim = vector_field.type.list_size
                if actual_dim != expected_dim:
                    logger.warning(
                        f"Vector dimension mismatch: table has {actual_dim}D, "
                        f"model expects {expected_dim}D"
                    )
                    return True
            return False
        except Exception as e:
            logger.warning(f"Failed to check dimension mismatch: {e}")
            return False

    async def recreate_table_with_new_dimensions(self, new_dim: int) -> None:
        """Drop and recreate vectors table with new dimensions.

        This is needed when switching embedding models with different dimensions
        (e.g., 384D -> 768D). Preserves chunk data - only vectors need re-embedding.

        Args:
            new_dim: New vector dimension

        Raises:
            DatabaseNotInitializedError: If backend not initialized
            DatabaseError: If recreation fails
        """
        if self._db is None:
            raise DatabaseNotInitializedError("Vectors backend not initialized")

        try:
            # Check if table exists
            table_names = self._list_table_names()

            if self.table_name in table_names:
                # Drop existing table
                self._db.drop_table(self.table_name)
                logger.info(
                    f"Dropped vectors table (dimension change: {self.vector_dim}D -> {new_dim}D)"
                )

            # Update dimension
            self.vector_dim = new_dim
            self._table = None
            logger.info(
                f"Vectors table will be recreated with {new_dim}D on first write"
            )

        except Exception as e:
            logger.error(f"Failed to recreate vectors table: {e}")
            raise DatabaseError(f"Failed to recreate vectors table: {e}") from e

    async def add_vectors(
        self,
        chunks_with_vectors: list[dict[str, Any]],
        model_version: str = "all-MiniLM-L6-v2",
    ) -> int:
        """Add embedded chunks to vectors table.

        All chunks must include a 'vector' field with the embedding.
        Other required fields are denormalized from chunks table for fast search.

        Args:
            chunks_with_vectors: List of dicts with chunk data + 'vector' field:
                - chunk_id (str): Unique identifier linking to chunks.lance
                - vector (list[float]): Embedding vector (dimension varies by model)
                - file_path (str): Source file path
                - content (str): Code content for display
                - language (str): Programming language
                - start_line (int): Starting line number
                - end_line (int): Ending line number
                - chunk_type (str): function, class, method, etc.
                - name (str): Function/class name
                - hierarchy_path (str): Dotted path (e.g., "MyClass.my_method")
            model_version: Embedding model identifier (default: all-MiniLM-L6-v2)

        Returns:
            Number of vectors added

        Raises:
            DatabaseNotInitializedError: If backend not initialized
            DatabaseError: If adding vectors fails
        """
        if self._db is None:
            raise DatabaseNotInitializedError("Vectors backend not initialized")

        if not chunks_with_vectors:
            return 0

        try:
            # Normalize and validate vectors
            normalized_vectors = []
            timestamp = datetime.utcnow().isoformat()

            for chunk in chunks_with_vectors:
                # Validate required fields
                if "chunk_id" not in chunk:
                    raise ValueError("Missing required field: chunk_id")
                if "vector" not in chunk:
                    raise ValueError(f"Missing vector for chunk: {chunk['chunk_id']}")

                # Auto-detect vector dimension from first vector
                vector = chunk["vector"]
                if self.vector_dim is None:
                    self.vector_dim = len(vector)
                    logger.info(
                        f"Auto-detected vector dimension: {self.vector_dim}D from first chunk"
                    )

                # Validate vector dimensions match detected/expected dimension
                if len(vector) != self.vector_dim:
                    raise ValueError(
                        f"Invalid vector dimension for chunk {chunk['chunk_id']}: "
                        f"expected {self.vector_dim}, got {len(vector)}"
                    )

                # Required fields
                normalized = {
                    "chunk_id": chunk["chunk_id"],
                    "vector": vector,
                    "file_path": chunk["file_path"],
                    "content": chunk["content"],
                    "language": chunk["language"],
                    "start_line": chunk["start_line"],
                    "end_line": chunk["end_line"],
                    "chunk_type": chunk["chunk_type"],
                    "name": chunk["name"],
                }

                # Optional fields with defaults
                normalized["function_name"] = chunk.get("function_name", "")
                normalized["class_name"] = chunk.get("class_name", "")
                normalized["project_name"] = chunk.get("project_name", "")
                normalized["hierarchy_path"] = chunk.get("hierarchy_path", "")

                # Metadata
                normalized["embedded_at"] = timestamp
                normalized["model_version"] = model_version

                normalized_vectors.append(normalized)

            # Convert to PyArrow Table with explicit schema to avoid type inference issues
            # This ensures vectors are treated as fixed-size lists, not variable-length lists
            schema = _create_vectors_schema(self.vector_dim)

            # Convert normalized vectors to PyArrow Table with explicit schema
            # This prevents LanceDB from inferring variable-length list types
            import pyarrow as pa_table_module

            pa_table = pa_table_module.Table.from_pylist(
                normalized_vectors, schema=schema
            )

            # Deduplicate: Extract chunk_ids to remove before adding new vectors
            chunk_ids_to_add = [row["chunk_id"] for row in normalized_vectors]

            # Create or append to table
            if self._table is None:
                # Idempotent create: exist_ok=True opens the existing table
                # if present, creates it otherwise.
                self._table = self._idempotent_create_table(
                    self.table_name, pa_table, schema=schema
                )

                # Check whether we got a fresh table or an existing one
                row_count = self._table.count_rows()
                if row_count > len(normalized_vectors):
                    # Existing table was opened — verify dimensions and evolve schema
                    existing_schema = self._table.schema
                    vector_field = existing_schema.field("vector")
                    existing_dim = getattr(vector_field.type, "list_size", None)

                    if existing_dim is not None and existing_dim != self.vector_dim:
                        # Dimension mismatch — force-replace the table
                        logger.warning(
                            f"Vector dimension mismatch: existing table has {existing_dim}D, "
                            f"new data has {self.vector_dim}D. Recreating table."
                        )
                        self._table = self._idempotent_create_table(
                            self.table_name, pa_table, schema=schema, mode="overwrite"
                        )
                    else:
                        # Dimensions match — evolve schema if needed, then append
                        # Fix 5: skip schema set-diff when schema is known stable
                        if not self._schema_verified:
                            self._evolve_schema_if_needed(existing_schema, schema)
                        self._delete_chunk_ids(chunk_ids_to_add)
                        self._table.add(pa_table, mode="append")
                        # Fix 5: mark schema as verified after first successful write
                        self._schema_verified = True
                        logger.debug(
                            f"Appended {len(normalized_vectors)} vectors to existing table"
                        )
                else:
                    # Fix 5: freshly created table has the current schema by definition
                    self._schema_verified = True
                    logger.debug(
                        f"Created vectors table with {len(normalized_vectors)} vectors "
                        f"(dimension: {self.vector_dim})"
                    )
            else:
                # Table handle already open — verify dimensions and evolve schema
                existing_schema = self._table.schema
                vector_field = existing_schema.field("vector")
                existing_dim = getattr(vector_field.type, "list_size", None)

                if existing_dim is not None and existing_dim != self.vector_dim:
                    # Dimension mismatch — force-replace the table
                    logger.warning(
                        f"Vector dimension mismatch during append: "
                        f"table has {existing_dim}D, new data has {self.vector_dim}D. "
                        f"Recreating table."
                    )
                    self._table = self._idempotent_create_table(
                        self.table_name, pa_table, schema=schema, mode="overwrite"
                    )
                    # Fix 5: schema cache is invalid after table recreation
                    self._schema_verified = False
                else:
                    # Fix 5: skip per-call schema set-diff when schema is known stable
                    if not self._schema_verified:
                        self._evolve_schema_if_needed(existing_schema, schema)

                    # Deduplicate: Delete existing vectors with same chunk_ids before appending
                    self._delete_chunk_ids(chunk_ids_to_add)

                    # OPTIMIZATION: Use mode='append' for faster bulk inserts
                    # This defers index updates until later, improving write throughput
                    self._table.add(pa_table, mode="append")
                    # Fix 5: mark schema as verified after first successful write
                    self._schema_verified = True
                    logger.debug(
                        f"Added {len(normalized_vectors)} vectors to table (append mode)"
                    )

            # Fix 4: count rows (not calls) for compaction threshold.
            # Old threshold: 500 calls × 4096 rows/call = 2M rows (never fires in practice).
            # New threshold: compact every ~20K rows regardless of call count.
            self._append_count += len(normalized_vectors)
            if self._append_count % 20_000 < len(normalized_vectors):
                self._compact_table()

            return len(normalized_vectors)

        except Exception as e:
            logger.error(f"Failed to add vectors: {e}")
            raise DatabaseError(f"Failed to add vectors: {e}") from e

    def _compact_table(self) -> None:
        """Compact LanceDB table to merge small fragments.

        LanceDB creates one file per append operation. During large reindexing,
        this accumulates thousands of small files, exhausting file descriptors.
        Periodic compaction merges fragments to reduce file count.

        This is called every 500 appends to keep file count manageable.
        Failures are non-fatal (best-effort optimization).

        WORKAROUND: Skipped on macOS to avoid SIGBUS crash caused by memory conflict
        between PyTorch MPS memory-mapped model files and LanceDB compaction operations.
        """
        if platform.system() == "Darwin":
            logger.debug(
                "Skipping vectors table compaction on macOS to avoid SIGBUS crash "
                "(PyTorch MPS + LanceDB compaction memory conflict)"
            )
            return

        if self._table is None:
            return

        # Linux guard: skip compaction for large tables to prevent arrow offset overflow
        # documented in lance issue #3330. compact_files() on tables > 100k rows can
        # trigger an int32 overflow in the offset buffer on Linux.
        try:
            row_count = self._table.count_rows()
            if row_count > 100_000:
                logger.debug(
                    "Skipping compaction: table has %d rows (lance#3330 safety)",
                    row_count,
                )
                return
        except Exception:
            pass

        try:
            # LanceDB 0.5+ API: compact_files() merges data fragments
            self._table.compact_files()
            logger.debug(
                f"Compacted vectors table after {self._append_count} appends (reduced fragment count)"
            )
        except AttributeError:
            # Fallback for older LanceDB versions without compact_files()
            try:
                self._table.cleanup_old_versions()
                logger.debug("Cleaned up old versions (compaction not available)")
            except Exception as e:
                logger.debug(f"Compaction/cleanup not available: {e}")
        except Exception as e:
            # Compaction is best-effort, don't fail on errors
            logger.debug(f"Compaction failed (non-fatal): {e}")

    async def search(
        self,
        query_vector: list[float],
        limit: int = 10,
        filters: dict[str, Any] | None = None,
    ) -> list[dict[str, Any]]:
        """Semantic vector search.

        Performs similarity search using LanceDB's IVF_SQ index for fast
        approximate nearest neighbor (ANN) search.

        Args:
            query_vector: Query embedding (dimension must match table)
            limit: Max results to return
            filters: Optional metadata filters:
                - language (str): Filter by programming language
                - file_path (str): Filter by file path
                - chunk_type (str): Filter by chunk type (function, class, etc.)
                - function_name (str): Filter by function name
                - class_name (str): Filter by class name

        Returns:
            List of matching chunks with similarity scores, sorted by relevance:
            [
                {
                    "chunk_id": "abc",
                    "content": "def foo()...",
                    "file_path": "src/main.py",
                    "start_line": 10,
                    "end_line": 20,
                    "language": "python",
                    "chunk_type": "function",
                    "name": "foo",
                    "function_name": "foo",
                    "class_name": "",
                    "hierarchy_path": "MyClass.foo",
                    "similarity": 0.95,
                    "_distance": 0.05
                }
            ]

        Raises:
            DatabaseNotInitializedError: If backend not initialized
            SearchError: If search fails
        """
        if self._table is None:
            # Table doesn't exist - raise error instead of silently returning []
            raise SearchError(
                "Vectors table not found. Index may be corrupted or not built yet. "
                "Run 'mcp-vector-search index' to rebuild the index."
            )

        # Validate query vector dimensions (if dimension is known)
        if self.vector_dim is not None and len(query_vector) != self.vector_dim:
            raise SearchError(
                f"Invalid query vector dimension: "
                f"expected {self.vector_dim}, got {len(query_vector)}"
            )

        try:
            # Build LanceDB query with cosine metric.
            # nprobes and refine_factor enable two-stage ANN retrieval:
            #   - nprobes=20: scan 20 IVF partitions (higher = better recall, slower)
            #   - refine_factor=5: re-rank 5x candidates with exact distances
            # LanceDB silently ignores these when no ANN index exists (brute-force).
            search = (
                self._table.search(query_vector)
                .metric("cosine")
                .nprobes(20)
                .refine_factor(5)
                .limit(limit)
            )

            # Apply metadata filters if provided
            if filters:
                filter_clauses = []
                for key, value in filters.items():
                    if value is not None:
                        # Handle different filter types
                        if isinstance(value, str):
                            # Escape single quotes in value
                            escaped_value = value.replace("'", "''")
                            filter_clauses.append(f"{key} = '{escaped_value}'")
                        elif isinstance(value, list):
                            # Support IN queries
                            escaped_values = [v.replace("'", "''") for v in value]
                            values_str = ", ".join(f"'{v}'" for v in escaped_values)
                            filter_clauses.append(f"{key} IN ({values_str})")
                        else:
                            filter_clauses.append(f"{key} = {value}")

                if filter_clauses:
                    where_clause = " AND ".join(filter_clauses)
                    search = search.where(where_clause)

            # Execute search
            results = search.to_list()

            # Convert LanceDB results to standard format
            search_results = []
            for result in results:
                # LanceDB returns _distance (L2 distance for vectors)
                # Since vectors aren't normalized, we get large L2 distances
                # Convert cosine distance to similarity score
                # Cosine distance ranges from 0 (identical) to 2 (opposite)
                # Convert to similarity: 0 distance -> 1.0 similarity, 2 distance -> 0.0
                distance = result.get("_distance", 0.0)
                similarity = max(0.0, 1.0 - (distance / 2.0))

                search_result = {
                    "chunk_id": result["chunk_id"],
                    "content": result["content"],
                    "file_path": result["file_path"],
                    "start_line": result["start_line"],
                    "end_line": result["end_line"],
                    "language": result["language"],
                    "chunk_type": result["chunk_type"],
                    "name": result["name"],
                    "function_name": result.get("function_name", ""),
                    "class_name": result.get("class_name", ""),
                    "project_name": result.get("project_name", ""),
                    "hierarchy_path": result.get("hierarchy_path", ""),
                    "similarity": similarity,
                    "_distance": distance,
                }
                search_results.append(search_result)

            logger.debug(
                f"Vector search returned {len(search_results)} results (limit: {limit})"
            )
            return search_results

        except Exception as e:
            # Check for corruption and auto-recover
            if self._handle_corrupt_table(e, self.table_name):
                logger.error(
                    "Vectors table corrupted. Search unavailable. Run 'mcp-vector-search index' to rebuild."
                )
                raise SearchError(
                    "Index corrupted. Run 'mcp-vector-search index' to rebuild."
                ) from e

            logger.error(f"Vector search failed: {e}")
            raise SearchError(f"Vector search failed: {e}") from e

    async def search_by_file(
        self, file_path: str, query_vector: list[float], limit: int = 10
    ) -> list[dict[str, Any]]:
        """Search within a specific file.

        Convenience method that applies file_path filter to search.

        Args:
            file_path: Path to file to search within
            query_vector: Query embedding (dimension must match table)
            limit: Max results to return

        Returns:
            List of matching chunks from specified file
        """
        return await self.search(
            query_vector, limit=limit, filters={"file_path": file_path}
        )

    async def delete_file_vectors(self, file_path: str) -> int:
        """Delete all vectors for a file (for re-indexing).

        Args:
            file_path: Path to file whose vectors should be deleted

        Returns:
            Number of vectors deleted

        Raises:
            DatabaseNotInitializedError: If backend not initialized
            DatabaseError: If deletion fails
        """
        if self._table is None:
            return 0

        try:
            # Fix 3: column-projected scanner to count before deletion — avoids
            # loading 768-float vectors just to count rows for a file.
            escaped = file_path.replace(chr(39), chr(39) * 2)
            filter_expr = f"file_path = '{escaped}'"
            try:
                scanner = self._table.to_lance().scanner(
                    filter=filter_expr,
                    columns=["chunk_id"],
                )
                count = len(scanner.to_table())
            except Exception as proj_err:
                logger.warning(
                    "Column-projected pre-delete count failed, falling back: %s",
                    proj_err,
                )
                df_fallback = self._table.to_pandas().query(
                    f"file_path == '{file_path}'"
                )
                count = len(df_fallback)

            if count == 0:
                return 0

            # Delete matching rows
            self._table.delete(filter_expr)

            logger.debug(f"Deleted {count} vectors for file: {file_path}")
            return count

        except Exception as e:
            logger.error(f"Failed to delete vectors for {file_path}: {e}")
            raise DatabaseError(f"Failed to delete vectors: {e}") from e

    async def delete_files_batch(self, file_paths: list[str]) -> int:
        """Delete vectors for multiple files in batched IN expressions.

        Uses IN clauses capped at DELETE_BATCH_LIMIT per batch to prevent
        DataFusion's recursive descent parser from stack-overflowing on large
        OR chains, which causes SEGV (signal 139) on Linux.

        Args:
            file_paths: List of relative file paths whose vectors should be deleted

        Returns:
            Total number of file-path batches processed (not exact vector count,
            since LanceDB delete() returns None)
        """
        if self._table is None or not file_paths:
            return 0

        deleted_count = 0
        total_batches = (len(file_paths) + DELETE_BATCH_LIMIT - 1) // DELETE_BATCH_LIMIT

        for batch_num, i in enumerate(
            range(0, len(file_paths), DELETE_BATCH_LIMIT), start=1
        ):
            batch = file_paths[i : i + DELETE_BATCH_LIMIT]

            # Escape single quotes and build SQL IN clause.
            # IN instead of chained OR keeps DataFusion's parse tree shallow,
            # preventing stack overflow (SEGV signal 139) on Linux with large batches.
            escaped = [fp.replace("'", "''") for fp in batch]
            values_sql = ", ".join(f"'{fp}'" for fp in escaped)
            filter_expr = f"file_path IN ({values_sql})"

            try:
                self._table.delete(filter_expr)
                deleted_count += len(batch)
                logger.debug(
                    "Deleted vectors batch %d/%d (%d files, running total: %d)",
                    batch_num,
                    total_batches,
                    len(batch),
                    deleted_count,
                )
            except Exception as e:
                # Handle LanceDB "Not found" errors gracefully; continue other batches
                error_msg = str(e).lower()
                if "not found" in error_msg:
                    logger.debug(
                        "No vectors to delete for batch %d/%d (not in index)",
                        batch_num,
                        total_batches,
                    )
                else:
                    logger.error(
                        "Failed to delete vectors batch %d/%d: %s",
                        batch_num,
                        total_batches,
                        e,
                    )

        logger.debug("Batch delete complete: %d files processed", deleted_count)
        return deleted_count

    async def update_file_path(self, old_path: str, new_path: str) -> int:
        """Update file_path for all vectors of a moved/renamed file.

        Uses LanceDB table.update() for in-place metadata update — no re-embedding needed.
        Safe on macOS (update doesn't trigger mmap compaction like delete does).

        Args:
            old_path: Current (old) relative file path stored in the table
            new_path: New relative file path to set

        Returns:
            Number of rows updated (0 if table not initialized or update fails)
        """
        if self._table is None:
            return 0
        try:
            escaped_old = old_path.replace("'", "''")
            result = self._table.update(
                where=f"file_path = '{escaped_old}'",
                values={"file_path": new_path},
            )
            rows = getattr(result, "rows_updated", 0) if result else 0
            logger.debug(
                "Updated vector file_path: %s → %s (%d vectors)",
                old_path,
                new_path,
                rows,
            )
            return rows
        except Exception as e:
            logger.error(
                "Failed to update vector file_path %s → %s: %s", old_path, new_path, e
            )
            return 0

    async def get_chunk_vector(self, chunk_id: str) -> list[float] | None:
        """Get vector for a specific chunk.

        Args:
            chunk_id: Chunk identifier

        Returns:
            Vector embedding if found, None otherwise
        """
        if self._table is None:
            return None

        try:
            df = self._table.to_pandas().query(f"chunk_id == '{chunk_id}'")

            if df.empty:
                return None

            # Return first matching vector
            # Vector might be a numpy array, pyarrow array, or list
            vector = df.iloc[0]["vector"]
            if isinstance(vector, list):
                return vector
            # Convert numpy/pyarrow arrays to list
            try:
                return vector.tolist()
            except AttributeError:
                # If it's already iterable but not a list
                return list(vector)

        except Exception as e:
            logger.warning(f"Failed to get vector for chunk {chunk_id}: {e}")
            return None

    async def get_chunk_vectors_batch(
        self, chunk_ids: list[str]
    ) -> dict[str, list[float]]:
        """Get vectors for multiple chunks in a single query.

        More efficient than calling get_chunk_vector() in a loop.

        Args:
            chunk_ids: List of chunk identifiers

        Returns:
            Dictionary mapping chunk_id to vector embedding
            Only includes chunk_ids that were found in the table
        """
        if self._table is None or not chunk_ids:
            return {}

        try:
            # Build SQL IN clause with proper escaping
            # Batch into chunks of 500 to avoid SQL query length limits
            batch_size = 500
            all_vectors: dict[str, list[float]] = {}

            for i in range(0, len(chunk_ids), batch_size):
                batch = chunk_ids[i : i + batch_size]
                id_list = ", ".join(
                    f"'{cid.replace(chr(39), chr(39) * 2)}'" for cid in batch
                )

                # Query for this batch
                df = self._table.to_pandas().query(f"chunk_id in [{id_list}]")

                # Extract vectors from results
                for _, row in df.iterrows():
                    chunk_id = row["chunk_id"]
                    vector = row["vector"]

                    # Convert to list if needed
                    if isinstance(vector, list):
                        all_vectors[chunk_id] = vector
                    else:
                        try:
                            all_vectors[chunk_id] = vector.tolist()
                        except AttributeError:
                            all_vectors[chunk_id] = list(vector)

            logger.debug(
                f"Retrieved {len(all_vectors)} vectors from {len(chunk_ids)} requested chunk_ids"
            )
            return all_vectors

        except Exception as e:
            logger.warning(f"Failed to get vectors for batch: {e}")
            return {}

    async def has_vector(self, chunk_id: str) -> bool:
        """Check if chunk already has a vector.

        Args:
            chunk_id: Chunk identifier

        Returns:
            True if vector exists, False otherwise
        """
        if self._table is None:
            return False

        try:
            # Fix 3: column-projected scanner avoids loading 768-float vectors
            escaped = chunk_id.replace(chr(39), chr(39) * 2)
            scanner = self._table.to_lance().scanner(
                filter=f"chunk_id = '{escaped}'",
                columns=["chunk_id"],
                limit=1,
            )
            return len(scanner.to_table()) > 0
        except Exception as e:
            logger.warning(
                "Column-projected has_vector check failed, falling back: %s", e
            )
            try:
                df = self._table.to_pandas().query(f"chunk_id == '{chunk_id}'")
                return not df.empty
            except Exception as e2:
                logger.warning(f"Failed to check vector for chunk {chunk_id}: {e2}")
                return False

    async def get_stats(self) -> dict[str, Any]:
        """Get vector statistics.

        Returns:
            Dictionary with vector counts and metadata:
            {
                "total": 1000,
                "files": 50,
                "languages": {"python": 30, "javascript": 20, ...},
                "chunk_types": {"function": 40, "class": 10, ...},
                "models": {"all-MiniLM-L6-v2": 1000},
                "avg_vector_norm": 1.0
            }
        """
        if self._table is None:
            return {
                "total": 0,
                "files": 0,
                "languages": {},
                "chunk_types": {},
                "models": {},
                "avg_vector_norm": 0.0,
            }

        try:
            # Fix 3: use count_rows() for total and column-projected scanner for
            # aggregation metadata — avoids loading 768-float vectors (300MB for 100K rows).
            total = self._table.count_rows()

            try:
                scanner = self._table.to_lance().scanner(
                    columns=["file_path", "language", "chunk_type", "model_version"]
                )
                result = scanner.to_table()
                df = result.to_pandas()

                file_count = df["file_path"].nunique()
                language_counts = df["language"].value_counts().to_dict()
                chunk_type_counts = df["chunk_type"].value_counts().to_dict()
                model_counts = df["model_version"].value_counts().to_dict()
            except Exception as proj_err:
                logger.warning(
                    "Column-projected stats scanner failed, falling back: %s", proj_err
                )
                df_full = self._table.to_pandas()
                file_count = df_full["file_path"].nunique()
                language_counts = df_full["language"].value_counts().to_dict()
                chunk_type_counts = df_full["chunk_type"].value_counts().to_dict()
                model_counts = df_full["model_version"].value_counts().to_dict()

            return {
                "total": total,
                "files": file_count,
                "languages": language_counts,
                "chunk_types": chunk_type_counts,
                "models": model_counts,
                # avg_vector_norm omitted: computing it requires loading all vectors (300MB+)
                "avg_vector_norm": 0.0,
            }

        except Exception as e:
            # Check for corruption and auto-recover
            if self._handle_corrupt_table(e, self.table_name):
                logger.warning(
                    "Vectors table corrupted. Stats unavailable. Run 'mcp-vector-search index' to rebuild."
                )
                return {
                    "total": 0,
                    "files": 0,
                    "languages": {},
                    "chunk_types": {},
                    "models": {},
                    "avg_vector_norm": 0.0,
                }

            logger.error(f"Failed to get vector stats: {e}")
            return {
                "total": 0,
                "files": 0,
                "languages": {},
                "chunk_types": {},
                "models": {},
                "avg_vector_norm": 0.0,
            }

    async def rebuild_index(self) -> None:
        """Rebuild the vector index for ANN search.

        Creates an IVF_SQ approximate nearest neighbor index after bulk inserts
        to enable fast similarity search at scale.  Skipped for small datasets
        (< 4,096 rows) where brute-force is faster and IVF KMeans produces
        mostly-empty clusters.  Non-fatal: if index creation fails search
        continues with exact brute-force scan.

        Note: This is an expensive operation and should not be called after every add.
        """
        if self._table is None:
            logger.warning("Cannot rebuild index: table not initialized")
            return

        try:
            import math

            row_count = self._table.count_rows()

            # Skip index for small datasets — brute-force is faster and the
            # IVF KMeans training produces mostly-empty clusters when N < 4096.
            # Lance itself warns "dataset is too small to have a meaningful
            # index (N < 4096)".  For these datasets brute-force is fast and
            # avoids the noisy WARN logs about empty clusters.
            if row_count < 4096:
                logger.info(
                    f"Skipping vector index creation: {row_count:,} rows "
                    f"(< 4,096 minimum for meaningful IVF index)"
                )
                return

            # Determine vector dimension from a sample row
            sample = self._table.search().limit(1).to_list()
            if not sample or "vector" not in sample[0]:
                logger.warning("Cannot determine vector dimension for index")
                return

            vector_dim = len(sample[0]["vector"])

            # Adaptive partition count: sqrt(N) clamped to [16, 512]
            # Rule of thumb from LanceDB docs: good recall vs. latency balance
            num_partitions = min(512, max(16, int(math.sqrt(row_count))))

            # Cap partitions so each has ≥4096 rows for clean KMeans training.
            # Lance warns "dataset is too small" when partition size < 4096,
            # producing ~30 lines of noisy WARN output per index build.
            max_for_clean_kmeans = max(1, row_count // 4096)
            if num_partitions > max_for_clean_kmeans:
                num_partitions = max_for_clean_kmeans

            logger.info(
                f"Creating IVF_SQ vector index: {row_count:,} rows, {vector_dim}d, "
                f"{num_partitions} partitions (int8 scalar quantization)"
            )

            # Suppress Rust-level KMeans "empty cluster" warnings from Lance.
            # These are informational (index still works) but produce noisy
            # stderr output that bypasses Python logging.
            import os as _os

            _stderr_fd = sys.stderr.fileno()
            _saved = _os.dup(_stderr_fd)
            _devnull = _os.open(_os.devnull, _os.O_WRONLY)
            _os.dup2(_devnull, _stderr_fd)
            try:
                self._table.create_index(
                    metric="cosine",
                    num_partitions=num_partitions,
                    index_type="IVF_SQ",
                    replace=True,
                )
            finally:
                _os.dup2(_saved, _stderr_fd)
                _os.close(_saved)
                _os.close(_devnull)

            logger.info("Vector index created successfully")

        except Exception as e:
            # Check for corruption and auto-recover
            if self._handle_corrupt_table(e, self.table_name):
                logger.warning(
                    "Vectors table corrupted during index rebuild. Skipped. "
                    "Run 'mcp-vector-search index' to rebuild."
                )
                return

            # Non-fatal — search falls back to brute-force automatically
            logger.warning(f"Vector index creation failed (non-fatal): {e}")

    async def get_unembedded_chunk_ids(self, all_chunk_ids: list[str]) -> list[str]:
        """Find which chunks don't have vectors yet.

        This enables incremental embedding - only embed chunks that don't
        already have vectors in the table.

        Args:
            all_chunk_ids: List of all chunk IDs from chunks table

        Returns:
            List of chunk IDs that don't have vectors yet
        """
        if self._table is None or not all_chunk_ids:
            # No vectors table yet, all chunks need embedding
            return all_chunk_ids

        try:
            # Fix 1: column-projected scanner — loads only chunk_id strings instead of
            # ALL columns including 768-float vectors (300MB for 100K rows).
            try:
                scanner = self._table.to_lance().scanner(columns=["chunk_id"])
                result = scanner.to_table()
                embedded_ids = set(result.column("chunk_id").to_pylist())
            except Exception as scan_err:
                logger.warning(
                    "Column-projected chunk_id scanner failed, falling back: %s",
                    scan_err,
                )
                df = self._table.to_pandas()
                embedded_ids = set(df["chunk_id"].values)

            # Find chunks without vectors
            unembedded = [cid for cid in all_chunk_ids if cid not in embedded_ids]

            logger.debug(
                f"Found {len(unembedded)} unembedded chunks out of {len(all_chunk_ids)} total"
            )
            return unembedded

        except Exception as e:
            logger.error(f"Failed to get unembedded chunk IDs: {e}")
            # On error, assume all chunks need embedding
            return all_chunk_ids

    async def close(self) -> None:
        """Close database connections.

        LanceDB doesn't require explicit closing, but we set references to None
        for consistency with other backends.
        """
        self._table = None
        self._db = None
        logger.debug("Vectors backend closed")

    async def __aenter__(self) -> "VectorsBackend":
        """Async context manager entry."""
        await self.initialize()
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
        """Async context manager exit."""
        await self.close()
