//! HNSW Index core implementation
//!
//! Contains the HNSWIndex struct and core methods for insert, search, delete.

use std::collections::{BinaryHeap, HashMap, HashSet};
use std::sync::atomic::{AtomicUsize, AtomicU64, AtomicBool, Ordering as AtomicOrdering};
use std::sync::Arc;

use parking_lot::RwLock;
use rand::Rng;
use ordered_float::OrderedFloat;

use crate::types::{HNSWConfig, DistanceMetric};
use crate::vector::distance::{Distance, DistanceCalculator};
use crate::vector::quantizer::{ProductQuantizer, PQConfig};

use super::node::HNSWNode;
use super::types::{
    Operation, Transaction, Savepoint,
    Neighbor, MaxNeighbor, SearchResult, HNSWError,
};

/// HNSW Index for approximate nearest neighbor search
/// 
/// Optimizations:
/// - Fine-grained locking for concurrent access
/// - Arc-based vector sharing to reduce cloning
/// - RNG heuristic for better neighbor selection
pub struct HNSWIndex {
    /// Configuration
    pub(crate) config: HNSWConfig,
    
    /// Distance calculator
    pub(crate) distance: DistanceCalculator,
    
    /// All nodes in the index (with fine-grained locking)
    pub(crate) nodes: RwLock<Vec<HNSWNode>>,
    
    /// Per-node locks for neighbor modifications
    pub(crate) neighbor_locks: RwLock<Vec<RwLock<()>>>,
    
    /// Entry point (node with highest level)
    pub(crate) entry_point: RwLock<Option<usize>>,
    
    /// Current maximum level
    pub(crate) max_level: AtomicUsize,
    
    /// ID mapping: external_id -> internal_id
    pub(crate) id_map: RwLock<HashMap<String, usize>>,
    
    /// Reverse ID mapping: internal_id -> external_id
    pub(crate) reverse_id_map: RwLock<Vec<String>>,
    
    /// Level multiplier for random level generation
    pub(crate) level_mult: f64,
    
    /// Random seed
    pub(crate) seed: u64,
    
    /// Set of deleted internal node IDs (P0-2: soft delete)
    pub(crate) deleted: RwLock<HashSet<usize>>,
    
    // ========== P0-3: Transaction Support ==========
    /// Current active transaction
    pub(crate) active_transaction: RwLock<Option<Transaction>>,
    
    /// Transaction ID counter
    pub(crate) transaction_counter: AtomicU64,
    
    /// Savepoints in the current transaction
    pub(crate) savepoints: RwLock<Vec<Savepoint>>,
    
    // ========== P1-1: PQ Integration ==========
    /// Product Quantizer (None if compression disabled)
    pub(crate) quantizer: RwLock<Option<ProductQuantizer>>,
    
    /// PQ codes for each node (empty if compression disabled)
    pub(crate) pq_codes: RwLock<Vec<Vec<u8>>>,
    
    /// Whether to store original vectors
    pub(crate) store_original: bool,
    
    // ========== Optimization flags ==========
    /// Whether to use RNG heuristic for neighbor selection
    pub(crate) use_rng_heuristic: AtomicBool,
}

impl HNSWIndex {
    /// Create a new HNSW index with the given configuration
    pub fn new(config: HNSWConfig) -> Self {
        let level_mult = 1.0 / (config.m as f64).ln();
        let seed = config.seed.unwrap_or_else(|| {
            use std::time::{SystemTime, UNIX_EPOCH};
            SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .unwrap()
                .as_nanos() as u64
        });
        
        Self {
            config: config.clone(),
            distance: DistanceCalculator::new(DistanceMetric::Cosine),
            nodes: RwLock::new(Vec::with_capacity(config.max_elements)),
            neighbor_locks: RwLock::new(Vec::with_capacity(config.max_elements)),
            entry_point: RwLock::new(None),
            max_level: AtomicUsize::new(0),
            id_map: RwLock::new(HashMap::new()),
            reverse_id_map: RwLock::new(Vec::new()),
            level_mult,
            seed,
            deleted: RwLock::new(HashSet::new()),
            // P0-3: Transaction support
            active_transaction: RwLock::new(None),
            transaction_counter: AtomicU64::new(0),
            savepoints: RwLock::new(Vec::new()),
            // P1-1: PQ support (disabled by default)
            quantizer: RwLock::new(None),
            pq_codes: RwLock::new(Vec::new()),
            store_original: true,
            // Optimization flags
            use_rng_heuristic: AtomicBool::new(true),
        }
    }
    
    /// Enable or disable RNG heuristic for neighbor selection
    pub fn set_rng_heuristic(&self, enabled: bool) {
        self.use_rng_heuristic.store(enabled, AtomicOrdering::SeqCst);
    }
    
    /// Check if RNG heuristic is enabled
    pub fn rng_heuristic_enabled(&self) -> bool {
        self.use_rng_heuristic.load(AtomicOrdering::SeqCst)
    }
    
    /// Create a new HNSW index with PQ compression enabled
    pub fn with_pq(config: HNSWConfig, pq_config: PQConfig) -> Self {
        let mut index = Self::new(config.clone());
        let quantizer = ProductQuantizer::new(config.dimension, pq_config);
        *index.quantizer.write() = Some(quantizer);
        index.store_original = true; // Store originals for training
        index
    }
    
    /// Create a new HNSW index with PQ compression (no original vectors)
    pub fn with_pq_only(config: HNSWConfig, pq_config: PQConfig) -> Self {
        let mut index = Self::new(config.clone());
        let quantizer = ProductQuantizer::new(config.dimension, pq_config);
        *index.quantizer.write() = Some(quantizer);
        index.store_original = false;
        index
    }
    
    /// Create with a specific distance metric
    pub fn with_distance_metric(mut self, metric: DistanceMetric) -> Self {
        self.distance = DistanceCalculator::new(metric);
        self
    }
    
    /// Get the number of elements in the index
    pub fn len(&self) -> usize {
        self.nodes.read().len()
    }
    
    /// Check if the index is empty
    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }
    
    /// Get the configuration
    pub fn config(&self) -> &HNSWConfig {
        &self.config
    }
    
    /// Generate a random level for a new node
    pub(crate) fn random_level(&self, _node_id: usize) -> usize {
        // Use node_id as part of seed for determinism (currently unused but reserved)
        let mut rng = rand::thread_rng();
        let uniform: f64 = rng.gen();
        
        let level = (-uniform.ln() * self.level_mult).floor() as usize;
        level.min(self.config.m.max(1) - 1) // Cap at reasonable maximum
    }
    
    /// Insert a vector with an external ID
    /// 
    /// Optimized to minimize vector cloning using Arc sharing.
    pub fn insert(&self, external_id: impl Into<String>, vector: Vec<f32>) -> Result<usize, HNSWError> {
        let external_id = external_id.into();
        
        // Validate vector dimension
        if vector.len() != self.config.dimension {
            return Err(HNSWError::DimensionMismatch {
                expected: self.config.dimension,
                got: vector.len(),
            });
        }
        
        // Check if ID already exists
        {
            let id_map = self.id_map.read();
            if id_map.contains_key(&external_id) {
                return Err(HNSWError::DuplicateId(external_id));
            }
        }
        
        // Wrap vector in Arc for sharing (no clone needed after this)
        let vector_arc = Arc::new(vector);
        
        // Get internal ID and create node
        let internal_id = {
            let mut nodes = self.nodes.write();
            let id = nodes.len();
            
            // Check capacity
            if id >= self.config.max_elements {
                return Err(HNSWError::CapacityExceeded);
            }
            
            let level = self.random_level(id);
            // Use Arc-based node creation (no vector clone)
            let node = HNSWNode::with_arc_vector(
                Arc::clone(&vector_arc), 
                level, 
                self.config.m_max0, 
                self.config.m
            );
            nodes.push(node);
            
            // Add per-node lock
            let mut locks = self.neighbor_locks.write();
            locks.push(RwLock::new(()));
            
            id
        };
        
        // Update ID mappings
        {
            let mut id_map = self.id_map.write();
            let mut reverse_map = self.reverse_id_map.write();
            id_map.insert(external_id.clone(), internal_id);
            reverse_map.push(external_id.clone());
        }
        
        // P1-1: Encode with PQ if trained (use Arc reference, no clone)
        {
            let quantizer = self.quantizer.read();
            if let Some(ref q) = *quantizer {
                if q.is_trained() {
                    if let Ok(code) = q.encode(&vector_arc) {
                        let mut pq_codes = self.pq_codes.write();
                        // Ensure pq_codes is large enough
                        while pq_codes.len() <= internal_id {
                            pq_codes.push(Vec::new());
                        }
                        pq_codes[internal_id] = code;
                    }
                }
            }
        }
        
        // Insert into graph
        self.insert_node(internal_id)?;
        
        // P0-3: Record transaction operation
        // Clone vector only for transaction log (unavoidable for rollback support)
        self.record_operation(Operation::Insert {
            external_id,
            internal_id,
            vector: (*vector_arc).clone(),
        });
        
        Ok(internal_id)
    }
    
    /// Batch insert multiple vectors (P1-3: parallel batch insert)
    /// 
    /// This method is optimized for inserting multiple vectors at once,
    /// providing better performance than individual inserts.
    /// 
    /// Optimizations:
    /// - Pre-allocates all nodes at once
    /// - Uses Arc for vector sharing (no cloning during graph construction)
    /// - Sorts by level for better graph quality
    pub fn insert_batch(&self, items: Vec<(String, Vec<f32>)>) -> Result<Vec<usize>, HNSWError> {
        
        if items.is_empty() {
            return Ok(Vec::new());
        }
        
        // Step 1: Validate all vectors
        for (external_id, vector) in &items {
            if vector.len() != self.config.dimension {
                return Err(HNSWError::DimensionMismatch {
                    expected: self.config.dimension,
                    got: vector.len(),
                });
            }
            
            // Check for duplicates in input
            let id_map = self.id_map.read();
            if id_map.contains_key(external_id) {
                return Err(HNSWError::DuplicateId(external_id.clone()));
            }
        }
        
        // Step 2: Wrap vectors in Arc and allocate all nodes at once
        let start_id = {
            let mut nodes = self.nodes.write();
            let mut locks = self.neighbor_locks.write();
            let start = nodes.len();
            
            // Check capacity
            if start + items.len() > self.config.max_elements {
                return Err(HNSWError::CapacityExceeded);
            }
            
            // Pre-allocate nodes with random levels using Arc vectors (no clone!)
            for (_, vector) in &items {
                let id = nodes.len();
                let level = self.random_level(id);
                let vector_arc = Arc::new(vector.clone()); // Clone once into Arc
                let node = HNSWNode::with_arc_vector(vector_arc, level, self.config.m_max0, self.config.m);
                nodes.push(node);
                locks.push(RwLock::new(()));
            }
            
            start
        };
        
        // Step 3: Update ID mappings
        {
            let mut id_map = self.id_map.write();
            let mut reverse_map = self.reverse_id_map.write();
            
            for (i, (external_id, _)) in items.iter().enumerate() {
                let internal_id = start_id + i;
                id_map.insert(external_id.clone(), internal_id);
                reverse_map.push(external_id.clone());
            }
        }
        
        // Step 4: Get sorted indices by level (higher levels first for better graph construction)
        let mut indices_with_levels: Vec<(usize, usize)> = {
            let nodes = self.nodes.read();
            (0..items.len())
                .map(|i| (start_id + i, nodes[start_id + i].max_level))
                .collect()
        };
        indices_with_levels.sort_by(|a, b| b.1.cmp(&a.1)); // Sort descending by level
        
        // Step 5: Insert nodes into graph structure
        // Higher level nodes first, then lower level nodes can use them as entry points
        for (node_id, _level) in &indices_with_levels {
            self.insert_node(*node_id)?;
        }
        
        // Return all internal IDs in original order
        let result: Vec<usize> = (0..items.len()).map(|i| start_id + i).collect();
        Ok(result)
    }
    
    /// Batch insert with parallel graph construction (experimental)
    /// 
    /// Uses parallel processing for neighbor search but sequential graph updates.
    /// This can provide speedup for large batches.
    #[allow(dead_code)]
    pub fn insert_batch_parallel(&self, items: Vec<(String, Vec<f32>)>) -> Result<Vec<usize>, HNSWError> {
        
        if items.is_empty() {
            return Ok(Vec::new());
        }
        
        // Validate all vectors
        for (external_id, vector) in &items {
            if vector.len() != self.config.dimension {
                return Err(HNSWError::DimensionMismatch {
                    expected: self.config.dimension,
                    got: vector.len(),
                });
            }
            
            let id_map = self.id_map.read();
            if id_map.contains_key(external_id) {
                return Err(HNSWError::DuplicateId(external_id.clone()));
            }
        }
        
        // Allocate nodes
        let start_id = {
            let mut nodes = self.nodes.write();
            let start = nodes.len();
            
            if start + items.len() > self.config.max_elements {
                return Err(HNSWError::CapacityExceeded);
            }
            
            for (_, vector) in &items {
                let id = nodes.len();
                let level = self.random_level(id);
                let node = HNSWNode::new(vector.clone(), level, self.config.m_max0, self.config.m);
                nodes.push(node);
            }
            
            start
        };
        
        // Update ID mappings
        {
            let mut id_map = self.id_map.write();
            let mut reverse_map = self.reverse_id_map.write();
            
            for (i, (external_id, _)) in items.iter().enumerate() {
                let internal_id = start_id + i;
                id_map.insert(external_id.clone(), internal_id);
                reverse_map.push(external_id.clone());
            }
        }
        
        // Insert in chunks to balance parallelism and graph quality
        let chunk_size = (items.len() / rayon::current_num_threads()).max(1);
        let node_ids: Vec<usize> = (0..items.len()).map(|i| start_id + i).collect();
        
        for chunk in node_ids.chunks(chunk_size) {
            // Sequential insertion within chunks for correctness
            for &node_id in chunk {
                self.insert_node(node_id)?;
            }
        }
        
        Ok(node_ids)
    }
    
    /// Insert a node into the graph structure
    /// 
    /// Optimized with:
    /// - Fine-grained locking: only lock individual nodes during neighbor updates
    /// - Arc-based vector sharing: no cloning during search
    pub(crate) fn insert_node(&self, node_id: usize) -> Result<(), HNSWError> {
        let node_level = {
            let nodes = self.nodes.read();
            nodes[node_id].max_level
        };
        
        // Handle first node
        let mut entry = self.entry_point.write();
        if entry.is_none() {
            *entry = Some(node_id);
            self.max_level.store(node_level, AtomicOrdering::SeqCst);
            return Ok(());
        }
        
        let entry_point_id = entry.unwrap();
        let current_max_level = self.max_level.load(AtomicOrdering::SeqCst);
        
        // Get query vector as Arc reference (no clone!)
        let query_arc = {
            let nodes = self.nodes.read();
            nodes[node_id].vector_arc()
        };
        let query = query_arc.as_ref();
        
        // Find entry point at the node's level
        let mut curr_id = entry_point_id;
        
        // Traverse from top level down to node_level + 1
        for level in (node_level + 1..=current_max_level).rev() {
            curr_id = self.greedy_search_level(query, curr_id, level);
        }
        
        // Insert at each level from node_level down to 0
        for level in (0..=node_level.min(current_max_level)).rev() {
            // Find neighbors at this level
            let neighbors = self.search_level(query, curr_id, self.config.ef_construction, level);
            
            // Select best neighbors
            let max_neighbors = if level == 0 { self.config.m_max0 } else { self.config.m };
            let selected = self.select_neighbors(query, &neighbors, max_neighbors);
            
            // Set new node's neighbors (with fine-grained lock)
            let selected_ids: Vec<usize> = selected.iter().map(|n| n.id).collect();
            self.set_neighbors_fine(node_id, level, selected_ids.clone());
            
            // Add new node to each neighbor's list and prune if needed
            for neighbor in &selected {
                self.add_neighbor_and_prune(neighbor.id, node_id, level, max_neighbors);
            }
            
            // Update curr_id for next level
            if !selected.is_empty() {
                curr_id = selected[0].id;
            }
        }
        
        // Update entry point if this node has higher level
        if node_level > current_max_level {
            *entry = Some(node_id);
            self.max_level.store(node_level, AtomicOrdering::SeqCst);
        }
        
        Ok(())
    }
    
    /// Set neighbors with fine-grained locking
    pub(crate) fn set_neighbors_fine(&self, node_id: usize, level: usize, neighbors: Vec<usize>) {
        // Acquire per-node lock within a scope
        {
            let locks = self.neighbor_locks.read();
            if node_id >= locks.len() {
                return;
            }
            let _node_lock = locks[node_id].write();
            
            // Modify neighbors while holding the lock
            let mut nodes = self.nodes.write();
            if node_id < nodes.len() && level < nodes[node_id].neighbors.len() {
                nodes[node_id].neighbors[level] = neighbors;
            }
        }
    }
    
    /// Add a neighbor to a node and prune if necessary (with fine-grained lock)
    pub(crate) fn add_neighbor_and_prune(&self, node_id: usize, new_neighbor: usize, level: usize, max_neighbors: usize) {
        // Get current neighbors and node vector first (read lock)
        let (current_neighbors, node_vec) = {
            let nodes = self.nodes.read();
            if node_id >= nodes.len() || level >= nodes[node_id].neighbors.len() {
                return;
            }
            let mut neighbors = nodes[node_id].neighbors[level].clone();
            neighbors.push(new_neighbor);
            (neighbors, nodes[node_id].vector_arc())
        };
        
        // Prune if needed (no lock held during computation)
        let final_neighbors = if current_neighbors.len() > max_neighbors {
            self.prune_neighbors_with_arc(&node_vec, &current_neighbors, max_neighbors)
        } else {
            current_neighbors
        };
        
        // Acquire per-node lock and update
        {
            let locks = self.neighbor_locks.read();
            if node_id >= locks.len() {
                return;
            }
            let _node_lock = locks[node_id].write();
            
            // Update neighbors while holding the lock
            let mut nodes = self.nodes.write();
            if node_id < nodes.len() && level < nodes[node_id].neighbors.len() {
                nodes[node_id].neighbors[level] = final_neighbors;
            }
        }
    }
    
    /// Prune neighbors using Arc vector (no cloning)
    pub(crate) fn prune_neighbors_with_arc(&self, node_vec: &Arc<Vec<f32>>, neighbors: &[usize], max_count: usize) -> Vec<usize> {
        if neighbors.len() <= max_count {
            return neighbors.to_vec();
        }
        
        let nodes = self.nodes.read();
        let mut with_dist: Vec<_> = neighbors.iter()
            .filter_map(|&id| {
                if id < nodes.len() {
                    let dist = self.distance.calculate(node_vec, nodes[id].vector());
                    Some((id, OrderedFloat(dist)))
                } else {
                    None
                }
            })
            .collect();
        
        with_dist.sort_by(|a, b| a.1.cmp(&b.1));
        with_dist.truncate(max_count);
        with_dist.into_iter().map(|(id, _)| id).collect()
    }
    
    /// Greedy search to find closest node at a specific level
    /// 
    /// Optimized: uses node.vector() reference instead of cloning
    pub(crate) fn greedy_search_level(&self, query: &[f32], entry_id: usize, level: usize) -> usize {
        let nodes = self.nodes.read();
        let mut curr_id = entry_id;
        let mut curr_dist = self.distance.calculate(query, nodes[curr_id].vector());
        
        loop {
            let mut changed = false;
            
            for &neighbor_id in &nodes[curr_id].neighbors[level] {
                let dist = self.distance.calculate(query, nodes[neighbor_id].vector());
                if dist < curr_dist {
                    curr_dist = dist;
                    curr_id = neighbor_id;
                    changed = true;
                }
            }
            
            if !changed {
                break;
            }
        }
        
        curr_id
    }
    
    /// Search at a specific level to find ef nearest neighbors
    /// 
    /// Optimized: uses node.vector() reference instead of cloning
    pub(crate) fn search_level(&self, query: &[f32], entry_id: usize, ef: usize, level: usize) -> Vec<Neighbor> {
        let nodes = self.nodes.read();
        
        let mut visited = HashSet::new();
        let mut candidates = BinaryHeap::new(); // min-heap
        let mut results = BinaryHeap::new(); // max-heap (using MaxNeighbor)
        
        let entry_dist = self.distance.calculate(query, nodes[entry_id].vector());
        
        visited.insert(entry_id);
        candidates.push(Neighbor { id: entry_id, distance: OrderedFloat(entry_dist) });
        results.push(MaxNeighbor { id: entry_id, distance: OrderedFloat(entry_dist) });
        
        while let Some(curr) = candidates.pop() {
            // Get furthest result distance
            let furthest_dist = results.peek().map(|n| n.distance.0).unwrap_or(f32::MAX);
            
            // Stop if current candidate is further than furthest result
            if curr.distance.0 > furthest_dist {
                break;
            }
            
            // Explore neighbors
            for &neighbor_id in &nodes[curr.id].neighbors[level] {
                if visited.contains(&neighbor_id) {
                    continue;
                }
                visited.insert(neighbor_id);
                
                let neighbor_dist = self.distance.calculate(query, nodes[neighbor_id].vector());
                let furthest_dist = results.peek().map(|n| n.distance.0).unwrap_or(f32::MAX);
                
                if neighbor_dist < furthest_dist || results.len() < ef {
                    candidates.push(Neighbor { id: neighbor_id, distance: OrderedFloat(neighbor_dist) });
                    results.push(MaxNeighbor { id: neighbor_id, distance: OrderedFloat(neighbor_dist) });
                    
                    if results.len() > ef {
                        results.pop();
                    }
                }
            }
        }
        
        // Convert results to Neighbor vec
        results.into_iter()
            .map(|n| Neighbor { id: n.id, distance: n.distance })
            .collect()
    }
    
    /// Select best neighbors using RNG (Relative Neighborhood Graph) heuristic
    /// 
    /// This heuristic improves recall by ensuring diversity among selected neighbors.
    /// A candidate is only selected if it's not "closer through" any already-selected neighbor.
    /// 
    /// Algorithm:
    /// 1. Sort candidates by distance (ascending)
    /// 2. For each candidate, check if it's closer to any selected neighbor than to the query
    /// 3. If so, skip it (it can be reached through that neighbor)
    /// 4. Otherwise, add it to the selected set
    /// 
    /// Optimized: uses node.vector() reference instead of cloning
    pub(crate) fn select_neighbors(&self, _query: &[f32], candidates: &[Neighbor], max_count: usize) -> Vec<Neighbor> {
        if candidates.is_empty() || max_count == 0 {
            return Vec::new();
        }
        
        // Sort candidates by distance (closest first)
        let mut sorted: Vec<_> = candidates.to_vec();
        sorted.sort_by(|a, b| a.distance.cmp(&b.distance));
        
        // If we have fewer candidates than max_count, return all
        if sorted.len() <= max_count {
            return sorted;
        }
        
        // Check if RNG heuristic is enabled
        if !self.use_rng_heuristic.load(AtomicOrdering::Relaxed) {
            // Simple selection: just take closest
            sorted.truncate(max_count);
            return sorted;
        }
        
        let nodes = self.nodes.read();
        let mut selected: Vec<Neighbor> = Vec::with_capacity(max_count);
        
        for candidate in sorted {
            if selected.len() >= max_count {
                break;
            }
            
            // Check if this candidate should be pruned by RNG heuristic
            let mut should_add = true;
            
            for existing in &selected {
                // Calculate distance between candidate and existing selected neighbor
                // Using vector() reference - no cloning!
                let dist_candidate_to_existing = self.distance.calculate(
                    nodes[candidate.id].vector(),
                    nodes[existing.id].vector()
                );
                
                // RNG heuristic: if candidate is closer to an existing neighbor
                // than to the query, it can be "reached through" that neighbor
                // Use a relaxation factor (alpha) to control pruning aggressiveness
                // alpha = 1.0 is standard RNG, alpha < 1.0 is less aggressive (keeps more neighbors)
                // We use alpha = 0.8 for better recall while still maintaining diversity
                let alpha = 0.8f32;
                if dist_candidate_to_existing < candidate.distance.0 * alpha {
                    should_add = false;
                    break;
                }
            }
            
            if should_add {
                selected.push(candidate);
            }
        }
        
        // If RNG heuristic pruned too aggressively, fill with closest remaining candidates
        if selected.len() < max_count {
            let selected_ids: std::collections::HashSet<_> = selected.iter().map(|n| n.id).collect();
            let mut sorted_candidates: Vec<_> = candidates.to_vec();
            sorted_candidates.sort_by(|a, b| a.distance.cmp(&b.distance));
            
            for candidate in sorted_candidates {
                if selected.len() >= max_count {
                    break;
                }
                if !selected_ids.contains(&candidate.id) {
                    selected.push(candidate);
                }
            }
        }
        
        selected
    }
    
    /// Select neighbors with configurable heuristic
    /// 
    /// - `extend_candidates`: If true, extend candidate set with neighbors of candidates
    /// - `keep_pruned`: If true, add back pruned candidates to fill up to max_count
    /// 
    /// Optimized: uses node.vector() reference instead of cloning
    #[allow(dead_code)]
    pub(crate) fn select_neighbors_heuristic(
        &self, 
        query: &[f32], 
        candidates: &[Neighbor], 
        max_count: usize,
        extend_candidates: bool,
        keep_pruned: bool,
    ) -> Vec<Neighbor> {
        if candidates.is_empty() || max_count == 0 {
            return Vec::new();
        }
        
        let nodes = self.nodes.read();
        
        // Optionally extend candidate set with neighbors of candidates (level 0)
        let mut working_set: Vec<Neighbor> = candidates.to_vec();
        
        if extend_candidates {
            let mut extended_ids: std::collections::HashSet<usize> = 
                candidates.iter().map(|n| n.id).collect();
            
            for candidate in candidates {
                if candidate.id < nodes.len() && !nodes[candidate.id].neighbors.is_empty() {
                    for &neighbor_id in &nodes[candidate.id].neighbors[0] {
                        if !extended_ids.contains(&neighbor_id) {
                            extended_ids.insert(neighbor_id);
                            let dist = self.distance.calculate(query, nodes[neighbor_id].vector());
                            working_set.push(Neighbor { 
                                id: neighbor_id, 
                                distance: OrderedFloat(dist) 
                            });
                        }
                    }
                }
            }
        }
        
        // Sort by distance
        working_set.sort_by(|a, b| a.distance.cmp(&b.distance));
        
        let mut selected: Vec<Neighbor> = Vec::with_capacity(max_count);
        let mut pruned: Vec<Neighbor> = Vec::new();
        
        for candidate in working_set {
            if selected.len() >= max_count {
                break;
            }
            
            // Check RNG condition
            let mut is_good = true;
            
            for existing in &selected {
                let dist_to_existing = self.distance.calculate(
                    nodes[candidate.id].vector(),
                    nodes[existing.id].vector()
                );
                
                if dist_to_existing < candidate.distance.0 {
                    is_good = false;
                    break;
                }
            }
            
            if is_good {
                selected.push(candidate);
            } else if keep_pruned {
                pruned.push(candidate);
            }
        }
        
        // Fill with pruned candidates if needed
        if keep_pruned && selected.len() < max_count {
            for candidate in pruned {
                if selected.len() >= max_count {
                    break;
                }
                selected.push(candidate);
            }
        }
        
        selected
    }
    
    /// Prune neighbor list to max_count (standalone version for borrow checker)
    /// 
    /// Optimized: uses node.vector() reference instead of cloning
    #[allow(dead_code)]
    pub(crate) fn prune_neighbors_standalone(&self, node_vec: &[f32], neighbors: &[usize], max_count: usize, nodes: &[HNSWNode]) -> Vec<usize> {
        if neighbors.len() <= max_count {
            return neighbors.to_vec();
        }
        
        let mut with_dist: Vec<_> = neighbors.iter()
            .map(|&id| {
                let dist = self.distance.calculate(node_vec, nodes[id].vector());
                (id, OrderedFloat(dist))
            })
            .collect();
        
        with_dist.sort_by(|a, b| a.1.cmp(&b.1));
        with_dist.truncate(max_count);
        with_dist.into_iter().map(|(id, _)| id).collect()
    }
    
    /// Search for k nearest neighbors
    pub fn search(&self, query: &[f32], k: usize) -> Result<Vec<SearchResult>, HNSWError> {
        if query.len() != self.config.dimension {
            return Err(HNSWError::DimensionMismatch {
                expected: self.config.dimension,
                got: query.len(),
            });
        }
        
        let entry = self.entry_point.read();
        let entry_id = match *entry {
            Some(id) => id,
            None => return Ok(Vec::new()),
        };
        drop(entry);
        
        let max_level = self.max_level.load(AtomicOrdering::SeqCst);
        
        // Start from entry point
        let mut curr_id = entry_id;
        
        // Traverse from top level to level 1
        for level in (1..=max_level).rev() {
            curr_id = self.greedy_search_level(query, curr_id, level);
        }
        
        // Search at level 0 with ef_search candidates
        // Request more candidates to account for deleted nodes
        let deleted = self.deleted.read();
        let deleted_count = deleted.len();
        let ef_adjusted = (self.config.ef_search.max(k) + deleted_count).min(self.nodes.read().len());
        drop(deleted);
        
        let neighbors = self.search_level(query, curr_id, ef_adjusted, 0);
        
        // Filter out deleted nodes and return top k results
        let deleted = self.deleted.read();
        let mut results: Vec<_> = neighbors.into_iter()
            .filter(|n| !deleted.contains(&n.id))
            .map(|n| SearchResult { id: n.id, distance: n.distance.0 })
            .collect();
        
        results.sort_by(|a, b| a.distance.partial_cmp(&b.distance).unwrap());
        results.truncate(k);
        
        Ok(results)
    }
    
    /// Search with external IDs in results
    pub fn search_with_ids(&self, query: &[f32], k: usize) -> Result<Vec<(String, f32)>, HNSWError> {
        let results = self.search(query, k)?;
        let reverse_map = self.reverse_id_map.read();
        
        Ok(results.into_iter()
            .map(|r| (reverse_map[r.id].clone(), r.distance))
            .collect())
    }
    
    /// Get vector by external ID
    /// 
    /// Returns a clone of the vector. For zero-copy access, use `get_arc`.
    pub fn get(&self, external_id: &str) -> Option<Vec<f32>> {
        let id_map = self.id_map.read();
        let internal_id = id_map.get(external_id)?;
        
        let nodes = self.nodes.read();
        Some((*nodes[*internal_id].vector).clone())
    }
    
    /// Get vector by external ID as Arc (zero-copy)
    /// 
    /// Returns an Arc reference to the vector, avoiding cloning.
    pub fn get_arc(&self, external_id: &str) -> Option<Arc<Vec<f32>>> {
        let id_map = self.id_map.read();
        let internal_id = id_map.get(external_id)?;
        
        let nodes = self.nodes.read();
        Some(nodes[*internal_id].vector_arc())
    }
    
    /// Check if an external ID exists
    pub fn contains(&self, external_id: &str) -> bool {
        self.id_map.read().contains_key(external_id)
    }
    
    /// Get all external IDs
    pub fn ids(&self) -> Vec<String> {
        self.reverse_id_map.read().clone()
    }
    
    // ============================================================================
    // Soft Delete Methods (P0-2)
    // ============================================================================
    
    /// Soft delete a node by external ID
    /// 
    /// The node is marked as deleted but not removed from the index.
    /// Deleted nodes are filtered out during search operations.
    pub fn delete(&self, external_id: &str) -> Result<(), HNSWError> {
        let internal_id = {
            let id_map = self.id_map.read();
            *id_map.get(external_id)
                .ok_or_else(|| HNSWError::IdNotFound(external_id.to_string()))?
        };
        
        self.deleted.write().insert(internal_id);
        
        // P0-3: Record transaction operation
        self.record_operation(Operation::Delete {
            external_id: external_id.to_string(),
            internal_id,
        });
        
        Ok(())
    }
    
    /// Check if an internal node ID is deleted
    pub fn is_deleted(&self, internal_id: usize) -> bool {
        self.deleted.read().contains(&internal_id)
    }
    
    /// Check if an external ID is deleted
    pub fn is_deleted_external(&self, external_id: &str) -> bool {
        let id_map = self.id_map.read();
        if let Some(&internal_id) = id_map.get(external_id) {
            self.deleted.read().contains(&internal_id)
        } else {
            false
        }
    }
    
    /// Get the ratio of deleted nodes
    pub fn deleted_ratio(&self) -> f64 {
        let total = self.nodes.read().len();
        if total == 0 {
            return 0.0;
        }
        let deleted_count = self.deleted.read().len();
        deleted_count as f64 / total as f64
    }
    
    /// Get the number of deleted nodes
    pub fn deleted_count(&self) -> usize {
        self.deleted.read().len()
    }
    
    /// Get the number of active (non-deleted) nodes
    pub fn active_count(&self) -> usize {
        self.nodes.read().len() - self.deleted.read().len()
    }
    
    /// Check if the index needs compaction
    /// 
    /// Returns true if the deleted ratio exceeds the threshold (default 0.3)
    pub fn needs_compact(&self, threshold: f64) -> bool {
        self.deleted_ratio() > threshold
    }
    
    /// Compact the index by rebuilding without deleted nodes
    /// 
    /// This creates a new index with only the active nodes.
    /// The old index remains unchanged - caller should replace it.
    pub fn compact(&self) -> Result<HNSWIndex, HNSWError> {
        let deleted = self.deleted.read();
        let nodes = self.nodes.read();
        let id_map = self.id_map.read();
        
        // Create new index with same config
        let new_index = HNSWIndex::new(self.config.clone());
        
        // Collect active nodes with their external IDs
        // Use Arc to avoid unnecessary cloning
        let mut active_items: Vec<(String, Arc<Vec<f32>>)> = Vec::new();
        
        for (external_id, &internal_id) in id_map.iter() {
            if !deleted.contains(&internal_id) {
                let vector_arc = nodes[internal_id].vector_arc();
                active_items.push((external_id.clone(), vector_arc));
            }
        }
        
        // Drop locks before inserting
        drop(deleted);
        drop(nodes);
        drop(id_map);
        
        // Insert all active nodes into new index
        for (external_id, vector_arc) in active_items {
            // Clone only when inserting (necessary for new index)
            new_index.insert(external_id, (*vector_arc).clone())?;
        }
        
        Ok(new_index)
    }
    
    /// Restore a soft-deleted node
    pub fn restore(&self, external_id: &str) -> Result<(), HNSWError> {
        let internal_id = {
            let id_map = self.id_map.read();
            *id_map.get(external_id)
                .ok_or_else(|| HNSWError::IdNotFound(external_id.to_string()))?
        };
        
        self.deleted.write().remove(&internal_id);
        
        // P0-3: Record transaction operation
        self.record_operation(Operation::Restore {
            external_id: external_id.to_string(),
            internal_id,
        });
        
        Ok(())
    }
    
    /// Record an operation in the current transaction
    pub(crate) fn record_operation(&self, op: Operation) {
        if let Some(ref mut tx) = *self.active_transaction.write() {
            if tx.is_active() {
                tx.operations.push(op);
            }
        }
    }
}
