//! Lance columnar storage backend implementation
//!
//! Implements `StorageBackend` using Lance datasets for persistent storage.
//! Provides 100x faster random access than JSONL, built-in versioning,
//! column pruning, and Arrow ecosystem interoperability.
//!
//! # Object Store Support (Phase-55)
//!
//! Supports remote object stores (S3, GCS, Azure) via the `object_store` crate
//! integrated through Lance. Use URI-based paths (e.g., `s3://bucket/path`)
//! and `ObjectStoreConfig` for credentials and connection options.

use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;

use arrow_array::{RecordBatch, RecordBatchIterator};
use arrow_schema::Schema;
use lance::Dataset;
use lance::dataset::builder::DatasetBuilder;
use lance::dataset::{
    MergeInsertBuilder, MergeStats, WhenMatched, WhenNotMatched, WhenNotMatchedBySource,
    DeleteBuilder,
};
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use tokio::runtime::Runtime;

use crate::types::{MemoryNode, validate_id};
use crate::storage::backend::{StorageBackend, StorageError, StorageResult, QueryFilter};
use super::arrow_convert::{
    memory_node_arrow_schema, memory_node_to_record_batch, record_batch_to_memory_nodes,
};
use super::object_store_config::{ObjectStoreConfig, is_remote_uri};

#[cfg(feature = "full-text-search")]
use crate::search::FullTextIndex;

/// Optimistic Concurrency Control (OCC) configuration.
///
/// Controls how Lance merge_insert handles concurrent write conflicts.
/// Lance uses optimistic concurrency: multiple writers can proceed in parallel,
/// but if they conflict at commit time, the operation is retried with the latest
/// dataset version.
///
/// # Defaults
///
/// - `max_retries`: 10 (same as Lance default)
/// - `retry_timeout_ms`: 30000 (30 seconds)
///
/// # Example
///
/// ```rust
/// use aifastdb_core::storage::lance::lance_backend::OCCConfig;
///
/// let occ = OCCConfig::default()
///     .with_max_retries(20)
///     .with_retry_timeout_ms(60_000);
/// ```
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OCCConfig {
    /// Maximum number of retry attempts on commit conflict (default: 10)
    pub max_retries: u32,

    /// Maximum total time (ms) to spend on retries before giving up (default: 30000)
    pub retry_timeout_ms: u64,
}

impl Default for OCCConfig {
    fn default() -> Self {
        Self {
            max_retries: 10,
            retry_timeout_ms: 30_000,
        }
    }
}

impl OCCConfig {
    /// Set the maximum number of retry attempts.
    pub fn with_max_retries(mut self, n: u32) -> Self {
        self.max_retries = n;
        self
    }

    /// Set the retry timeout in milliseconds.
    pub fn with_retry_timeout_ms(mut self, ms: u64) -> Self {
        self.retry_timeout_ms = ms;
        self
    }

    /// Convert to a `Duration` for the retry timeout.
    pub fn retry_timeout(&self) -> Duration {
        Duration::from_millis(self.retry_timeout_ms)
    }
}

/// Runtime OCC statistics (atomic counters, zero-cost reads).
///
/// Tracks merge_insert operations, conflict retries, and error rates.
/// All fields are atomic for lock-free concurrent access.
#[derive(Debug, Default)]
pub struct OCCMetrics {
    /// Total merge_insert operations attempted
    pub merge_ops: AtomicU64,
    /// Total rows inserted via merge_insert
    pub rows_inserted: AtomicU64,
    /// Total rows updated via merge_insert
    pub rows_updated: AtomicU64,
    /// Total retries across all operations (num_attempts - merge_ops)
    pub total_retries: AtomicU64,
    /// Operations that failed with VersionConflict (all retries exhausted)
    pub conflict_failures: AtomicU64,
    /// Legacy append writes (when no dataset exists yet)
    pub legacy_appends: AtomicU64,
}

impl OCCMetrics {
    /// Record a successful merge_insert operation.
    fn record_merge(&self, stats: &MergeStats) {
        self.merge_ops.fetch_add(1, Ordering::Relaxed);
        self.rows_inserted.fetch_add(stats.num_inserted_rows, Ordering::Relaxed);
        self.rows_updated.fetch_add(stats.num_updated_rows, Ordering::Relaxed);
        // num_attempts includes the initial attempt, so retries = attempts - 1
        if stats.num_attempts > 1 {
            self.total_retries.fetch_add((stats.num_attempts - 1) as u64, Ordering::Relaxed);
        }
    }

    /// Record a conflict failure (all retries exhausted).
    fn record_conflict_failure(&self) {
        self.conflict_failures.fetch_add(1, Ordering::Relaxed);
    }

    /// Record a legacy append (no existing dataset).
    fn record_legacy_append(&self) {
        self.legacy_appends.fetch_add(1, Ordering::Relaxed);
    }

    /// Get a snapshot of all metrics.
    pub fn snapshot(&self) -> OCCMetricsSnapshot {
        OCCMetricsSnapshot {
            merge_ops: self.merge_ops.load(Ordering::Relaxed),
            rows_inserted: self.rows_inserted.load(Ordering::Relaxed),
            rows_updated: self.rows_updated.load(Ordering::Relaxed),
            total_retries: self.total_retries.load(Ordering::Relaxed),
            conflict_failures: self.conflict_failures.load(Ordering::Relaxed),
            legacy_appends: self.legacy_appends.load(Ordering::Relaxed),
        }
    }
}

/// Serializable snapshot of OCC metrics.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OCCMetricsSnapshot {
    pub merge_ops: u64,
    pub rows_inserted: u64,
    pub rows_updated: u64,
    pub total_retries: u64,
    pub conflict_failures: u64,
    pub legacy_appends: u64,
}

impl OCCMetricsSnapshot {
    /// Retry success rate: 1.0 means all retries eventually succeeded.
    pub fn retry_success_rate(&self) -> f64 {
        let total_ops = self.merge_ops + self.conflict_failures;
        if total_ops == 0 {
            1.0
        } else {
            self.merge_ops as f64 / total_ops as f64
        }
    }
}

/// Lance storage backend configuration.
///
/// Supports both local filesystem paths and remote object store URIs.
///
/// # Local Storage
///
/// ```no_run
/// use aifastdb_core::storage::lance::lance_backend::LanceConfig;
///
/// let config = LanceConfig::new("./data/memories.lance");
/// ```
///
/// # Remote Storage (S3)
///
/// ```no_run
/// use aifastdb_core::storage::lance::lance_backend::LanceConfig;
/// use aifastdb_core::storage::lance::ObjectStoreConfig;
///
/// let config = LanceConfig::from_uri("s3://my-bucket/aifastdb/memories.lance")
///     .with_object_store_config(
///         ObjectStoreConfig::s3()
///             .with_region("us-east-1")
///             .with_access_key("AKID", "SECRET")
///     );
/// ```
#[derive(Debug, Clone)]
pub struct LanceConfig {
    /// Path to the Lance dataset directory (e.g., "./data/memories.lance")
    ///
    /// For local storage, this is used as-is.
    /// For remote storage, prefer using `uri` instead.
    pub path: PathBuf,

    /// URI for the Lance dataset (supports s3://, gs://, az:// schemes).
    ///
    /// When set, this takes precedence over `path` for opening/writing datasets.
    /// For local storage, this can be `None` (falls back to `path`).
    pub uri: Option<String>,

    /// Embedding dimension (auto-detected if None)
    pub embedding_dim: Option<i32>,

    /// Write batch size — accumulate this many puts before flushing to Lance
    pub write_batch_size: usize,

    /// Object store configuration for remote storage (S3/GCS/Azure).
    ///
    /// When `None`, uses environment variables for credentials (if needed).
    pub object_store_config: Option<ObjectStoreConfig>,

    /// Optimistic Concurrency Control configuration for merge_insert.
    ///
    /// Controls conflict retry behavior when multiple writers compete.
    pub occ: OCCConfig,
}

impl Default for LanceConfig {
    fn default() -> Self {
        Self {
            path: PathBuf::from("./aifastdb_data/memories.lance"),
            uri: None,
            embedding_dim: None,
            write_batch_size: 256,
            object_store_config: None,
            occ: OCCConfig::default(),
        }
    }
}

impl LanceConfig {
    /// Create a new config with the given local path.
    pub fn new(path: impl Into<PathBuf>) -> Self {
        Self {
            path: path.into(),
            ..Default::default()
        }
    }

    /// Create a new config from a URI string (supports s3://, gs://, az://, or local paths).
    ///
    /// For remote URIs, you'll typically chain `.with_object_store_config()` to set credentials.
    pub fn from_uri(uri: impl Into<String>) -> Self {
        let uri_str: String = uri.into();
        Self {
            // For local URIs, also set path for backward compatibility
            path: if is_remote_uri(&uri_str) {
                PathBuf::from(".")
            } else {
                PathBuf::from(&uri_str)
            },
            uri: Some(uri_str),
            ..Default::default()
        }
    }

    /// Set the embedding dimension.
    pub fn with_embedding_dim(mut self, dim: i32) -> Self {
        self.embedding_dim = Some(dim);
        self
    }

    /// Set the write batch size.
    pub fn with_write_batch_size(mut self, size: usize) -> Self {
        self.write_batch_size = size;
        self
    }

    /// Set the object store configuration for remote storage.
    pub fn with_object_store_config(mut self, config: ObjectStoreConfig) -> Self {
        self.object_store_config = Some(config);
        self
    }

    /// Set the OCC (Optimistic Concurrency Control) configuration.
    pub fn with_occ(mut self, occ: OCCConfig) -> Self {
        self.occ = occ;
        self
    }

    /// Get the effective URI for this configuration.
    ///
    /// Returns `uri` if set, otherwise converts `path` to a string.
    pub fn effective_uri(&self) -> String {
        self.uri.clone().unwrap_or_else(|| self.path.to_string_lossy().to_string())
    }

    /// Whether this config targets a remote object store.
    pub fn is_remote(&self) -> bool {
        if let Some(ref uri) = self.uri {
            is_remote_uri(uri)
        } else {
            false
        }
    }

    /// Build Lance `WriteParams` with object store configuration.
    pub(crate) fn write_params(&self, mode: lance::dataset::WriteMode) -> lance::dataset::WriteParams {
        let mut params = lance::dataset::WriteParams {
            mode,
            ..Default::default()
        };

        if let Some(ref os_config) = self.object_store_config {
            if os_config.is_remote() {
                params.store_params = Some(os_config.to_object_store_params());
            }
        }

        params
    }
}

/// Lance-backed storage implementation.
///
/// Uses an in-memory write buffer that is periodically flushed to the
/// Lance dataset. Reads first check the buffer, then fall back to the
/// dataset on disk.
///
/// # Thread Safety
///
/// All mutable state is behind `RwLock`, and the Lance async operations
/// are executed on a dedicated tokio `Runtime`.
///
/// # Performance Optimizations
///
/// - **Inverted tag index**: Maintains a `tag → Set<node_id>` mapping for
///   O(1) tag lookups instead of full-cache scans.
/// - **Cache-direct query()**: Overrides the default `query()` to filter
///   directly on the cache HashMap without cloning all nodes first.
/// - **Batch get**: Single lock acquisition for multiple ID lookups.
/// - **Pending deletes**: Accumulates delete operations and flushes them
///   in batch to avoid re-opening the dataset on each delete.
pub struct LanceStorageBackend {
    /// Configuration
    config: LanceConfig,

    /// Arrow schema (computed once on creation)
    schema: Arc<Schema>,

    /// Lance dataset handle (None if dataset doesn't exist yet).
    /// Wrapped in Arc for compatibility with Lance's MergeInsertBuilder/DeleteBuilder APIs.
    dataset: RwLock<Option<Arc<Dataset>>>,

    /// In-memory write buffer (pending writes not yet flushed)
    write_buffer: RwLock<Vec<MemoryNode>>,

    /// In-memory cache for fast reads (id → node)
    /// Populated on open; kept in sync with writes/deletes.
    cache: RwLock<std::collections::HashMap<String, MemoryNode>>,

    /// Inverted tag index for O(1) tag-based lookups.
    /// Maps tag → set of node IDs that carry this tag.
    tag_index: RwLock<std::collections::HashMap<String, std::collections::HashSet<String>>>,

    /// Pending delete IDs (flushed to Lance dataset in batch).
    pending_deletes: RwLock<Vec<String>>,

    /// Tokio runtime for async Lance operations
    runtime: Runtime,

    /// OCC metrics (conflict retries, merge stats, etc.)
    occ_metrics: Arc<OCCMetrics>,

    /// Optional Tantivy full-text search index (enabled with `full-text-search` feature)
    #[cfg(feature = "full-text-search")]
    fts_index: Option<Arc<crate::search::TantivyFullTextIndex>>,
}

impl LanceStorageBackend {
    /// Open or create a Lance storage backend.
    ///
    /// If the dataset already exists at `config.path` (local) or `config.uri` (remote),
    /// it is opened and all records are loaded into the in-memory cache.
    /// Otherwise, the dataset will be created on the first `flush`.
    ///
    /// # Remote Storage
    ///
    /// For remote URIs (s3://, gs://, az://), the backend uses Lance's
    /// `DatasetBuilder` with storage options from `ObjectStoreConfig`.
    /// All data is cached in-memory after initial load.
    pub fn open(config: LanceConfig) -> StorageResult<Self> {
        let runtime = Runtime::new()
            .map_err(|e| StorageError::Backend(format!("Failed to create tokio runtime: {}", e)))?;

        let dim = config.embedding_dim.unwrap_or(1536);
        let schema = Arc::new(memory_node_arrow_schema(Some(dim)));

        let uri = config.effective_uri();
        let is_remote = config.is_remote();

        // Try to open existing dataset (local path check or remote attempt)
        let dataset_exists = if is_remote {
            // For remote URIs, we try to open and handle errors gracefully
            true // Will attempt open; if it fails, treat as non-existent
        } else {
            config.path.exists()
        };

        let (dataset, cache) = if dataset_exists {
            // Build dataset opener with optional storage options
            let open_result = runtime.block_on(async {
                let mut builder = DatasetBuilder::from_uri(&uri);

                // Apply storage options for remote backends
                if let Some(ref os_config) = config.object_store_config {
                    if !os_config.to_storage_options().is_empty() {
                        builder = builder.with_storage_options(os_config.to_storage_options());
                    }
                }

                builder.load().await
            });

            match open_result {
                Ok(ds) => {
                    // Load all records into cache
                    let cache = runtime.block_on(async {
                        load_all_from_dataset(&ds).await
                    })?;
                    (Some(Arc::new(ds)), cache)
                }
                Err(e) => {
                    if is_remote {
                        // Remote dataset doesn't exist yet — will be created on first flush
                        tracing::info!(
                            "Remote dataset at '{}' not found ({}), will create on first write",
                            uri, e
                        );
                        (None, std::collections::HashMap::new())
                    } else {
                        return Err(StorageError::Backend(format!(
                            "Failed to open Lance dataset at '{}': {}", uri, e
                        )));
                    }
                }
            }
        } else {
            (None, std::collections::HashMap::new())
        };

        // Build the inverted tag index from the loaded cache
        let tag_index = Self::build_tag_index(&cache);

        // Initialize Tantivy FTS index if feature enabled
        #[cfg(feature = "full-text-search")]
        let fts_index = {
            let fts_path = config.path.parent()
                .unwrap_or(std::path::Path::new("."))
                .join("tantivy_fts");
            let fts_config = crate::search::FTSConfig::new(fts_path.to_string_lossy().to_string());
            match crate::search::TantivyFullTextIndex::open(fts_config) {
                Ok(index) => {
                    // Build initial FTS index from existing cache
                    if !cache.is_empty() {
                        let docs: Vec<crate::search::FTSDocument> = cache.values()
                            .map(Self::node_to_fts_doc)
                            .collect();
                        let _ = index.add_documents(&docs);
                    }
                    Some(Arc::new(index))
                }
                Err(e) => {
                    eprintln!("[aifastdb] Warning: Failed to initialize Tantivy FTS index: {}", e);
                    None
                }
            }
        };

        Ok(Self {
            config,
            schema,
            dataset: RwLock::new(dataset),
            write_buffer: RwLock::new(Vec::new()),
            cache: RwLock::new(cache),
            tag_index: RwLock::new(tag_index),
            pending_deletes: RwLock::new(Vec::new()),
            runtime,
            occ_metrics: Arc::new(OCCMetrics::default()),
            #[cfg(feature = "full-text-search")]
            fts_index,
        })
    }

    /// Enable full-text search on this backend with a custom config.
    #[cfg(feature = "full-text-search")]
    pub fn enable_fts(&mut self, config: crate::search::FTSConfig) -> StorageResult<()> {
        let index = crate::search::TantivyFullTextIndex::open(config)
            .map_err(|e| StorageError::Backend(format!("Failed to create FTS index: {}", e)))?;

        // Build index from current cache
        let nodes: Vec<MemoryNode> = self.cache.read().values().cloned().collect();
        if !nodes.is_empty() {
            let docs: Vec<crate::search::FTSDocument> = nodes.iter().map(Self::node_to_fts_doc).collect();
            index.add_documents(&docs)
                .map_err(|e| StorageError::Backend(format!("Failed to build FTS index: {}", e)))?;
        }

        self.fts_index = Some(Arc::new(index));
        Ok(())
    }

    /// Get a reference to the FTS index if enabled.
    #[cfg(feature = "full-text-search")]
    pub fn fts_index(&self) -> Option<&Arc<crate::search::TantivyFullTextIndex>> {
        self.fts_index.as_ref()
    }

    /// Convert a MemoryNode to an FTSDocument for indexing.
    #[cfg(feature = "full-text-search")]
    fn node_to_fts_doc(node: &MemoryNode) -> crate::search::FTSDocument {
        let mut metadata: std::collections::HashMap<String, String> = std::collections::HashMap::new();
        for (k, v) in &node.metadata {
            metadata.insert(k.clone(), v.to_string());
        }
        crate::search::FTSDocument {
            id: node.id.clone(),
            content: node.content.clone(),
            tags: node.tags.clone(),
            metadata,
        }
    }

    /// Sync a node to the FTS index (called after put/update).
    #[cfg(feature = "full-text-search")]
    fn fts_sync_put(&self, node: &MemoryNode) {
        if let Some(ref idx) = self.fts_index {
            let doc = Self::node_to_fts_doc(node);
            let _ = idx.update_document(&doc);
            let _ = idx.commit();
        }
    }

    /// Remove nodes from the FTS index (called after delete).
    #[cfg(feature = "full-text-search")]
    fn fts_sync_delete(&self, ids: &[String]) {
        if let Some(ref idx) = self.fts_index {
            for id in ids {
                let _ = idx.delete_document(id);
            }
            let _ = idx.commit();
        }
    }

    /// Create scalar indexes on the underlying Lance dataset.
    ///
    /// This creates BTree indexes on high-cardinality columns (id, created_at,
    /// importance, etc.) and Bitmap indexes on low-cardinality columns (content_type).
    /// Indexes accelerate Lance-level predicate push-down filtering.
    ///
    /// This is idempotent — existing indexes are skipped.
    /// Returns the number of newly created indexes.
    pub fn ensure_scalar_indexes(&self) -> StorageResult<usize> {
        let mut ds_lock = self.dataset.write();
        let ds_arc = match ds_lock.as_mut() {
            Some(ds) => ds,
            None => return Ok(0), // No dataset yet — indexes will be created on first flush
        };

        // Arc::get_mut is safe here because we hold the exclusive write lock,
        // and no other code retains a clone of this Arc.
        let ds = Arc::get_mut(ds_arc).ok_or_else(|| {
            StorageError::Backend("Cannot get mutable access to dataset (Arc still shared)".into())
        })?;

        let indexes = super::scalar_index::default_scalar_indexes();
        let created = self.runtime.block_on(
            super::scalar_index::ensure_scalar_indexes(ds, &indexes)
        ).map_err(|e| StorageError::Backend(format!("Failed to create scalar indexes: {}", e)))?;

        Ok(created)
    }

    /// Build the inverted tag index from a node cache.
    fn build_tag_index(
        cache: &std::collections::HashMap<String, MemoryNode>,
    ) -> std::collections::HashMap<String, std::collections::HashSet<String>> {
        let mut index: std::collections::HashMap<String, std::collections::HashSet<String>> =
            std::collections::HashMap::new();
        for (id, node) in cache {
            for tag in &node.tags {
                index
                    .entry(tag.clone())
                    .or_default()
                    .insert(id.clone());
            }
        }
        index
    }

    /// Add a node's tags to the inverted tag index.
    fn index_node_tags(&self, id: &str, tags: &[String]) {
        let mut tag_idx = self.tag_index.write();
        for tag in tags {
            tag_idx
                .entry(tag.clone())
                .or_default()
                .insert(id.to_string());
        }
    }

    /// Remove a node's tags from the inverted tag index.
    fn unindex_node_tags(&self, id: &str, tags: &[String]) {
        let mut tag_idx = self.tag_index.write();
        for tag in tags {
            if let Some(set) = tag_idx.get_mut(tag) {
                set.remove(id);
                if set.is_empty() {
                    tag_idx.remove(tag);
                }
            }
        }
    }

    /// Async helper: open a Lance dataset with storage options from config.
    ///
    /// Used internally for re-opening datasets after writes/deletes,
    /// supporting both local paths and remote URIs (S3/GCS/Azure).
    #[allow(dead_code)]
    async fn open_dataset_async(&self, uri: &str) -> Result<Dataset, StorageError> {
        let mut builder = DatasetBuilder::from_uri(uri);

        // Apply storage options for remote backends
        if let Some(ref os_config) = self.config.object_store_config {
            if !os_config.to_storage_options().is_empty() {
                builder = builder.with_storage_options(os_config.to_storage_options());
            }
        }

        builder.load().await
            .map_err(|e| StorageError::Backend(format!("Failed to open Lance dataset at '{}': {}", uri, e)))
    }

    /// Flush pending deletes to the Lance dataset in a single batch.
    ///
    /// Uses `DeleteBuilder` with OCC conflict retries for safe concurrent deletes.
    fn flush_pending_deletes(&self) -> StorageResult<()> {
        let mut pending = self.pending_deletes.write();
        if pending.is_empty() {
            return Ok(());
        }

        let ids: Vec<String> = pending.drain(..).collect();
        drop(pending);

        let mut ds_guard = self.dataset.write();

        if let Some(ref ds_arc) = *ds_guard {
            // Build a single predicate: id IN ('id1', 'id2', ...)
            let predicates: Vec<String> = ids.iter().map(|id| format!("id = '{}'", id)).collect();
            let combined = predicates.join(" OR ");

            let result = self.runtime.block_on(async {
                DeleteBuilder::new(ds_arc.clone(), &combined)
                    .conflict_retries(self.config.occ.max_retries)
                    .retry_timeout(self.config.occ.retry_timeout())
                    .execute()
                    .await
            });

            match result {
                Ok(new_ds) => {
                    *ds_guard = Some(new_ds);
                }
                Err(e) => {
                    tracing::warn!("Failed to flush pending deletes: {}", e);
                }
            }
        }

        Ok(())
    }

    /// Force flush the write buffer to the Lance dataset.
    ///
    /// - **Existing dataset**: Uses `merge_insert` (atomic upsert) with OCC conflict retries.
    ///   This replaces the old delete+add pattern with a single atomic operation that
    ///   handles both inserts and updates via the `id` field as the merge key.
    /// - **New dataset**: Uses `Dataset::write()` to create the initial dataset.
    fn flush_buffer(&self) -> StorageResult<()> {
        let mut buffer = self.write_buffer.write();
        if buffer.is_empty() {
            return Ok(());
        }

        let nodes: Vec<MemoryNode> = buffer.drain(..).collect();
        drop(buffer); // release lock before I/O

        let dim = self
            .config
            .embedding_dim
            .or_else(|| nodes.iter().find_map(|n| n.embedding.as_ref().map(|e| e.len() as i32)))
            .unwrap_or(1536);

        let batch = memory_node_to_record_batch(&nodes, Some(dim))?;

        let uri = self.config.effective_uri();
        let mut ds_guard = self.dataset.write();

        self.runtime.block_on(async {
            if let Some(ref ds_arc) = *ds_guard {
                // ── merge_insert path (atomic upsert with OCC) ──
                let batches: Vec<Result<RecordBatch, arrow_schema::ArrowError>> = vec![Ok(batch)];
                let reader = RecordBatchIterator::new(
                    batches.into_iter(),
                    self.schema.clone(),
                );

                let merge_result = MergeInsertBuilder::try_new(
                    ds_arc.clone(),
                    vec!["id".to_string()],
                )
                .map_err(|e| StorageError::Backend(format!("Failed to build merge_insert: {}", e)));

                let mut builder = match merge_result {
                    Ok(b) => b,
                    Err(e) => return Err(e),
                };

                builder
                    .when_matched(WhenMatched::UpdateAll)
                    .when_not_matched(WhenNotMatched::InsertAll)
                    .when_not_matched_by_source(WhenNotMatchedBySource::Keep)
                    .conflict_retries(self.config.occ.max_retries)
                    .retry_timeout(self.config.occ.retry_timeout());

                let job = builder.try_build()
                    .map_err(|e| StorageError::Backend(format!("Failed to build merge_insert job: {}", e)))?;

                match job.execute_reader(reader).await {
                    Ok((new_ds, stats)) => {
                        self.occ_metrics.record_merge(&stats);
                        *ds_guard = Some(new_ds);
                    }
                    Err(e) => {
                        // Check if this is a contention exhaustion error
                        let err_msg = e.to_string();
                        if err_msg.contains("contention") || err_msg.contains("retry") {
                            self.occ_metrics.record_conflict_failure();
                            return Err(StorageError::VersionConflict(format!(
                                "Lance merge_insert failed after {} retries: {}",
                                self.config.occ.max_retries, e
                            )));
                        }
                        return Err(StorageError::Backend(format!(
                            "Failed to merge_insert into Lance: {}", e
                        )));
                    }
                }
            } else {
                // ── Create path (first write, no existing dataset) ──
                let batches: Vec<Result<RecordBatch, arrow_schema::ArrowError>> = vec![Ok(batch)];
                let reader = RecordBatchIterator::new(
                    batches.into_iter(),
                    self.schema.clone(),
                );

                let write_params = if self.config.is_remote() {
                    Some(self.config.write_params(lance::dataset::WriteMode::Create))
                } else {
                    None
                };
                let new_ds: Dataset = Dataset::write(reader, &*uri, write_params)
                    .await
                    .map_err(|e| StorageError::Backend(format!(
                        "Failed to create Lance dataset: {}", e
                    )))?;
                self.occ_metrics.record_legacy_append();
                *ds_guard = Some(Arc::new(new_ds));
            }

            Ok::<(), StorageError>(())
        })?;

        Ok(())
    }

    /// 语义感知 Compaction（BriLLM SiFu 启发）
    ///
    /// 按 Morton Code（Z-order curve）对所有节点重排，使语义相似的节点
    /// 在 Lance 文件中物理相邻，从而提升范围扫描和语义门控查询的 I/O 效率。
    ///
    /// 相当于大脑皮层中"语义相似的神经元聚集在相邻区域"的布局优化。
    ///
    /// # Returns
    /// `(total_nodes, nodes_with_axes)` — 总节点数和具有语义轴的节点数
    pub fn semantic_compact(&self) -> StorageResult<(usize, usize)> {
        use crate::semantic::projector::morton_code_3d;

        // 1. 先 flush 所有 pending 数据
        self.flush_pending_deletes()?;
        self.flush_buffer()?;

        // 2. 从缓存中取出所有节点并按 Morton Code 排序
        let mut nodes: Vec<MemoryNode> = {
            let cache = self.cache.read();
            cache.values().cloned().collect()
        };

        let total = nodes.len();
        if total == 0 {
            return Ok((0, 0));
        }

        let nodes_with_axes = nodes.iter().filter(|n| n.semantic_axes.is_some()).count();

        // 按 Morton Code 排序：有语义轴的排在前面（按 morton 排序），
        // 无语义轴的排在后面（保持原顺序）
        nodes.sort_by(|a, b| {
            match (&a.semantic_axes, &b.semantic_axes) {
                (Some(ax_a), Some(ax_b)) => {
                    let m_a = morton_code_3d(ax_a.axis_0, ax_a.axis_1, ax_a.axis_2);
                    let m_b = morton_code_3d(ax_b.axis_0, ax_b.axis_1, ax_b.axis_2);
                    m_a.cmp(&m_b)
                }
                (Some(_), None) => std::cmp::Ordering::Less,
                (None, Some(_)) => std::cmp::Ordering::Greater,
                (None, None) => std::cmp::Ordering::Equal,
            }
        });

        // 3. 用 Overwrite 模式重写整个 Lance 数据集
        let dim = self
            .config
            .embedding_dim
            .or_else(|| nodes.iter().find_map(|n| n.embedding.as_ref().map(|e| e.len() as i32)))
            .unwrap_or(1536);

        let batch = memory_node_to_record_batch(&nodes, Some(dim))?;
        let uri = self.config.effective_uri();
        let mut ds_guard = self.dataset.write();

        self.runtime.block_on(async {
            let batches: Vec<Result<RecordBatch, arrow_schema::ArrowError>> = vec![Ok(batch)];
            let reader = RecordBatchIterator::new(batches.into_iter(), self.schema.clone());

            let write_params = self.config.write_params(lance::dataset::WriteMode::Overwrite);
            let new_ds = Dataset::write(reader, &*uri, Some(write_params))
                .await
                .map_err(|e| StorageError::Backend(format!("Semantic compact failed: {}", e)))?;

            *ds_guard = Some(Arc::new(new_ds));
            Ok::<(), StorageError>(())
        })?;

        Ok((total, nodes_with_axes))
    }

    /// Get OCC metrics snapshot.
    pub fn occ_stats(&self) -> OCCMetricsSnapshot {
        self.occ_metrics.snapshot()
    }

    /// Get a reference to the OCC metrics (for shared access).
    pub fn occ_metrics(&self) -> &Arc<OCCMetrics> {
        &self.occ_metrics
    }
}

impl StorageBackend for LanceStorageBackend {
    fn put(&self, node: MemoryNode) -> StorageResult<()> {
        validate_id(&node.id).map_err(|e| StorageError::InvalidId(e.to_string()))?;

        let id = node.id.clone();
        let new_tags = node.tags.clone();

        // Remove old tags from index if this is an update
        {
            let cache = self.cache.read();
            if let Some(old_node) = cache.get(&id) {
                if old_node.tags != new_tags {
                    drop(cache);
                    // Collect old tags to unindex, then index new ones
                    let old_tags = self.cache.read().get(&id).map(|n| n.tags.clone());
                    if let Some(old_tags) = old_tags {
                        self.unindex_node_tags(&id, &old_tags);
                    }
                }
            }
        }

        // Index new tags
        self.index_node_tags(&id, &new_tags);

        // Update cache + write buffer with a single lock scope
        {
            let mut cache = self.cache.write();
            cache.insert(id, node.clone());
        }

        // Sync to FTS index
        #[cfg(feature = "full-text-search")]
        self.fts_sync_put(&node);

        let mut buffer = self.write_buffer.write();
        buffer.push(node);

        // Auto-flush if buffer is full
        if buffer.len() >= self.config.write_batch_size {
            drop(buffer);
            self.flush_buffer()?;
        }

        Ok(())
    }

    fn get(&self, id: &str) -> StorageResult<Option<MemoryNode>> {
        // Check cache first (always up-to-date)
        Ok(self.cache.read().get(id).cloned())
    }

    fn delete(&self, id: &str) -> StorageResult<Option<MemoryNode>> {
        let removed = self.cache.write().remove(id);

        if let Some(ref node) = removed {
            // Remove tags from inverted index
            self.unindex_node_tags(id, &node.tags);

            // Queue for batch deletion from Lance dataset
            self.pending_deletes.write().push(id.to_string());

            // Sync FTS index
            #[cfg(feature = "full-text-search")]
            self.fts_sync_delete(&[id.to_string()]);
        }

        Ok(removed)
    }

    fn contains(&self, id: &str) -> StorageResult<bool> {
        Ok(self.cache.read().contains_key(id))
    }

    fn put_batch(&self, nodes: Vec<MemoryNode>) -> StorageResult<()> {
        // Validate all IDs first
        for node in &nodes {
            validate_id(&node.id).map_err(|e| StorageError::InvalidId(e.to_string()))?;
        }

        // Update cache and tag index together
        {
            let mut cache = self.cache.write();
            let mut tag_idx = self.tag_index.write();

            for node in &nodes {
                // Remove old tags if this is an update
                if let Some(old_node) = cache.get(&node.id) {
                    for tag in &old_node.tags {
                        if let Some(set) = tag_idx.get_mut(tag) {
                            set.remove(&node.id);
                        }
                    }
                }
                // Index new tags
                for tag in &node.tags {
                    tag_idx
                        .entry(tag.clone())
                        .or_default()
                        .insert(node.id.clone());
                }
                cache.insert(node.id.clone(), node.clone());
            }
        }

        // Sync FTS index for batch
        #[cfg(feature = "full-text-search")]
        if let Some(ref idx) = self.fts_index {
            let docs: Vec<crate::search::FTSDocument> = nodes.iter()
                .map(Self::node_to_fts_doc)
                .collect();
            let _ = idx.add_documents(&docs);
            let _ = idx.commit();
        }

        // Add all to write buffer and flush
        {
            let mut buffer = self.write_buffer.write();
            buffer.extend(nodes);
        }

        self.flush_buffer()
    }

    fn ids(&self) -> StorageResult<Vec<String>> {
        Ok(self.cache.read().keys().cloned().collect())
    }

    fn len(&self) -> StorageResult<usize> {
        Ok(self.cache.read().len())
    }

    fn all(&self) -> StorageResult<Vec<MemoryNode>> {
        Ok(self.cache.read().values().cloned().collect())
    }

    fn get_batch(&self, ids: &[&str]) -> StorageResult<Vec<Option<MemoryNode>>> {
        // Single read lock acquisition for all lookups
        let cache = self.cache.read();
        Ok(ids.iter().map(|id| cache.get(*id).cloned()).collect())
    }

    fn find_by_tag(&self, tag: &str) -> StorageResult<Vec<MemoryNode>> {
        // O(k) via inverted tag index where k = nodes matching the tag
        let tag_idx = self.tag_index.read();
        let ids = match tag_idx.get(tag) {
            Some(set) => set,
            None => return Ok(Vec::new()),
        };

        let cache = self.cache.read();
        let mut results = Vec::with_capacity(ids.len());
        for id in ids {
            if let Some(node) = cache.get(id) {
                results.push(node.clone());
            }
        }
        Ok(results)
    }

    fn query(&self, filter: &QueryFilter) -> StorageResult<Vec<MemoryNode>> {
        // Optimization: if only tag filter is used, leverage the inverted index
        let candidates: Vec<MemoryNode> = if let Some(ref tags) = filter.tags {
            if tags.len() == 1
                && filter.content_type.is_none()
                && filter.parent_id.is_none()
                && filter.min_importance.is_none()
                && filter.time_range.is_none()
                && filter.metadata.is_none()
                && filter.semantic_gate.is_none()
                && filter.sort_by.is_none()
                && filter.offset == 0
                && filter.limit == 0
            {
                // Fast path: single-tag query with no other filters
                return self.find_by_tag(&tags[0]);
            }

            // Use tag index to narrow candidates, then apply remaining filters
            let tag_idx = self.tag_index.read();
            let mut candidate_ids: std::collections::HashSet<String> =
                std::collections::HashSet::new();
            for tag in tags {
                if let Some(set) = tag_idx.get(tag) {
                    candidate_ids.extend(set.iter().cloned());
                }
            }
            drop(tag_idx);

            let cache = self.cache.read();
            candidate_ids
                .iter()
                .filter_map(|id| cache.get(id).cloned())
                .collect::<Vec<MemoryNode>>()
        } else {
            // No tag filter — scan cache directly without cloning through all()
            let cache = self.cache.read();
            cache.values().cloned().collect::<Vec<MemoryNode>>()
        };

        // Apply remaining filters on candidates
        let mut results = candidates;

        if let Some(content_type) = filter.content_type {
            results.retain(|node| node.content_type == content_type);
        }
        if let Some(ref parent_id) = filter.parent_id {
            results.retain(|node| node.parent_id.as_deref() == Some(parent_id.as_str()));
        }
        if let Some(min_importance) = filter.min_importance {
            results.retain(|node| node.computed_importance() >= min_importance);
        }
        if let Some((start, end)) = filter.time_range {
            results.retain(|node| node.created_at >= start && node.created_at <= end);
        }
        if let Some(ref metadata_filters) = filter.metadata {
            results.retain(|node| {
                metadata_filters
                    .iter()
                    .all(|mf| node.metadata.get(&mf.key) == Some(&mf.value))
            });
        }

        // Semantic gate filter (BriLLM SiFu inspired — "cortical region localization")
        if let Some(ref gate) = filter.semantic_gate {
            if gate.is_active() {
                results.retain(|node| {
                    match &node.semantic_axes {
                        Some(axes) => gate.matches(axes),
                        None => false,
                    }
                });
            }
        }

        // Sorting
        if let Some(sort_by) = filter.sort_by {
            use crate::storage::backend::SortField;
            match sort_by {
                SortField::CreatedAt => results.sort_by_key(|n| n.created_at),
                SortField::LastAccessedAt => results.sort_by_key(|n| n.last_accessed_at),
                SortField::Importance => results.sort_by(|a, b| {
                    a.computed_importance()
                        .partial_cmp(&b.computed_importance())
                        .unwrap_or(std::cmp::Ordering::Equal)
                }),
                SortField::AccessCount => results.sort_by_key(|n| n.access_count),
            }
            if filter.sort_order == crate::storage::backend::SortOrder::Desc {
                results.reverse();
            }
        }

        // Pagination
        if filter.offset > 0 {
            if filter.offset >= results.len() {
                return Ok(Vec::new());
            }
            results = results.split_off(filter.offset);
        }
        if filter.limit > 0 && results.len() > filter.limit {
            results.truncate(filter.limit);
        }

        Ok(results)
    }

    fn record_access(&self, id: &str) -> StorageResult<()> {
        let mut cache = self.cache.write();
        let node = cache
            .get_mut(id)
            .ok_or_else(|| StorageError::NotFound(id.to_string()))?;
        node.record_access();

        // Queue the updated node for write-back
        let updated = node.clone();
        drop(cache);

        self.write_buffer.write().push(updated);
        Ok(())
    }

    fn flush(&self) -> StorageResult<()> {
        // Flush pending deletes first (so appended writes don't conflict)
        self.flush_pending_deletes()?;
        self.flush_buffer()
    }

    fn backend_name(&self) -> &str {
        "lance"
    }

    /// Return OCC metrics snapshot as JSON (Phase-74).
    fn occ_stats_json(&self) -> Option<serde_json::Value> {
        let snap = self.occ_stats();
        Some(serde_json::json!({
            "merge_ops": snap.merge_ops,
            "rows_inserted": snap.rows_inserted,
            "rows_updated": snap.rows_updated,
            "total_retries": snap.total_retries,
            "conflict_failures": snap.conflict_failures,
            "legacy_appends": snap.legacy_appends,
            "retry_success_rate": snap.retry_success_rate(),
        }))
    }

    /// Override query_expr for optimized FilterExpr evaluation.
    ///
    /// The Lance backend evaluates FilterExpr directly on the in-memory cache,
    /// leveraging the tag index for Tag predicates. For datasets flushed to disk,
    /// the `to_lance_sql()` predicate can be used for true columnar push-down.
    fn query_expr(
        &self,
        expr: &crate::storage::filter_expr::FilterExpr,
        sort_by: Option<crate::storage::backend::SortField>,
        sort_order: crate::storage::backend::SortOrder,
        offset: usize,
        limit: usize,
    ) -> StorageResult<Vec<MemoryNode>> {
        use crate::storage::filter_expr::FilterExpr;
        use crate::storage::backend::{SortField, SortOrder};

        // Optimization: try to extract single-tag from the expression for index usage
        let candidates = if let Some(tag) = extract_single_tag_eq(expr) {
            // Fast path: use tag index
            let tag_idx = self.tag_index.read();
            if let Some(ids) = tag_idx.get(tag) {
                let cache = self.cache.read();
                ids.iter()
                    .filter_map(|id| cache.get(id).cloned())
                    .collect::<Vec<MemoryNode>>()
            } else {
                Vec::new()
            }
        } else {
            // General path: scan cache with FilterExpr evaluation
            let cache = self.cache.read();
            if expr.is_always_true() {
                cache.values().cloned().collect()
            } else {
                cache.values()
                    .filter(|node| expr.evaluate(node))
                    .cloned()
                    .collect()
            }
        };

        let mut results = candidates;

        // For complex expressions, apply full filter even on tag-indexed candidates
        // (tag index only narrows by tag, other conditions still need checking)
        if extract_single_tag_eq(expr).is_some() && !expr.is_always_true() {
            results.retain(|node| expr.evaluate(node));
        }

        // Sorting
        if let Some(sort) = sort_by {
            match sort {
                SortField::CreatedAt => results.sort_by_key(|n| n.created_at),
                SortField::LastAccessedAt => results.sort_by_key(|n| n.last_accessed_at),
                SortField::Importance => results.sort_by(|a, b| {
                    a.computed_importance()
                        .partial_cmp(&b.computed_importance())
                        .unwrap_or(std::cmp::Ordering::Equal)
                }),
                SortField::AccessCount => results.sort_by_key(|n| n.access_count),
            }
            if sort_order == SortOrder::Desc {
                results.reverse();
            }
        }

        // Pagination
        if offset > 0 {
            if offset >= results.len() {
                return Ok(Vec::new());
            }
            results = results.split_off(offset);
        }
        if limit > 0 && results.len() > limit {
            results.truncate(limit);
        }

        Ok(results)
    }

    #[cfg(feature = "full-text-search")]
    fn supports_fts(&self) -> bool {
        self.fts_index.is_some()
    }

    #[cfg(feature = "full-text-search")]
    fn full_text_search(
        &self,
        query_text: &str,
        limit: usize,
    ) -> StorageResult<Vec<(String, f64)>> {
        if let Some(ref idx) = self.fts_index {
            let query = crate::search::FTSQuery::new(query_text).with_limit(limit);
            let results = idx.search(&query)
                .map_err(|e| StorageError::Backend(format!("FTS search error: {}", e)))?;

            Ok(results
                .into_iter()
                .map(|hit| (hit.id, hit.score))
                .collect())
        } else {
            // Fallback to default keyword matching
            let keywords: Vec<String> = query_text.to_lowercase()
                .split_whitespace()
                .map(|s| s.to_string())
                .collect();

            if keywords.is_empty() {
                return Ok(Vec::new());
            }

            let cache = self.cache.read();
            let mut results: Vec<(String, f64)> = Vec::new();

            for (id, node) in cache.iter() {
                let content_lower = node.content.to_lowercase();
                let matches: usize = keywords.iter()
                    .filter(|kw| content_lower.contains(kw.as_str()))
                    .count();
                if matches > 0 {
                    let score = matches as f64 / keywords.len() as f64;
                    results.push((id.clone(), score));
                }
            }

            results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
            results.truncate(limit);
            Ok(results)
        }
    }
}

/// Helper: extract a single tag equality from a FilterExpr for tag-index acceleration.
///
/// Matches patterns like `Eq(Tag, "value")` at the top level or as the sole child
/// of an `And(...)` node.
fn extract_single_tag_eq(expr: &crate::storage::filter_expr::FilterExpr) -> Option<&str> {
    use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};

    match expr {
        FilterExpr::Eq(FilterField::Tag, FilterValue::String(tag)) => Some(tag.as_str()),
        FilterExpr::And(children) if children.len() >= 1 => {
            // Check if the first child is a tag eq — use index for that
            match &children[0] {
                FilterExpr::Eq(FilterField::Tag, FilterValue::String(tag)) => Some(tag.as_str()),
                _ => None,
            }
        }
        _ => None,
    }
}

/// Load all MemoryNodes from a Lance dataset into a HashMap.
async fn load_all_from_dataset(
    ds: &Dataset,
) -> StorageResult<std::collections::HashMap<String, MemoryNode>> {
    use futures::stream::TryStreamExt;

    let mut cache = std::collections::HashMap::new();

    let stream = ds
        .scan()
        .try_into_stream()
        .await
        .map_err(|e| StorageError::Backend(format!("Failed to scan Lance dataset: {}", e)))?;

    let batches: Vec<RecordBatch> = stream
        .try_collect::<Vec<_>>()
        .await
        .map_err(|e| StorageError::Backend(format!("Failed to collect Lance batches: {}", e)))?;

    for batch in &batches {
        let nodes = record_batch_to_memory_nodes(batch)?;
        for node in nodes {
            cache.insert(node.id.clone(), node);
        }
    }

    Ok(cache)
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::tempdir;

    #[test]
    fn test_lance_backend_create_and_put() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test.lance");

        let config = LanceConfig::new(&lance_path)
            .with_embedding_dim(4)
            .with_write_batch_size(2); // small batch for testing

        let backend = LanceStorageBackend::open(config).unwrap();
        assert_eq!(backend.backend_name(), "lance");
        assert_eq!(backend.len().unwrap(), 0);

        // Put a node
        let node = MemoryNode::new("Test content")
            .with_embedding(vec![0.1, 0.2, 0.3, 0.4])
            .with_tag("test");
        let id = node.id.clone();

        backend.put(node).unwrap();
        assert_eq!(backend.len().unwrap(), 1);

        // Get the node
        let retrieved = backend.get(&id).unwrap().unwrap();
        assert_eq!(retrieved.content, "Test content");
        assert_eq!(retrieved.embedding, Some(vec![0.1, 0.2, 0.3, 0.4]));
    }

    #[test]
    fn test_lance_backend_flush_and_reopen() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test.lance");

        let id;

        // Create and write
        {
            let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
            let backend = LanceStorageBackend::open(config).unwrap();

            let node = MemoryNode::new("Persistent data")
                .with_embedding(vec![1.0, 2.0, 3.0, 4.0]);
            id = node.id.clone();

            backend.put(node).unwrap();
            backend.flush().unwrap();
        }

        // Reopen and verify
        {
            let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
            let backend = LanceStorageBackend::open(config).unwrap();

            assert_eq!(backend.len().unwrap(), 1);
            let retrieved = backend.get(&id).unwrap().unwrap();
            assert_eq!(retrieved.content, "Persistent data");
        }
    }

    #[test]
    fn test_lance_backend_delete() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test.lance");

        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let node = MemoryNode::new("To be deleted")
            .with_embedding(vec![0.0; 4]);
        let id = node.id.clone();

        backend.put(node).unwrap();
        backend.flush().unwrap();
        assert!(backend.contains(&id).unwrap());

        let deleted = backend.delete(&id).unwrap();
        assert!(deleted.is_some());
        assert!(!backend.contains(&id).unwrap());
    }

    #[test]
    fn test_lance_backend_batch_put() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test.lance");

        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let nodes: Vec<MemoryNode> = (0..10)
            .map(|i| {
                MemoryNode::new(format!("Node {}", i))
                    .with_embedding(vec![i as f32; 4])
            })
            .collect();

        let ids: Vec<String> = nodes.iter().map(|n| n.id.clone()).collect();

        backend.put_batch(nodes).unwrap();
        assert_eq!(backend.len().unwrap(), 10);

        for id in &ids {
            assert!(backend.contains(id).unwrap());
        }
    }

    // ====================================================================
    // Optimization Tests: Tag Index, Batch Get, Query Override
    // ====================================================================

    #[test]
    fn test_tag_index_find_by_tag() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("tag_test.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert nodes with different tags
        let n1 = MemoryNode::new("Alpha").with_tag("rust").with_tag("code");
        let n2 = MemoryNode::new("Beta").with_tag("python").with_tag("code");
        let n3 = MemoryNode::new("Gamma").with_tag("rust");
        let id1 = n1.id.clone();
        let id3 = n3.id.clone();

        backend.put(n1).unwrap();
        backend.put(n2).unwrap();
        backend.put(n3).unwrap();

        // Find by "rust" should return 2 nodes
        let rust_nodes = backend.find_by_tag("rust").unwrap();
        assert_eq!(rust_nodes.len(), 2);
        let rust_ids: std::collections::HashSet<String> =
            rust_nodes.iter().map(|n| n.id.clone()).collect();
        assert!(rust_ids.contains(&id1));
        assert!(rust_ids.contains(&id3));

        // Find by "code" should return 2 nodes
        let code_nodes = backend.find_by_tag("code").unwrap();
        assert_eq!(code_nodes.len(), 2);

        // Find by "python" should return 1 node
        let py_nodes = backend.find_by_tag("python").unwrap();
        assert_eq!(py_nodes.len(), 1);

        // Find by non-existent tag should return empty
        let empty = backend.find_by_tag("nonexistent").unwrap();
        assert!(empty.is_empty());
    }

    #[test]
    fn test_tag_index_maintained_on_delete() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("tag_delete.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let n1 = MemoryNode::new("One").with_tag("shared");
        let n2 = MemoryNode::new("Two").with_tag("shared");
        let id1 = n1.id.clone();

        backend.put(n1).unwrap();
        backend.put(n2).unwrap();
        assert_eq!(backend.find_by_tag("shared").unwrap().len(), 2);

        // Delete first node — tag index should be updated
        backend.delete(&id1).unwrap();
        assert_eq!(backend.find_by_tag("shared").unwrap().len(), 1);
    }

    #[test]
    fn test_tag_index_maintained_on_update() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("tag_update.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let mut n1 = MemoryNode::new("Evolving").with_tag("old-tag");
        let id1 = n1.id.clone();

        backend.put(n1.clone()).unwrap();
        assert_eq!(backend.find_by_tag("old-tag").unwrap().len(), 1);

        // Update the node with a new tag
        n1.tags = vec!["new-tag".to_string()];
        backend.put(n1).unwrap();

        // old-tag should be gone
        assert_eq!(backend.find_by_tag("old-tag").unwrap().len(), 0);
        // new-tag should be present
        let results = backend.find_by_tag("new-tag").unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, id1);
    }

    #[test]
    fn test_get_batch_optimization() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("batch_get.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let n1 = MemoryNode::new("First");
        let n2 = MemoryNode::new("Second");
        let id1 = n1.id.clone();
        let id2 = n2.id.clone();

        backend.put(n1).unwrap();
        backend.put(n2).unwrap();

        // Batch get with mixed existing/non-existing
        let results = backend.get_batch(&[&id1, "nonexistent", &id2]).unwrap();
        assert_eq!(results.len(), 3);
        assert!(results[0].is_some());
        assert_eq!(results[0].as_ref().unwrap().content, "First");
        assert!(results[1].is_none());
        assert!(results[2].is_some());
        assert_eq!(results[2].as_ref().unwrap().content, "Second");
    }

    #[test]
    fn test_query_with_tag_filter_uses_index() {
        use crate::storage::backend::QueryFilter;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("query_test.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert 100 nodes, ~half with "alpha" tag, ~half with "beta"
        for i in 0..100 {
            let tag = if i % 2 == 0 { "alpha" } else { "beta" };
            let node = MemoryNode::new(format!("Node {}", i)).with_tag(tag);
            backend.put(node).unwrap();
        }

        // Query by tag
        let filter = QueryFilter {
            tags: Some(vec!["alpha".to_string()]),
            ..Default::default()
        };
        let results = backend.query(&filter).unwrap();
        assert_eq!(results.len(), 50);

        // Query by tag + content_type (should narrow down)
        let filter = QueryFilter {
            tags: Some(vec!["alpha".to_string()]),
            content_type: Some(crate::types::ContentType::Text),
            ..Default::default()
        };
        let results = backend.query(&filter).unwrap();
        // All should be Text since MemoryNode::new defaults to Text
        assert_eq!(results.len(), 50);

        // Query with limit
        let filter = QueryFilter {
            tags: Some(vec!["alpha".to_string()]),
            limit: 10,
            ..Default::default()
        };
        let results = backend.query(&filter).unwrap();
        assert_eq!(results.len(), 10);
    }

    #[test]
    fn test_pending_deletes_flushed_on_flush() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("pending_del.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let n1 = MemoryNode::new("Deletable").with_embedding(vec![1.0, 2.0, 3.0, 4.0]);
        let id1 = n1.id.clone();

        backend.put(n1).unwrap();
        backend.flush().unwrap(); // persist to disk

        // Delete (queued as pending)
        backend.delete(&id1).unwrap();
        assert!(!backend.contains(&id1).unwrap()); // removed from cache

        // Flush to apply pending deletes to disk
        backend.flush().unwrap();

        // Reopen and verify the node is gone from disk too
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend2 = LanceStorageBackend::open(config).unwrap();
        assert_eq!(backend2.len().unwrap(), 0);
        assert!(!backend2.contains(&id1).unwrap());
    }

    // ================================================================
    // Semantic Gate Integration Tests (BriLLM SiFu)
    // ================================================================

    #[test]
    fn test_semantic_gate_query_filter() {
        use crate::types::SemanticAxes;
        use crate::storage::backend::SemanticGateFilter;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("sem_gate.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert nodes with different semantic axes (simulating cortical regions)
        let social_node = MemoryNode::new("Social topic: family dinner")
            .with_embedding(vec![1.0, 0.0, 0.0, 0.0])
            .with_semantic_axes(SemanticAxes::new(0.8, 0.1, 0.2));  // High social
        let sensory_node = MemoryNode::new("Sensory topic: bright colors")
            .with_embedding(vec![0.0, 1.0, 0.0, 0.0])
            .with_semantic_axes(SemanticAxes::new(0.1, 0.9, 0.3));  // High sensory
        let spatial_node = MemoryNode::new("Spatial topic: mountain trail")
            .with_embedding(vec![0.0, 0.0, 1.0, 0.0])
            .with_semantic_axes(SemanticAxes::new(0.2, 0.3, 0.9));  // High locality
        let no_axes_node = MemoryNode::new("No semantic axes")
            .with_embedding(vec![0.0, 0.0, 0.0, 1.0]);  // No semantic_axes

        backend.put(social_node.clone()).unwrap();
        backend.put(sensory_node.clone()).unwrap();
        backend.put(spatial_node.clone()).unwrap();
        backend.put(no_axes_node.clone()).unwrap();

        // Gate: high social (axis_0 > 0.5)
        let gate = SemanticGateFilter::axis_0(0.5, 1.0);
        let results = backend.find_by_semantic_gate(&gate).unwrap();
        assert_eq!(results.len(), 1, "Only social node should match");
        assert_eq!(results[0].id, social_node.id);

        // Gate: high sensory (axis_1 > 0.5)
        let gate = SemanticGateFilter {
            axis_1_range: Some((0.5, 1.0)),
            ..Default::default()
        };
        let results = backend.find_by_semantic_gate(&gate).unwrap();
        assert_eq!(results.len(), 1, "Only sensory node should match");
        assert_eq!(results[0].id, sensory_node.id);

        // Gate: all axes wide range (all 3 nodes with axes should match)
        let gate = SemanticGateFilter::all_axes((0.0, 1.0), (0.0, 1.0), (0.0, 1.0));
        let results = backend.find_by_semantic_gate(&gate).unwrap();
        assert_eq!(results.len(), 3, "All nodes with axes should match, but not no_axes");

        // Gate: narrow range that excludes all
        let gate = SemanticGateFilter::all_axes((0.95, 1.0), (0.95, 1.0), (0.95, 1.0));
        let results = backend.find_by_semantic_gate(&gate).unwrap();
        assert_eq!(results.len(), 0, "No nodes in this narrow range");
    }

    #[test]
    fn test_semantic_gate_combined_with_tag_filter() {
        use crate::types::SemanticAxes;
        use crate::storage::backend::SemanticGateFilter;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("sem_gate_tags.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let n1 = MemoryNode::new("Social science")
            .with_embedding(vec![1.0, 0.0, 0.0, 0.0])
            .with_tags(vec!["science".to_string()])
            .with_semantic_axes(SemanticAxes::new(0.8, 0.1, 0.2));
        let n2 = MemoryNode::new("Natural science")
            .with_embedding(vec![0.0, 1.0, 0.0, 0.0])
            .with_tags(vec!["science".to_string()])
            .with_semantic_axes(SemanticAxes::new(0.1, 0.7, 0.5));
        let n3 = MemoryNode::new("Social art")
            .with_embedding(vec![0.0, 0.0, 1.0, 0.0])
            .with_tags(vec!["art".to_string()])
            .with_semantic_axes(SemanticAxes::new(0.9, 0.2, 0.1));

        let id1 = n1.id.clone();
        backend.put(n1).unwrap();
        backend.put(n2).unwrap();
        backend.put(n3).unwrap();

        // Combined: tag=science AND high social
        let filter = QueryFilter {
            tags: Some(vec!["science".to_string()]),
            semantic_gate: Some(SemanticGateFilter::axis_0(0.5, 1.0)),
            ..Default::default()
        };
        let results = backend.query(&filter).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, id1, "Only 'Social science' matches both filters");
    }

    #[test]
    fn test_semantic_gate_roundtrip_lance() {
        use crate::types::SemanticAxes;
        use crate::storage::backend::SemanticGateFilter;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("sem_roundtrip.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let axes = SemanticAxes::new(0.42, -0.17, 0.88);
        let node = MemoryNode::new("Roundtrip test")
            .with_embedding(vec![1.0, 2.0, 3.0, 4.0])
            .with_semantic_axes(axes);
        let id = node.id.clone();

        backend.put(node).unwrap();
        backend.flush().unwrap();

        // Reopen from disk
        let config2 = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend2 = LanceStorageBackend::open(config2).unwrap();

        let loaded = backend2.get(&id).unwrap().unwrap();
        let loaded_axes = loaded.semantic_axes.unwrap();
        assert!((loaded_axes.axis_0 - 0.42).abs() < 1e-5);
        assert!((loaded_axes.axis_1 - (-0.17)).abs() < 1e-5);
        assert!((loaded_axes.axis_2 - 0.88).abs() < 1e-5);

        // Query via semantic gate on reopened store
        let gate = SemanticGateFilter::axis_0(0.3, 0.5);
        let results = backend2.find_by_semantic_gate(&gate).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, id);
    }

    // ================================================================
    // Morton Code Semantic Compact Tests
    // ================================================================

    #[test]
    fn test_semantic_compact_basic() {
        use crate::types::SemanticAxes;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("compact.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert nodes with diverse semantic axes
        let ids: Vec<String> = (0..10).map(|i| {
            let axes = SemanticAxes::new(
                (i as f32) * 0.1,
                1.0 - (i as f32) * 0.1,
                ((i as f32) * 0.3).sin(),
            );
            let node = MemoryNode::new(&format!("Node {}", i))
                .with_embedding(vec![i as f32; 4])
                .with_semantic_axes(axes);
            let id = node.id.clone();
            backend.put(node).unwrap();
            id
        }).collect();

        // Also insert a node without semantic axes
        let no_axes = MemoryNode::new("No axes node")
            .with_embedding(vec![99.0; 4]);
        backend.put(no_axes).unwrap();

        // Run semantic compact
        let (total, with_axes) = backend.semantic_compact().unwrap();
        assert_eq!(total, 11);
        assert_eq!(with_axes, 10);

        // Verify all nodes still exist after compaction
        assert_eq!(backend.len().unwrap(), 11);
        for id in &ids {
            assert!(backend.contains(id).unwrap(), "Node {} should still exist", id);
        }

        // Reopen from disk and verify persistence
        let config2 = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend2 = LanceStorageBackend::open(config2).unwrap();
        assert_eq!(backend2.len().unwrap(), 11);

        // Verify semantic axes are preserved after compaction
        let first = backend2.get(&ids[0]).unwrap().unwrap();
        assert!(first.semantic_axes.is_some());
    }

    #[test]
    fn test_semantic_compact_empty_store() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("compact_empty.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let (total, with_axes) = backend.semantic_compact().unwrap();
        assert_eq!(total, 0);
        assert_eq!(with_axes, 0);
    }

    #[test]
    fn test_semantic_compact_preserves_data_integrity() {
        use crate::types::SemanticAxes;
        use crate::semantic::projector::morton_code_3d;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("compact_order.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert nodes with distinct semantic positions
        let test_data = vec![
            ("Far", SemanticAxes::new(2.0, 2.0, 2.0)),
            ("Near-neg", SemanticAxes::new(-1.0, -1.0, -1.0)),
            ("Center", SemanticAxes::new(0.5, 0.5, 0.5)),
        ];

        let mut ids = Vec::new();
        for (i, (label, axes)) in test_data.iter().enumerate() {
            let node = MemoryNode::new(*label)
                .with_embedding(vec![i as f32; 4])
                .with_semantic_axes(*axes);
            ids.push(node.id.clone());
            backend.put(node).unwrap();
        }

        // Run compact
        let (total, with_axes) = backend.semantic_compact().unwrap();
        assert_eq!(total, 3);
        assert_eq!(with_axes, 3);

        // After compact, reopen and verify all data is intact
        let config2 = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend2 = LanceStorageBackend::open(config2).unwrap();
        assert_eq!(backend2.len().unwrap(), 3);

        // Each node should have its original semantic axes and content
        for (i, (label, expected_axes)) in test_data.iter().enumerate() {
            let node = backend2.get(&ids[i]).unwrap().expect("Node should exist");
            assert_eq!(node.content, *label);
            let axes = node.semantic_axes.expect("Should have semantic axes");
            assert!((axes.axis_0 - expected_axes.axis_0).abs() < 1e-5);
            assert!((axes.axis_1 - expected_axes.axis_1).abs() < 1e-5);
            assert!((axes.axis_2 - expected_axes.axis_2).abs() < 1e-5);
        }

        // Verify the Morton code ordering logic is correct (unit test the sort)
        let mut codes: Vec<(u64, &str)> = test_data.iter().map(|(label, axes)| {
            (morton_code_3d(axes.axis_0, axes.axis_1, axes.axis_2), *label)
        }).collect();
        codes.sort_by_key(|(m, _)| *m);

        // "Near-neg" (-1,-1,-1) should have the smallest Morton code
        assert_eq!(codes[0].1, "Near-neg", "Smallest Morton code");
    }

    // ================================================================
    // Full-text search integration tests (requires full-text-search feature)
    // ================================================================

    #[cfg(feature = "full-text-search")]
    #[test]
    fn test_lance_fts_auto_sync_on_put() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("fts_test.lance");

        let mut backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        // Enable FTS explicitly
        let fts_path = dir.path().join("fts_index");
        backend.enable_fts(crate::search::FTSConfig::new(fts_path.to_string_lossy().to_string())).unwrap();
        assert!(backend.supports_fts());

        // Put some documents
        let n1 = MemoryNode::new("Rust is a systems programming language");
        let n2 = MemoryNode::new("Python is great for data science");
        let n3 = MemoryNode::new("Rust and Python can work together via PyO3");
        let id1 = n1.id.clone();
        backend.put(n1).unwrap();
        backend.put(n2).unwrap();
        backend.put(n3).unwrap();

        // Full-text search should work
        let results = backend.full_text_search("rust programming", 10).unwrap();
        assert!(!results.is_empty(), "Should find documents matching 'rust programming'");
        // The top result should contain "Rust" and "programming"
        assert!(results.iter().any(|(id, _)| *id == id1),
            "doc with 'Rust is a systems programming language' should be in results");
    }

    #[cfg(feature = "full-text-search")]
    #[test]
    fn test_lance_fts_delete_sync() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("fts_del.lance");

        let mut backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        let fts_path = dir.path().join("fts_del_idx");
        backend.enable_fts(crate::search::FTSConfig::new(fts_path.to_string_lossy().to_string())).unwrap();

        let n_a = MemoryNode::new("Tantivy full text search engine");
        let n_b = MemoryNode::new("Elasticsearch alternative in Rust");
        let id_a = n_a.id.clone();
        backend.put(n_a).unwrap();
        backend.put(n_b).unwrap();

        // Should find doc-a
        let results = backend.full_text_search("tantivy", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].0, id_a);

        // Delete doc-a
        backend.delete(&id_a).unwrap();

        // Should no longer find doc-a
        let results = backend.full_text_search("tantivy", 10).unwrap();
        assert!(results.is_empty(), "Deleted doc should not appear in FTS results");
    }

    #[cfg(feature = "full-text-search")]
    #[test]
    fn test_lance_fts_batch_put_sync() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("fts_batch.lance");

        let mut backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        let fts_path = dir.path().join("fts_batch_idx");
        backend.enable_fts(crate::search::FTSConfig::new(fts_path.to_string_lossy().to_string())).unwrap();

        let b1 = MemoryNode::new("Machine learning with neural networks");
        let b2 = MemoryNode::new("Deep learning and convolutional neural networks");
        let b3 = MemoryNode::new("Database indexing and query optimization");
        let id_b3 = b3.id.clone();
        backend.put_batch(vec![b1, b2, b3]).unwrap();

        // Search for neural
        let results = backend.full_text_search("neural networks", 10).unwrap();
        assert_eq!(results.len(), 2, "Should find 2 docs about neural networks");

        // Search for database
        let results = backend.full_text_search("database indexing", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].0, id_b3);
    }

    #[cfg(feature = "full-text-search")]
    #[test]
    fn test_lance_fts_update_sync() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("fts_upd.lance");

        let mut backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        let fts_path = dir.path().join("fts_upd_idx");
        backend.enable_fts(crate::search::FTSConfig::new(fts_path.to_string_lossy().to_string())).unwrap();

        let n = MemoryNode::new("Original content about cats");
        let id = n.id.clone();
        backend.put(n).unwrap();

        // Verify can find original
        let results = backend.full_text_search("cats", 10).unwrap();
        assert_eq!(results.len(), 1);

        // Update content: create new node with same ID
        let mut updated = MemoryNode::new("Updated content about dogs");
        updated.id = id.clone();
        backend.put(updated).unwrap();

        // Should NOT find old content
        let results = backend.full_text_search("cats", 10).unwrap();
        assert!(results.is_empty(), "Old content should be gone after update");

        // Should find new content
        let results = backend.full_text_search("dogs", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].0, id);
    }

    #[cfg(feature = "full-text-search")]
    #[test]
    fn test_lance_fts_auto_enabled_on_open() {
        // With full-text-search feature, open() automatically creates FTS index
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("fts_auto.lance");

        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        // FTS should be auto-enabled when feature is compiled in
        assert!(backend.supports_fts(), "FTS should be auto-enabled with full-text-search feature");

        let n1 = MemoryNode::new("hello world example");
        let id1 = n1.id.clone();
        let n2 = MemoryNode::new("goodbye world test");
        backend.put(n1).unwrap();
        backend.put(n2).unwrap();

        // BM25 search should work automatically
        let results = backend.full_text_search("hello", 10).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].0, id1);
    }

    // ── query_expr integration tests ─────────────────────────────

    #[test]
    fn test_query_expr_basic_tag_filter() {
        use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("qe_tag.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        let mut n1 = MemoryNode::new("Node about Rust");
        n1.tags = vec!["rust".to_string(), "programming".to_string()];
        let id1 = n1.id.clone();

        let mut n2 = MemoryNode::new("Node about Python");
        n2.tags = vec!["python".to_string(), "programming".to_string()];

        let mut n3 = MemoryNode::new("Node about cooking");
        n3.tags = vec!["cooking".to_string()];

        backend.put(n1).unwrap();
        backend.put(n2).unwrap();
        backend.put(n3).unwrap();

        // Single tag filter
        let expr = FilterExpr::tag_eq("rust");
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, id1);

        // OR tag filter
        let expr = FilterExpr::tag_any(&["rust", "python"]);
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 2);

        // Tag + content contains
        let expr = FilterExpr::and(vec![
            FilterExpr::tag_eq("programming"),
            FilterExpr::contains(FilterField::Content, "Python"),
        ]);
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 1);
    }

    #[test]
    fn test_query_expr_complex_and_or_not() {
        use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("qe_complex.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        for i in 0..10 {
            let mut node = MemoryNode::new(format!("Document number {}", i));
            node.importance = i as f64 * 0.1;
            node.tags = if i % 2 == 0 {
                vec!["even".to_string()]
            } else {
                vec!["odd".to_string()]
            };
            let now = chrono::Utc::now().timestamp_millis();
            node.last_accessed_at = now;
            node.access_count = i as u64;
            backend.put(node).unwrap();
        }

        // (tag=even OR tag=odd) — should match all 10
        let expr = FilterExpr::tag_any(&["even", "odd"]);
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 10);

        // NOT tag=even — should match 5 odd nodes
        let expr = FilterExpr::not(FilterExpr::tag_eq("even"));
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 5);

        // access_count BETWEEN 3 AND 7
        let expr = FilterExpr::between(
            FilterField::AccessCount,
            FilterValue::Int(3),
            FilterValue::Int(7),
        );
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 5);

        // access_count IN (1, 3, 5)
        let expr = FilterExpr::is_in(
            FilterField::AccessCount,
            vec![FilterValue::Int(1), FilterValue::Int(3), FilterValue::Int(5)],
        );
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 3);
    }

    #[test]
    fn test_query_expr_sorting_and_pagination() {
        use crate::storage::filter_expr::FilterExpr;
        use crate::storage::backend::{SortField, SortOrder};

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("qe_sort.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        for i in 0..5u64 {
            let mut node = MemoryNode::new(format!("Item {}", i));
            node.access_count = i;
            backend.put(node).unwrap();
        }

        // Sort by access_count ascending
        let results = backend.query_expr(
            &FilterExpr::True,
            Some(SortField::AccessCount),
            SortOrder::Asc,
            0, 0,
        ).unwrap();
        assert_eq!(results.len(), 5);
        assert_eq!(results[0].access_count, 0);
        assert_eq!(results[4].access_count, 4);

        // Sort by access_count descending
        let results = backend.query_expr(
            &FilterExpr::True,
            Some(SortField::AccessCount),
            SortOrder::Desc,
            0, 0,
        ).unwrap();
        assert_eq!(results[0].access_count, 4);
        assert_eq!(results[4].access_count, 0);

        // Pagination: offset=2, limit=2
        let results = backend.query_expr(
            &FilterExpr::True,
            Some(SortField::AccessCount),
            SortOrder::Asc,
            2, 2,
        ).unwrap();
        assert_eq!(results.len(), 2);
        assert_eq!(results[0].access_count, 2);
        assert_eq!(results[1].access_count, 3);
    }

    #[test]
    fn test_query_expr_string_predicates() {
        use crate::storage::filter_expr::{FilterExpr, FilterField};

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("qe_str.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        backend.put(MemoryNode::new("Hello World")).unwrap();
        backend.put(MemoryNode::new("Hello Rust")).unwrap();
        backend.put(MemoryNode::new("Goodbye World")).unwrap();

        // Contains
        let results = backend.query_expr(
            &FilterExpr::contains(FilterField::Content, "Hello"),
            None, Default::default(), 0, 0,
        ).unwrap();
        assert_eq!(results.len(), 2);

        // StartsWith
        let results = backend.query_expr(
            &FilterExpr::starts_with(FilterField::Content, "Goodbye"),
            None, Default::default(), 0, 0,
        ).unwrap();
        assert_eq!(results.len(), 1);

        // LIKE pattern
        let results = backend.query_expr(
            &FilterExpr::like(FilterField::Content, "Hello%"),
            None, Default::default(), 0, 0,
        ).unwrap();
        assert_eq!(results.len(), 2);
    }

    #[test]
    fn test_query_expr_lance_sql_generation() {
        use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};

        // Verify that FilterExpr correctly generates Lance SQL predicates
        let expr = FilterExpr::and(vec![
            FilterExpr::gte(FilterField::Importance, FilterValue::Float(0.5)),
            FilterExpr::or(vec![
                FilterExpr::tag_eq("rust"),
                FilterExpr::tag_eq("python"),
            ]),
            FilterExpr::between(
                FilterField::CreatedAt,
                FilterValue::Int(1000),
                FilterValue::Int(2000),
            ),
        ]);

        let sql = expr.to_lance_sql().unwrap();
        assert!(sql.contains("importance >= "));
        assert!(sql.contains("tags_json LIKE"));
        assert!(sql.contains("created_at BETWEEN 1000 AND 2000"));
    }

    #[test]
    fn test_ensure_scalar_indexes() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("idx_test.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        // Insert some data and flush to create the dataset
        let mut node = MemoryNode::new("test node");
        node.embedding = Some(vec![1.0, 2.0, 3.0, 4.0]);
        backend.put(node).unwrap();
        backend.flush().unwrap();

        // Create scalar indexes
        let created = backend.ensure_scalar_indexes().unwrap();
        // Should create some indexes (the exact count depends on schema columns present)
        assert!(created > 0, "Should have created at least one scalar index");

        // Calling again should be idempotent (0 new indexes)
        let created2 = backend.ensure_scalar_indexes().unwrap();
        assert_eq!(created2, 0, "Second call should be idempotent");
    }

    // ── T53.6: 综合集成测试 ──────────────────────────────────────

    #[test]
    fn test_filter_expr_end_to_end_workflow() {
        use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};
        use crate::storage::backend::{SortField, SortOrder};
        use crate::types::{ContentType, SemanticAxes};

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("e2e_filter.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        // Build a dataset of 20 nodes with varied attributes
        for i in 0u64..20 {
            let mut node = MemoryNode::new(format!("Document {} about topic {}", i, if i % 3 == 0 { "AI" } else if i % 3 == 1 { "database" } else { "networking" }));
            node.importance = (i as f64) * 0.05;
            node.access_count = i;
            node.embedding = Some(vec![i as f32, (i * 2) as f32, (i * 3) as f32, (i * 4) as f32]);
            node.tags = match i % 4 {
                0 => vec!["rust".to_string(), "backend".to_string()],
                1 => vec!["python".to_string(), "ml".to_string()],
                2 => vec!["rust".to_string(), "frontend".to_string()],
                _ => vec!["typescript".to_string(), "frontend".to_string()],
            };
            if i < 10 {
                node.semantic_axes = Some(SemanticAxes::new(
                    i as f32 * 0.1,
                    (20 - i) as f32 * 0.1,
                    (i % 5) as f32 * 0.2,
                ));
            }
            backend.put(node).unwrap();
        }

        // 1. Basic tag + importance filter
        let expr = FilterExpr::and(vec![
            FilterExpr::tag_eq("rust"),
            FilterExpr::gte(FilterField::Importance, FilterValue::Float(0.3)),
        ]);
        let results = backend.query_expr(&expr, None, SortOrder::Asc, 0, 0).unwrap();
        for r in &results {
            assert!(r.tags.contains(&"rust".to_string()));
            assert!(r.computed_importance() >= 0.3);
        }

        // 2. Complex OR + NOT filter
        let expr = FilterExpr::and(vec![
            FilterExpr::or(vec![
                FilterExpr::tag_eq("python"),
                FilterExpr::tag_eq("typescript"),
            ]),
            FilterExpr::not(FilterExpr::contains(FilterField::Content, "networking")),
        ]);
        let results = backend.query_expr(&expr, None, SortOrder::Asc, 0, 0).unwrap();
        for r in &results {
            let has_py_or_ts = r.tags.contains(&"python".to_string())
                || r.tags.contains(&"typescript".to_string());
            assert!(has_py_or_ts);
            assert!(!r.content.contains("networking"));
        }

        // 3. Sorting by access count descending + pagination
        let results = backend.query_expr(
            &FilterExpr::True,
            Some(SortField::AccessCount),
            SortOrder::Desc,
            5, 3,  // skip 5, take 3
        ).unwrap();
        assert_eq!(results.len(), 3);
        assert_eq!(results[0].access_count, 14);
        assert_eq!(results[1].access_count, 13);
        assert_eq!(results[2].access_count, 12);

        // 4. BETWEEN on access_count
        let expr = FilterExpr::between(
            FilterField::AccessCount,
            FilterValue::UInt(5),
            FilterValue::UInt(10),
        );
        let results = backend.query_expr(&expr, None, SortOrder::Asc, 0, 0).unwrap();
        assert_eq!(results.len(), 6); // 5,6,7,8,9,10
        for r in &results {
            assert!(r.access_count >= 5 && r.access_count <= 10);
        }

        // 5. String LIKE search
        let expr = FilterExpr::like(FilterField::Content, "%AI%");
        let results = backend.query_expr(&expr, None, SortOrder::Asc, 0, 0).unwrap();
        // Nodes 0,3,6,9,12,15,18 have "AI" in content (i % 3 == 0)
        assert_eq!(results.len(), 7);

        // 6. Lance SQL generation (verify it produces valid SQL strings)
        let complex_expr = FilterExpr::and(vec![
            FilterExpr::gte(FilterField::Importance, FilterValue::Float(0.25)),
            FilterExpr::lt(FilterField::AccessCount, FilterValue::UInt(15)),
            FilterExpr::or(vec![
                FilterExpr::tag_eq("rust"),
                FilterExpr::tag_eq("python"),
            ]),
        ]);
        let sql = complex_expr.to_lance_sql();
        assert!(sql.is_some(), "Complex filter should generate valid SQL");
        let sql_str = sql.unwrap();
        assert!(sql_str.contains("importance >= "));
        assert!(sql_str.contains("access_count < "));
    }

    #[test]
    fn test_query_filter_backward_compat_with_filter_expr() {
        use crate::storage::filter_expr::FilterExpr;
        use crate::storage::backend::QueryFilter;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("compat.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        // Insert test data
        let mut n1 = MemoryNode::new("Hello Rust world");
        n1.tags = vec!["rust".to_string()];
        n1.importance = 0.9;
        let id1 = n1.id.clone();

        let mut n2 = MemoryNode::new("Hello Python world");
        n2.tags = vec!["python".to_string()];
        n2.importance = 0.3;

        backend.put(n1).unwrap();
        backend.put(n2).unwrap();

        // Old-style QueryFilter
        let old_filter = QueryFilter {
            tags: Some(vec!["rust".to_string()]),
            ..Default::default()
        };
        let old_results = backend.query(&old_filter).unwrap();

        // New FilterExpr from QueryFilter conversion
        let expr = FilterExpr::from(&old_filter);
        let new_results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();

        // Both should return the same results
        assert_eq!(old_results.len(), new_results.len());
        assert_eq!(old_results[0].id, new_results[0].id);
        assert_eq!(old_results[0].id, id1);
    }

    #[test]
    fn test_scalar_indexes_with_query_performance() {
        // Test that scalar indexes are created and queries work after index creation
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("idx_perf.lance");
        let backend = LanceStorageBackend::open(
            LanceConfig::new(&lance_path).with_embedding_dim(4),
        ).unwrap();

        // Insert 100 nodes
        for i in 0u64..100 {
            let mut node = MemoryNode::new(format!("Perf test node {}", i));
            node.importance = (i as f64) / 100.0;
            node.access_count = i;
            node.embedding = Some(vec![i as f32; 4]);
            backend.put(node).unwrap();
        }

        // Flush to create dataset on disk
        backend.flush().unwrap();

        // Create scalar indexes
        let created = backend.ensure_scalar_indexes().unwrap();
        assert!(created > 0);

        // Query using FilterExpr (should benefit from indexes)
        use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};

        // All nodes have boosted importance due to access_count via computed_importance()
        // Use a high threshold that most won't pass: raw importance of 0.95 + boost
        // Node 95..99 have raw importance 0.95..0.99
        let expr = FilterExpr::gte(FilterField::Importance, FilterValue::Float(0.5));
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        // Many nodes pass 0.5 threshold after computed_importance() boost
        assert!(!results.is_empty(), "Should find nodes with importance >= 0.5");
        for r in &results {
            assert!(r.computed_importance() >= 0.5);
        }

        // Range query
        let expr = FilterExpr::between(
            FilterField::AccessCount,
            FilterValue::UInt(50),
            FilterValue::UInt(60),
        );
        let results = backend.query_expr(&expr, None, Default::default(), 0, 0).unwrap();
        assert_eq!(results.len(), 11); // 50,51,...,60
    }

    // ── Upsert tests ───────────────────────────────────────────────

    #[test]
    fn test_upsert_insert_new() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test_upsert_insert.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let mut node = MemoryNode::new("new node");
        node.id = "upsert-1".to_string();

        // First upsert: should insert (return false = not existing)
        let existed = backend.upsert(node.clone()).unwrap();
        assert!(!existed);

        // Verify it was stored
        let retrieved = backend.get("upsert-1").unwrap().unwrap();
        assert_eq!(retrieved.content, "new node");
    }

    #[test]
    fn test_upsert_update_existing() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test_upsert_update.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let mut node = MemoryNode::new("original content");
        node.id = "upsert-2".to_string();
        backend.put(node).unwrap();

        // Update via upsert
        let mut updated = MemoryNode::new("updated content");
        updated.id = "upsert-2".to_string();
        let existed = backend.upsert(updated).unwrap();
        assert!(existed);

        // Verify update
        let retrieved = backend.get("upsert-2").unwrap().unwrap();
        assert_eq!(retrieved.content, "updated content");
    }

    #[test]
    fn test_upsert_batch() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test_upsert_batch.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert initial nodes
        for i in 0..3 {
            let mut node = MemoryNode::new(format!("node-{}", i));
            node.id = format!("batch-{}", i);
            backend.put(node).unwrap();
        }

        // Upsert batch: 2 updates + 2 inserts
        let nodes: Vec<MemoryNode> = (0..4).map(|i| {
            let mut node = MemoryNode::new(format!("upserted-{}", i));
            node.id = format!("batch-{}", i); // batch-0,1,2 exist; batch-3 is new
            node
        }).collect();

        let updated_count = backend.upsert_batch(nodes).unwrap();
        assert_eq!(updated_count, 3); // batch-0,1,2 existed

        // Verify all 4 exist with new content
        assert_eq!(backend.len().unwrap(), 4);
        let n3 = backend.get("batch-3").unwrap().unwrap();
        assert_eq!(n3.content, "upserted-3");
    }

    // ── delete_where tests ──────────────────────────────────────────

    #[test]
    fn test_delete_where_by_tag() {
        use crate::storage::filter_expr::FilterExpr;
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test_delete_where.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert nodes with different tags
        for i in 0..6 {
            let mut node = MemoryNode::new(format!("node {}", i));
            node.id = format!("del-{}", i);
            if i < 3 {
                node.tags = vec!["delete-me".to_string()];
            } else {
                node.tags = vec!["keep-me".to_string()];
            }
            backend.put(node).unwrap();
        }
        assert_eq!(backend.len().unwrap(), 6);

        // Delete all with tag "delete-me"
        let filter = FilterExpr::tag_eq("delete-me");
        let deleted = backend.delete_where(&filter).unwrap();
        assert_eq!(deleted.len(), 3);
        assert_eq!(backend.len().unwrap(), 3);

        // Remaining should all have "keep-me" tag
        let remaining = backend.all().unwrap();
        for node in &remaining {
            assert!(node.tags.contains(&"keep-me".to_string()));
        }
    }

    #[test]
    fn test_delete_where_complex_filter() {
        use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test_delete_complex.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let now_ms = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_millis() as i64;

        // Insert nodes with varying access_count
        for i in 0..10 {
            let mut node = MemoryNode::new(format!("node {}", i));
            node.id = format!("cx-{}", i);
            node.access_count = i as u64;
            node.created_at = now_ms - 1000;
            node.last_accessed_at = now_ms;
            backend.put(node).unwrap();
        }

        // Delete nodes with access_count < 3
        let filter = FilterExpr::lt(
            FilterField::AccessCount,
            FilterValue::UInt(3),
        );
        let deleted = backend.delete_where(&filter).unwrap();
        assert_eq!(deleted.len(), 3); // cx-0, cx-1, cx-2

        assert_eq!(backend.len().unwrap(), 7);
    }

    #[test]
    fn test_count_where() {
        use crate::storage::filter_expr::{FilterExpr, FilterField, FilterValue};
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test_count_where.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        for i in 0..10 {
            let mut node = MemoryNode::new(format!("node {}", i));
            node.id = format!("cnt-{}", i);
            node.access_count = i as u64;
            backend.put(node).unwrap();
        }

        let filter = FilterExpr::gte(
            FilterField::AccessCount,
            FilterValue::UInt(5),
        );
        let count = backend.count_where(&filter).unwrap();
        assert_eq!(count, 5); // cnt-5,6,7,8,9
    }

    #[test]
    fn test_delete_where_no_match() {
        use crate::storage::filter_expr::FilterExpr;
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("test_delete_nomatch.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        let mut node = MemoryNode::new("keep this");
        node.id = "keep-1".to_string();
        backend.put(node).unwrap();

        let filter = FilterExpr::tag_eq("nonexistent-tag");
        let deleted = backend.delete_where(&filter).unwrap();
        assert_eq!(deleted.len(), 0);
        assert_eq!(backend.len().unwrap(), 1);
    }

    // ========================================================================
    // Phase-74: OCC (Optimistic Concurrency Control) Tests
    // ========================================================================

    #[test]
    fn test_occ_config_defaults() {
        let cfg = OCCConfig::default();
        assert_eq!(cfg.max_retries, 10);
        assert_eq!(cfg.retry_timeout_ms, 30_000);
        let timeout = cfg.retry_timeout();
        assert_eq!(timeout, std::time::Duration::from_millis(30_000));
    }

    #[test]
    fn test_occ_config_custom() {
        let cfg = OCCConfig {
            max_retries: 5,
            retry_timeout_ms: 5_000,
        };
        assert_eq!(cfg.max_retries, 5);
        assert_eq!(cfg.retry_timeout(), std::time::Duration::from_millis(5_000));
    }

    #[test]
    fn test_lance_config_with_occ() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("occ_cfg.lance");
        let occ = OCCConfig {
            max_retries: 3,
            retry_timeout_ms: 10_000,
        };
        let config = LanceConfig::new(&lance_path)
            .with_embedding_dim(4)
            .with_occ(occ);
        assert_eq!(config.occ.max_retries, 3);
        assert_eq!(config.occ.retry_timeout_ms, 10_000);
    }

    #[test]
    fn test_occ_metrics_snapshot() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("occ_metrics.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Initially all zeros
        let snap = backend.occ_stats();
        assert_eq!(snap.merge_ops, 0);
        assert_eq!(snap.rows_inserted, 0);
        assert_eq!(snap.rows_updated, 0);
        assert_eq!(snap.total_retries, 0);
        assert_eq!(snap.conflict_failures, 0);
        assert_eq!(snap.retry_success_rate(), 1.0); // 0/0 → 1.0

        // Put some data and flush → triggers merge_insert or legacy append
        let mut node = MemoryNode::new("occ test");
        node.id = "occ-1".to_string();
        backend.put(node).unwrap();
        backend.flush().unwrap();

        let snap = backend.occ_stats();
        // After first flush, either merge_ops > 0 or legacy_appends > 0
        assert!(snap.merge_ops + snap.legacy_appends >= 1,
            "Expected at least one write op, got merge={} legacy={}",
            snap.merge_ops, snap.legacy_appends);
    }

    #[test]
    fn test_occ_stats_json_via_trait() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("occ_json.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // StorageBackend::occ_stats_json() should return Some for Lance
        let json = backend.occ_stats_json();
        assert!(json.is_some());
        let val = json.unwrap();
        assert!(val.get("merge_ops").is_some());
        assert!(val.get("retry_success_rate").is_some());
    }

    #[test]
    fn test_occ_merge_insert_upsert_tracking() {
        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("occ_upsert_track.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = LanceStorageBackend::open(config).unwrap();

        // Insert initial data
        for i in 0..5 {
            let mut node = MemoryNode::new(&format!("content-{}", i));
            node.id = format!("upsert-{}", i);
            backend.put(node).unwrap();
        }
        backend.flush().unwrap();

        let snap1 = backend.occ_stats();
        let initial_ops = snap1.merge_ops + snap1.legacy_appends;

        // Update existing data (triggers merge_insert with when_matched_update_all)
        for i in 0..5 {
            let mut node = MemoryNode::new(&format!("updated-content-{}", i));
            node.id = format!("upsert-{}", i);
            backend.put(node).unwrap();
        }
        backend.flush().unwrap();

        let snap2 = backend.occ_stats();
        let after_ops = snap2.merge_ops + snap2.legacy_appends;
        assert!(after_ops > initial_ops,
            "Expected more write ops after upsert: before={} after={}",
            initial_ops, after_ops);

        // Verify data integrity — all 5 records should be updated, not duplicated
        let all = backend.all().unwrap();
        assert_eq!(all.len(), 5, "Should have exactly 5 records after upsert, got {}", all.len());
        for node in &all {
            assert!(node.content.starts_with("updated-"),
                "Node {} should have updated content, got: {}", node.id, node.content);
        }
    }

    #[test]
    fn test_occ_concurrent_writers_same_records() {
        // T74.4: 10 concurrent writers upserting the same 10 records
        use std::sync::Arc;
        use std::thread;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("occ_concurrent_same.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = Arc::new(LanceStorageBackend::open(config).unwrap());

        // Seed initial data
        for i in 0..10 {
            let mut node = MemoryNode::new(&format!("initial-{}", i));
            node.id = format!("shared-{}", i);
            backend.put(node).unwrap();
        }
        backend.flush().unwrap();

        // 10 threads, each updating all 10 records
        let num_writers = 10;
        let handles: Vec<_> = (0..num_writers).map(|writer_id| {
            let backend = Arc::clone(&backend);
            thread::spawn(move || {
                for i in 0..10 {
                    let mut node = MemoryNode::new(
                        &format!("writer-{}-content-{}", writer_id, i)
                    );
                    node.id = format!("shared-{}", i);
                    backend.put(node).unwrap();
                }
                // Each writer flushes independently
                backend.flush().unwrap();
            })
        }).collect();

        for h in handles {
            h.join().expect("Writer thread panicked");
        }

        // Verify: exactly 10 records (no duplicates), all content from some writer
        let all = backend.all().unwrap();
        assert_eq!(all.len(), 10,
            "Should have exactly 10 records after concurrent upsert, got {}",
            all.len());

        // Verify OCC metrics
        let snap = backend.occ_stats();
        assert!(snap.retry_success_rate() > 0.0,
            "Retry success rate should be positive");
    }

    #[test]
    fn test_occ_concurrent_writers_different_records() {
        // T74.4: 10 concurrent writers each upserting their own unique records
        use std::sync::Arc;
        use std::thread;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("occ_concurrent_diff.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = Arc::new(LanceStorageBackend::open(config).unwrap());

        let num_writers = 10;
        let records_per_writer = 10;
        let handles: Vec<_> = (0..num_writers).map(|writer_id| {
            let backend = Arc::clone(&backend);
            thread::spawn(move || {
                for i in 0..records_per_writer {
                    let mut node = MemoryNode::new(
                        &format!("w{}-data-{}", writer_id, i)
                    );
                    node.id = format!("w{}-{}", writer_id, i);
                    backend.put(node).unwrap();
                }
                backend.flush().unwrap();
            })
        }).collect();

        for h in handles {
            h.join().expect("Writer thread panicked");
        }

        // All records should be present — zero data loss
        let all = backend.all().unwrap();
        assert_eq!(all.len(), (num_writers * records_per_writer) as usize,
            "Expected {} records, got {} — data loss detected!",
            num_writers * records_per_writer, all.len());
    }

    #[test]
    fn test_occ_mixed_read_write() {
        // T74.4: Concurrent reads and writes
        // Each writer writes its own non-overlapping set of IDs to avoid
        // "ambiguous merge insert" (multiple source rows with same id in one batch).
        use std::sync::Arc;
        use std::thread;

        let dir = tempdir().unwrap();
        let lance_path = dir.path().join("occ_mixed_rw.lance");
        let config = LanceConfig::new(&lance_path).with_embedding_dim(4);
        let backend = Arc::new(LanceStorageBackend::open(config).unwrap());

        // Seed data: 20 records
        for i in 0..20 {
            let mut node = MemoryNode::new(&format!("initial-{}", i));
            node.id = format!("rw-{}", i);
            backend.put(node).unwrap();
        }
        backend.flush().unwrap();

        // 5 writer threads + 5 reader threads
        let mut handles = vec![];

        // Each writer updates 4 unique records (writer_id*4 .. writer_id*4+3)
        for writer_id in 0..5u32 {
            let backend = Arc::clone(&backend);
            handles.push(thread::spawn(move || {
                let base = writer_id * 4;
                for i in 0..4 {
                    let mut node = MemoryNode::new(
                        &format!("mixed-w{}-{}", writer_id, i)
                    );
                    node.id = format!("rw-{}", base + i);
                    backend.put(node).unwrap();
                }
                backend.flush().unwrap();
            }));
        }

        for _reader_id in 0..5 {
            let backend = Arc::clone(&backend);
            handles.push(thread::spawn(move || {
                for _ in 0..10 {
                    let _ = backend.all();
                    let _ = backend.get("rw-0");
                    let _ = backend.len();
                }
            }));
        }

        for h in handles {
            h.join().expect("Thread panicked");
        }

        // Data should be consistent — still exactly 20 records
        let all = backend.all().unwrap();
        assert_eq!(all.len(), 20, "Record count should remain 20, got {}", all.len());
    }

    #[test]
    fn test_version_conflict_error_variant() {
        // T74.5: Verify VersionConflict error variant exists and formats correctly
        let err = StorageError::VersionConflict("test conflict message".to_string());
        let msg = format!("{}", err);
        assert!(msg.contains("Version conflict"));
        assert!(msg.contains("test conflict message"));
    }
}
