//! Storage backend trait abstraction
//!
//! Defines the `StorageBackend` trait that provides a unified interface
//! for different storage implementations (JSONL legacy, Lance columnar, etc.).
//!
//! This trait enables transparent switching between storage backends
//! without modifying upper-layer code (queries, triggers, semantic checks).

use std::io;

use serde::{Serialize, Deserialize};
use crate::types::{ContentType, MemoryNode, SemanticAxes};

/// Result type for storage operations
pub type StorageResult<T> = Result<T, StorageError>;

/// Errors that can occur during storage operations
#[derive(Debug, thiserror::Error)]
pub enum StorageError {
    /// I/O error
    #[error("I/O error: {0}")]
    Io(#[from] io::Error),

    /// Serialization / deserialization error
    #[error("Serialization error: {0}")]
    Serialization(String),

    /// Invalid ID format
    #[error("Invalid ID: {0}")]
    InvalidId(String),

    /// Record not found
    #[error("Record not found: {0}")]
    NotFound(String),

    /// Schema mismatch (e.g., embedding dimension changed)
    #[error("Schema mismatch: {0}")]
    SchemaMismatch(String),

    /// Backend-specific error
    #[error("Backend error: {0}")]
    Backend(String),

    /// Version conflict (OCC contention exhausted)
    #[error("Version conflict: {0}")]
    VersionConflict(String),
}

impl From<StorageError> for io::Error {
    fn from(e: StorageError) -> Self {
        match e {
            StorageError::Io(inner) => inner,
            other => io::Error::new(io::ErrorKind::Other, other.to_string()),
        }
    }
}

/// Sort direction for query results
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum SortOrder {
    /// Ascending order (default)
    #[default]
    Asc,
    /// Descending order
    Desc,
}

/// Sort field for query results
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SortField {
    /// Sort by created_at timestamp
    CreatedAt,
    /// Sort by last_accessed_at timestamp
    LastAccessedAt,
    /// Sort by computed importance
    Importance,
    /// Sort by access count
    AccessCount,
}

/// Metadata filter condition
#[derive(Debug, Clone)]
pub struct MetadataFilter {
    /// Metadata key to match
    pub key: String,
    /// Expected value (exact match)
    pub value: serde_json::Value,
}

// ============================================================================
// Nested Field Filter (JSON Path)
// ============================================================================

/// Comparison operator for nested field filtering.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FilterOp {
    /// Exact equality (==)
    Eq,
    /// Not equal (!=)
    Ne,
    /// Greater than (>)
    Gt,
    /// Greater than or equal (>=)
    Gte,
    /// Less than (<)
    Lt,
    /// Less than or equal (<=)
    Lte,
    /// Value is in the given list
    In,
    /// Value is NOT in the given list
    NotIn,
    /// String contains substring
    Contains,
    /// String starts with prefix
    StartsWith,
    /// String ends with prefix
    EndsWith,
    /// Value exists (is not null)
    Exists,
    /// Value does not exist (is null)
    NotExists,
}

/// Filter on a nested JSON field using a dot-path or JSON Path expression.
///
/// Supports accessing deeply nested fields within `metadata` or `user_metadata`.
///
/// # Path Syntax
///
/// Simple dot-notation:
/// - `"metadata.location.city"` → `metadata["location"]["city"]`
/// - `"metadata.scores[0]"` → first element of `metadata.scores` array
/// - `"user_metadata.preferences.theme"` → nested user metadata
///
/// JSON Path (subset):
/// - `"$.metadata.tags[*]"` → all elements in tags array
/// - `"$.metadata.nested.deep.key"` → deeply nested access
///
/// # Examples
///
/// ```ignore
/// // Filter: metadata.location.country == "China"
/// let filter = NestedFieldFilter::new("metadata.location.country", FilterOp::Eq, json!("China"));
///
/// // Filter: metadata.score > 0.8
/// let filter = NestedFieldFilter::new("metadata.score", FilterOp::Gt, json!(0.8));
///
/// // Filter: metadata.tags contains "important"
/// let filter = NestedFieldFilter::new("metadata.tags", FilterOp::Contains, json!("important"));
/// ```
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NestedFieldFilter {
    /// Dot-separated path to the nested field (e.g., "metadata.location.city")
    pub path: String,
    /// Comparison operator
    pub op: FilterOp,
    /// Value to compare against
    pub value: serde_json::Value,
}

impl NestedFieldFilter {
    /// Create a new nested field filter.
    pub fn new(path: impl Into<String>, op: FilterOp, value: serde_json::Value) -> Self {
        Self {
            path: path.into(),
            op,
            value,
        }
    }

    /// Convenience: create an equality filter.
    pub fn eq(path: impl Into<String>, value: serde_json::Value) -> Self {
        Self::new(path, FilterOp::Eq, value)
    }

    /// Convenience: create a greater-than filter.
    pub fn gt(path: impl Into<String>, value: serde_json::Value) -> Self {
        Self::new(path, FilterOp::Gt, value)
    }

    /// Convenience: create a less-than filter.
    pub fn lt(path: impl Into<String>, value: serde_json::Value) -> Self {
        Self::new(path, FilterOp::Lt, value)
    }

    /// Convenience: create a contains filter.
    pub fn contains(path: impl Into<String>, value: serde_json::Value) -> Self {
        Self::new(path, FilterOp::Contains, value)
    }

    /// Convenience: create an exists filter.
    pub fn exists(path: impl Into<String>) -> Self {
        Self::new(path, FilterOp::Exists, serde_json::Value::Null)
    }

    /// Evaluate this filter against a JSON document.
    ///
    /// The `root` is typically the full MemoryNode metadata or the entire
    /// record serialized as JSON.
    pub fn matches(&self, root: &serde_json::Value) -> bool {
        let resolved = resolve_json_path(root, &self.path);
        match &self.op {
            FilterOp::Exists => resolved.is_some(),
            FilterOp::NotExists => resolved.is_none(),
            _ => {
                if let Some(val) = resolved {
                    compare_json_values(val, &self.op, &self.value)
                } else {
                    false
                }
            }
        }
    }
}

/// Resolve a dot-separated path against a JSON value.
///
/// Supports:
/// - `"key.subkey.deep"` → object traversal
/// - `"key[0]"` → array index access
/// - `"$.key.subkey"` → JSON Path root prefix ($ is stripped)
pub fn resolve_json_path<'a>(root: &'a serde_json::Value, path: &str) -> Option<&'a serde_json::Value> {
    // Strip leading "$." if present
    let path = path.strip_prefix("$.").unwrap_or(path);

    let mut current = root;
    for segment in split_path_segments(path) {
        match segment {
            PathSegment::Key(key) => {
                current = current.get(key)?;
            }
            PathSegment::Index(idx) => {
                current = current.get(idx)?;
            }
        }
    }
    Some(current)
}

/// A segment in a JSON path.
enum PathSegment<'a> {
    Key(&'a str),
    Index(usize),
}

/// Split a dot-separated path into segments, handling array indices.
///
/// `"a.b[2].c"` → [Key("a"), Key("b"), Index(2), Key("c")]
fn split_path_segments(path: &str) -> Vec<PathSegment<'_>> {
    let mut segments = Vec::new();
    for part in path.split('.') {
        if let Some(bracket_pos) = part.find('[') {
            let key = &part[..bracket_pos];
            if !key.is_empty() {
                segments.push(PathSegment::Key(key));
            }
            // Extract index from [N]
            if let Some(end) = part.find(']') {
                if let Ok(idx) = part[bracket_pos + 1..end].parse::<usize>() {
                    segments.push(PathSegment::Index(idx));
                }
            }
        } else {
            segments.push(PathSegment::Key(part));
        }
    }
    segments
}

/// Compare a resolved JSON value against a target using the given operator.
fn compare_json_values(actual: &serde_json::Value, op: &FilterOp, target: &serde_json::Value) -> bool {
    use serde_json::Value;
    match op {
        FilterOp::Eq => actual == target,
        FilterOp::Ne => actual != target,
        FilterOp::Gt | FilterOp::Gte | FilterOp::Lt | FilterOp::Lte => {
            compare_ordered(actual, op, target)
        }
        FilterOp::In => {
            if let Value::Array(arr) = target {
                arr.contains(actual)
            } else {
                false
            }
        }
        FilterOp::NotIn => {
            if let Value::Array(arr) = target {
                !arr.contains(actual)
            } else {
                true
            }
        }
        FilterOp::Contains => {
            // Check string contains or array contains
            match (actual, target) {
                (Value::String(s), Value::String(sub)) => s.contains(sub.as_str()),
                (Value::Array(arr), _) => arr.contains(target),
                _ => false,
            }
        }
        FilterOp::StartsWith => {
            match (actual, target) {
                (Value::String(s), Value::String(prefix)) => s.starts_with(prefix.as_str()),
                _ => false,
            }
        }
        FilterOp::EndsWith => {
            match (actual, target) {
                (Value::String(s), Value::String(suffix)) => s.ends_with(suffix.as_str()),
                _ => false,
            }
        }
        FilterOp::Exists | FilterOp::NotExists => {
            // Handled in NestedFieldFilter::matches
            true
        }
    }
}

/// Ordered comparison for numeric and string types.
fn compare_ordered(actual: &serde_json::Value, op: &FilterOp, target: &serde_json::Value) -> bool {
    use serde_json::Value;
    // Try numeric comparison first
    if let (Some(a), Some(b)) = (actual.as_f64(), target.as_f64()) {
        return match op {
            FilterOp::Gt => a > b,
            FilterOp::Gte => a >= b,
            FilterOp::Lt => a < b,
            FilterOp::Lte => a <= b,
            _ => false,
        };
    }
    // Try string comparison
    if let (Value::String(a), Value::String(b)) = (actual, target) {
        return match op {
            FilterOp::Gt => a > b,
            FilterOp::Gte => a >= b,
            FilterOp::Lt => a < b,
            FilterOp::Lte => a <= b,
            _ => false,
        };
    }
    false
}

/// 语义门控过滤器（Semantic Gate Filter）
///
/// 基于 Nature 2016 语义图谱论文的三轴语义模型，
/// 在进行全库 ANN 搜索前，先按语义轴范围缩小候选集。
///
/// 每个轴的范围是可选的：
/// - `None` = 不过滤该轴
/// - `Some((min, max))` = 仅保留该轴值在 [min, max] 内的节点
///
/// 类似大脑皮层的"语义分区"——先定位区域，再搜索。
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct SemanticGateFilter {
    /// 语义轴 0 范围 (min, max)（默认对应 Social 社会性维度）
    pub axis_0_range: Option<(f32, f32)>,
    /// 语义轴 1 范围 (min, max)（默认对应 Sensory 感官性维度）
    pub axis_1_range: Option<(f32, f32)>,
    /// 语义轴 2 范围 (min, max)（默认对应 Locality 位置性维度）
    pub axis_2_range: Option<(f32, f32)>,
}

impl SemanticGateFilter {
    /// 创建单轴过滤
    pub fn axis_0(min: f32, max: f32) -> Self {
        Self { axis_0_range: Some((min, max)), ..Default::default() }
    }

    /// 创建双轴过滤
    pub fn axes_01(a0_min: f32, a0_max: f32, a1_min: f32, a1_max: f32) -> Self {
        Self {
            axis_0_range: Some((a0_min, a0_max)),
            axis_1_range: Some((a1_min, a1_max)),
            ..Default::default()
        }
    }

    /// 创建三轴过滤
    pub fn all_axes(
        a0: (f32, f32),
        a1: (f32, f32),
        a2: (f32, f32),
    ) -> Self {
        Self {
            axis_0_range: Some(a0),
            axis_1_range: Some(a1),
            axis_2_range: Some(a2),
        }
    }

    /// 检查给定 SemanticAxes 是否满足门控条件
    pub fn matches(&self, axes: &SemanticAxes) -> bool {
        axes.in_range(self.axis_0_range, self.axis_1_range, self.axis_2_range)
    }

    /// 是否有任何轴设置了过滤
    pub fn is_active(&self) -> bool {
        self.axis_0_range.is_some() || self.axis_1_range.is_some() || self.axis_2_range.is_some()
    }
}

/// Query filter for retrieving memory nodes
#[derive(Debug, Clone, Default)]
pub struct QueryFilter {
    /// Filter by tags (any match)
    pub tags: Option<Vec<String>>,

    /// Filter by content type
    pub content_type: Option<ContentType>,

    /// Filter by parent ID
    pub parent_id: Option<String>,

    /// Minimum importance threshold (uses computed_importance)
    pub min_importance: Option<f64>,

    /// Time range filter on created_at (start_ms, end_ms)
    pub time_range: Option<(i64, i64)>,

    /// Metadata key-value filters (all must match)
    pub metadata: Option<Vec<MetadataFilter>>,

    /// Nested field filters using JSON Path (all must match)
    ///
    /// Allows filtering on deeply nested metadata fields.
    /// Example: `metadata.location.country == "China"`
    pub nested_filters: Option<Vec<NestedFieldFilter>>,

    /// 语义门控过滤（BriLLM SiFu 启发）
    ///
    /// 按语义轴范围预过滤候选集，类似大脑皮层的"语义分区定位"。
    /// Lance 后端可将此下推为 WHERE 子句（sem_axis_0 BETWEEN ... AND ...）。
    pub semantic_gate: Option<SemanticGateFilter>,

    /// Sort field
    pub sort_by: Option<SortField>,

    /// Sort order (default: ascending)
    pub sort_order: SortOrder,

    /// Number of results to skip (for pagination)
    pub offset: usize,

    /// Maximum number of results (0 = no limit)
    pub limit: usize,
}

/// Unified storage backend trait.
///
/// All storage implementations (JSONL legacy, Lance columnar, etc.)
/// must implement this trait. Upper-layer code (HybridQueryEngine,
/// TriggerRegistry, SemanticChecker, Collection, Federation, etc.)
/// interacts only through this interface.
///
/// # Design Notes
///
/// - Methods are synchronous to match the existing `DocumentStore` API.
///   Lance async operations are internally block_on'd in `LanceStorageBackend`.
/// - `put` / `delete` / `flush` take `&self` (interior mutability) to stay
///   compatible with the existing `RwLock`-based `DocumentStore`.
/// - All query/listing methods have default implementations that work
///   through the in-memory `all()` method. Backends may override for
///   optimized push-down queries.
pub trait StorageBackend: Send + Sync {
    // ====================================================================
    // CRUD Operations
    // ====================================================================

    /// Insert or update a memory node.
    ///
    /// The implementation must validate the node ID before storing.
    fn put(&self, node: MemoryNode) -> StorageResult<()>;

    /// Retrieve a memory node by ID.
    fn get(&self, id: &str) -> StorageResult<Option<MemoryNode>>;

    /// Delete a memory node by ID, returning the removed node if it existed.
    fn delete(&self, id: &str) -> StorageResult<Option<MemoryNode>>;

    /// Check if a memory node with the given ID exists.
    fn contains(&self, id: &str) -> StorageResult<bool>;

    // ====================================================================
    // Bulk Operations
    // ====================================================================

    /// Insert or update a batch of memory nodes.
    ///
    /// Default implementation calls `put` in a loop; backends may override
    /// for batch-optimized writes (e.g., Lance batch append).
    fn put_batch(&self, nodes: Vec<MemoryNode>) -> StorageResult<()> {
        for node in nodes {
            self.put(node)?;
        }
        Ok(())
    }

    /// Retrieve multiple memory nodes by IDs.
    ///
    /// Default implementation calls `get` in a loop; backends may override
    /// for batch-optimized reads.
    fn get_batch(&self, ids: &[&str]) -> StorageResult<Vec<Option<MemoryNode>>> {
        ids.iter().map(|id| self.get(id)).collect()
    }

    // ====================================================================
    // Listing & Counting
    // ====================================================================

    /// Return all node IDs in the store.
    fn ids(&self) -> StorageResult<Vec<String>>;

    /// Return the total number of stored nodes.
    fn len(&self) -> StorageResult<usize>;

    /// Check if the store is empty.
    fn is_empty(&self) -> StorageResult<bool> {
        Ok(self.len()? == 0)
    }

    /// Retrieve all memory nodes.
    ///
    /// ⚠️ Use with caution on large datasets. Prefer `query` with filters.
    fn all(&self) -> StorageResult<Vec<MemoryNode>>;

    // ====================================================================
    // Query Operations
    // ====================================================================

    /// Query memory nodes with filters, sorting, and pagination.
    ///
    /// Default implementation fetches all and filters in memory;
    /// backends may override with push-down predicates.
    fn query(&self, filter: &QueryFilter) -> StorageResult<Vec<MemoryNode>> {
        let mut results = self.all()?;

        // Apply tag filter
        if let Some(ref tags) = filter.tags {
            results.retain(|node| {
                tags.iter().any(|tag| node.tags.contains(tag))
            });
        }

        // Apply content type filter
        if let Some(content_type) = filter.content_type {
            results.retain(|node| node.content_type == content_type);
        }

        // Apply parent ID filter
        if let Some(ref parent_id) = filter.parent_id {
            results.retain(|node| node.parent_id.as_deref() == Some(parent_id.as_str()));
        }

        // Apply importance filter
        if let Some(min_importance) = filter.min_importance {
            results.retain(|node| node.computed_importance() >= min_importance);
        }

        // Apply time range filter
        if let Some((start, end)) = filter.time_range {
            results.retain(|node| node.created_at >= start && node.created_at <= end);
        }

        // Apply metadata filters
        if let Some(ref filters) = filter.metadata {
            results.retain(|node| {
                filters.iter().all(|mf| node.metadata.get(&mf.key) == Some(&mf.value))
            });
        }

        // Apply semantic gate filter (BriLLM SiFu inspired)
        if let Some(ref gate) = filter.semantic_gate {
            if gate.is_active() {
                results.retain(|node| {
                    match &node.semantic_axes {
                        Some(axes) => gate.matches(axes),
                        None => false, // 无语义轴的节点被排除
                    }
                });
            }
        }

        // Apply sorting
        if let Some(sort_by) = filter.sort_by {
            match sort_by {
                SortField::CreatedAt => {
                    results.sort_by(|a, b| a.created_at.cmp(&b.created_at));
                }
                SortField::LastAccessedAt => {
                    results.sort_by(|a, b| a.last_accessed_at.cmp(&b.last_accessed_at));
                }
                SortField::Importance => {
                    results.sort_by(|a, b| {
                        a.computed_importance()
                            .partial_cmp(&b.computed_importance())
                            .unwrap_or(std::cmp::Ordering::Equal)
                    });
                }
                SortField::AccessCount => {
                    results.sort_by(|a, b| a.access_count.cmp(&b.access_count));
                }
            }
            if filter.sort_order == SortOrder::Desc {
                results.reverse();
            }
        }

        // Apply offset (pagination)
        if filter.offset > 0 {
            if filter.offset >= results.len() {
                return Ok(Vec::new());
            }
            results = results.split_off(filter.offset);
        }

        // Apply limit
        if filter.limit > 0 && results.len() > filter.limit {
            results.truncate(filter.limit);
        }

        Ok(results)
    }

    /// Find nodes by tag (convenience wrapper around `query`).
    fn find_by_tag(&self, tag: &str) -> StorageResult<Vec<MemoryNode>> {
        self.query(&QueryFilter {
            tags: Some(vec![tag.to_string()]),
            ..Default::default()
        })
    }

    /// Find nodes by content type (convenience wrapper around `query`).
    fn find_by_content_type(&self, content_type: ContentType) -> StorageResult<Vec<MemoryNode>> {
        self.query(&QueryFilter {
            content_type: Some(content_type),
            ..Default::default()
        })
    }

    /// Find nodes by parent ID (convenience wrapper around `query`).
    fn find_by_parent(&self, parent_id: &str) -> StorageResult<Vec<MemoryNode>> {
        self.query(&QueryFilter {
            parent_id: Some(parent_id.to_string()),
            ..Default::default()
        })
    }

    /// Find nodes by time range on `created_at` (inclusive).
    fn find_by_time_range(&self, start: i64, end: i64) -> StorageResult<Vec<MemoryNode>> {
        self.query(&QueryFilter {
            time_range: Some((start, end)),
            ..Default::default()
        })
    }

    /// Find nodes with computed importance >= threshold.
    fn find_by_min_importance(&self, min_importance: f64) -> StorageResult<Vec<MemoryNode>> {
        self.query(&QueryFilter {
            min_importance: Some(min_importance),
            ..Default::default()
        })
    }

    /// Find nodes by metadata key-value pair (exact match).
    fn find_by_metadata(&self, key: &str, value: &serde_json::Value) -> StorageResult<Vec<MemoryNode>> {
        self.query(&QueryFilter {
            metadata: Some(vec![MetadataFilter {
                key: key.to_string(),
                value: value.clone(),
            }]),
            ..Default::default()
        })
    }

    /// 按语义门控过滤节点（Semantic Gating）
    ///
    /// 根据语义轴范围缩小候选集。等价于人脑"先定位皮层区域，再搜索"。
    /// Lance 后端可优化为下推 WHERE 子句。
    fn find_by_semantic_gate(&self, gate: &SemanticGateFilter) -> StorageResult<Vec<MemoryNode>> {
        self.query(&QueryFilter {
            semantic_gate: Some(gate.clone()),
            ..Default::default()
        })
    }

    // ====================================================================
    // Advanced Query with FilterExpr AST
    // ====================================================================

    /// Query memory nodes using the new `FilterExpr` AST tree.
    ///
    /// Supports arbitrary AND/OR/NOT nesting, IN, BETWEEN, LIKE, etc.
    /// Default implementation evaluates in-memory; Lance backend overrides
    /// with columnar predicate push-down for much better performance.
    ///
    /// The `sort_by`, `sort_order`, `offset`, and `limit` parameters from
    /// the legacy `QueryFilter` are passed separately.
    fn query_expr(
        &self,
        expr: &crate::storage::filter_expr::FilterExpr,
        sort_by: Option<SortField>,
        sort_order: SortOrder,
        offset: usize,
        limit: usize,
    ) -> StorageResult<Vec<MemoryNode>> {
        use crate::storage::filter_expr::FilterExpr;

        let mut results: Vec<MemoryNode> = if expr.is_always_true() {
            self.all()?
        } else {
            self.all()?
                .into_iter()
                .filter(|node| expr.evaluate(node))
                .collect()
        };

        // Apply sorting
        if let Some(sort) = sort_by {
            match sort {
                SortField::CreatedAt => results.sort_by_key(|n| n.created_at),
                SortField::LastAccessedAt => results.sort_by_key(|n| n.last_accessed_at),
                SortField::Importance => results.sort_by(|a, b| {
                    a.computed_importance()
                        .partial_cmp(&b.computed_importance())
                        .unwrap_or(std::cmp::Ordering::Equal)
                }),
                SortField::AccessCount => results.sort_by_key(|n| n.access_count),
            }
            if sort_order == SortOrder::Desc {
                results.reverse();
            }
        }

        // Apply pagination
        if offset > 0 {
            if offset >= results.len() {
                return Ok(Vec::new());
            }
            results = results.split_off(offset);
        }
        if limit > 0 && results.len() > limit {
            results.truncate(limit);
        }

        Ok(results)
    }

    // ====================================================================
    // Aggregation Queries
    // ====================================================================

    /// Return all unique tags across all nodes.
    fn all_tags(&self) -> StorageResult<Vec<String>> {
        let nodes = self.all()?;
        let mut tags = std::collections::HashSet::new();
        for node in &nodes {
            for tag in &node.tags {
                tags.insert(tag.clone());
            }
        }
        let mut sorted: Vec<String> = tags.into_iter().collect();
        sorted.sort();
        Ok(sorted)
    }

    /// Count nodes that have the given tag.
    fn count_by_tag(&self, tag: &str) -> StorageResult<usize> {
        Ok(self.find_by_tag(tag)?.len())
    }

    /// Count nodes with the given content type.
    fn count_by_type(&self, content_type: ContentType) -> StorageResult<usize> {
        Ok(self.find_by_content_type(content_type)?.len())
    }

    // ====================================================================
    // Access Tracking
    // ====================================================================

    /// Record an access to the specified node (updates last_accessed_at and access_count).
    fn record_access(&self, id: &str) -> StorageResult<()>;

    // ====================================================================
    // Upsert & Conditional Delete
    // ====================================================================

    /// Upsert (merge-insert): if a node with the same ID exists, update it;
    /// otherwise insert it.
    ///
    /// Returns `true` if an existing node was updated, `false` if a new node
    /// was inserted.
    ///
    /// Default implementation: get → put.
    fn upsert(&self, node: MemoryNode) -> StorageResult<bool> {
        let exists = self.contains(&node.id)?;
        self.put(node)?;
        Ok(exists)
    }

    /// Upsert a batch of nodes.
    ///
    /// Returns the number of nodes that were updated (vs newly inserted).
    fn upsert_batch(&self, nodes: Vec<MemoryNode>) -> StorageResult<usize> {
        let mut updated = 0;
        for node in nodes {
            if self.upsert(node)? {
                updated += 1;
            }
        }
        Ok(updated)
    }

    /// Delete all nodes matching a filter expression.
    ///
    /// Returns the list of deleted node IDs.
    ///
    /// Default implementation: scan all → evaluate filter → delete matches.
    fn delete_where(&self, filter: &super::FilterExpr) -> StorageResult<Vec<String>> {
        let all_nodes = self.all()?;
        let mut deleted_ids = Vec::new();
        for node in &all_nodes {
            if filter.evaluate(node) {
                self.delete(&node.id)?;
                deleted_ids.push(node.id.clone());
            }
        }
        Ok(deleted_ids)
    }

    /// Count nodes matching a filter expression (without loading full data).
    ///
    /// Default implementation: scan all → evaluate filter → count.
    fn count_where(&self, filter: &super::FilterExpr) -> StorageResult<usize> {
        let all_nodes = self.all()?;
        Ok(all_nodes.iter().filter(|n| filter.evaluate(n)).count())
    }

    // ====================================================================
    // Persistence
    // ====================================================================

    /// Flush any pending writes to durable storage.
    fn flush(&self) -> StorageResult<()>;

    // ====================================================================
    // Backend Info
    // ====================================================================

    /// Return the backend type name (e.g., "lance", "jsonl", "enhanced-jsonl").
    fn backend_name(&self) -> &str;

    // ====================================================================
    // Full-Text Search (optional, override to enable)
    // ====================================================================

    /// Check if this backend supports full-text search
    fn supports_fts(&self) -> bool {
        false
    }

    /// Execute a full-text search and return matching document IDs with scores.
    ///
    /// Returns `(id, bm25_score)` pairs sorted by relevance.
    /// Default implementation falls back to simple `content.contains()` matching.
    fn full_text_search(
        &self,
        query_text: &str,
        limit: usize,
    ) -> StorageResult<Vec<(String, f64)>> {
        // Fallback: simple keyword matching (same as old HybridQueryEngine behavior)
        let keywords: Vec<String> = query_text.to_lowercase()
            .split_whitespace()
            .map(|s| s.to_string())
            .collect();

        if keywords.is_empty() {
            return Ok(Vec::new());
        }

        let all_nodes = self.all()?;
        let mut results: Vec<(String, f64)> = Vec::new();

        for node in &all_nodes {
            let content_lower = node.content.to_lowercase();
            let matches: usize = keywords.iter()
                .filter(|kw| content_lower.contains(kw.as_str()))
                .count();

            if matches > 0 {
                let score = matches as f64 / keywords.len() as f64;
                results.push((node.id.clone(), score));
            }
        }

        results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
        results.truncate(limit);
        Ok(results)
    }

    // ====================================================================
    // OCC (Optimistic Concurrency Control) Stats — Phase-74
    // ====================================================================

    /// Return OCC metrics as a JSON value, if supported by this backend.
    ///
    /// Only `LanceStorageBackend` returns `Some(...)` — all other backends
    /// return `None` by default.
    fn occ_stats_json(&self) -> Option<serde_json::Value> {
        None
    }
}

// ============================================================================
// Tests for Nested Field Filter
// ============================================================================

#[cfg(test)]
mod nested_filter_tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn test_resolve_simple_path() {
        let doc = json!({
            "name": "Alice",
            "age": 30
        });
        assert_eq!(resolve_json_path(&doc, "name"), Some(&json!("Alice")));
        assert_eq!(resolve_json_path(&doc, "age"), Some(&json!(30)));
        assert_eq!(resolve_json_path(&doc, "missing"), None);
    }

    #[test]
    fn test_resolve_nested_path() {
        let doc = json!({
            "metadata": {
                "location": {
                    "country": "China",
                    "city": "Beijing"
                },
                "scores": [95, 87, 72]
            }
        });
        assert_eq!(
            resolve_json_path(&doc, "metadata.location.country"),
            Some(&json!("China"))
        );
        assert_eq!(
            resolve_json_path(&doc, "metadata.location.city"),
            Some(&json!("Beijing"))
        );
        assert_eq!(
            resolve_json_path(&doc, "metadata.scores[0]"),
            Some(&json!(95))
        );
        assert_eq!(
            resolve_json_path(&doc, "metadata.scores[2]"),
            Some(&json!(72))
        );
        assert_eq!(
            resolve_json_path(&doc, "metadata.scores[5]"),
            None // Out of bounds
        );
    }

    #[test]
    fn test_resolve_json_path_prefix() {
        let doc = json!({"a": {"b": 42}});
        assert_eq!(resolve_json_path(&doc, "$.a.b"), Some(&json!(42)));
    }

    #[test]
    fn test_filter_eq() {
        let doc = json!({"metadata": {"status": "active"}});
        let f = NestedFieldFilter::eq("metadata.status", json!("active"));
        assert!(f.matches(&doc));

        let f = NestedFieldFilter::eq("metadata.status", json!("inactive"));
        assert!(!f.matches(&doc));
    }

    #[test]
    fn test_filter_ne() {
        let doc = json!({"metadata": {"count": 5}});
        let f = NestedFieldFilter::new("metadata.count", FilterOp::Ne, json!(3));
        assert!(f.matches(&doc));

        let f = NestedFieldFilter::new("metadata.count", FilterOp::Ne, json!(5));
        assert!(!f.matches(&doc));
    }

    #[test]
    fn test_filter_gt_lt() {
        let doc = json!({"metadata": {"score": 0.85}});

        assert!(NestedFieldFilter::gt("metadata.score", json!(0.5)).matches(&doc));
        assert!(!NestedFieldFilter::gt("metadata.score", json!(0.9)).matches(&doc));
        assert!(NestedFieldFilter::lt("metadata.score", json!(0.9)).matches(&doc));
        assert!(!NestedFieldFilter::lt("metadata.score", json!(0.5)).matches(&doc));
    }

    #[test]
    fn test_filter_gte_lte() {
        let doc = json!({"val": 10});
        let gte = NestedFieldFilter::new("val", FilterOp::Gte, json!(10));
        assert!(gte.matches(&doc));
        let lte = NestedFieldFilter::new("val", FilterOp::Lte, json!(10));
        assert!(lte.matches(&doc));
    }

    #[test]
    fn test_filter_in_not_in() {
        let doc = json!({"status": "pending"});

        let f_in = NestedFieldFilter::new(
            "status", FilterOp::In, json!(["pending", "active", "done"])
        );
        assert!(f_in.matches(&doc));

        let f_not_in = NestedFieldFilter::new(
            "status", FilterOp::NotIn, json!(["deleted", "archived"])
        );
        assert!(f_not_in.matches(&doc));
    }

    #[test]
    fn test_filter_contains() {
        // String contains
        let doc = json!({"text": "Hello World"});
        assert!(NestedFieldFilter::contains("text", json!("World")).matches(&doc));
        assert!(!NestedFieldFilter::contains("text", json!("Goodbye")).matches(&doc));

        // Array contains
        let doc = json!({"tags": ["rust", "ai", "db"]});
        assert!(NestedFieldFilter::contains("tags", json!("ai")).matches(&doc));
        assert!(!NestedFieldFilter::contains("tags", json!("python")).matches(&doc));
    }

    #[test]
    fn test_filter_starts_ends_with() {
        let doc = json!({"path": "/api/v1/collections"});

        let sw = NestedFieldFilter::new("path", FilterOp::StartsWith, json!("/api"));
        assert!(sw.matches(&doc));

        let ew = NestedFieldFilter::new("path", FilterOp::EndsWith, json!("collections"));
        assert!(ew.matches(&doc));
    }

    #[test]
    fn test_filter_exists_not_exists() {
        let doc = json!({"metadata": {"key": "value"}});

        assert!(NestedFieldFilter::exists("metadata.key").matches(&doc));
        assert!(!NestedFieldFilter::exists("metadata.missing").matches(&doc));

        let ne = NestedFieldFilter::new("metadata.missing", FilterOp::NotExists, json!(null));
        assert!(ne.matches(&doc));
    }

    #[test]
    fn test_filter_deeply_nested() {
        let doc = json!({
            "a": {
                "b": {
                    "c": {
                        "d": {
                            "e": 42
                        }
                    }
                }
            }
        });
        assert!(NestedFieldFilter::eq("a.b.c.d.e", json!(42)).matches(&doc));
    }

    #[test]
    fn test_filter_string_comparison() {
        let doc = json!({"name": "Charlie"});
        assert!(NestedFieldFilter::gt("name", json!("Bob")).matches(&doc));
        assert!(NestedFieldFilter::lt("name", json!("Delta")).matches(&doc));
    }

    #[test]
    fn test_filter_on_missing_path() {
        let doc = json!({"a": 1});
        // All comparison filters should return false for missing paths
        assert!(!NestedFieldFilter::eq("missing.path", json!(1)).matches(&doc));
        assert!(!NestedFieldFilter::gt("missing.path", json!(0)).matches(&doc));
        assert!(!NestedFieldFilter::contains("missing", json!("x")).matches(&doc));
    }
}
