//! Lance multimodal storage backend implementation
//!
//! Implements `MultimodalBackend` using Lance datasets for persistent storage
//! of binary media data (images, audio, video) alongside text and embeddings.
//!
//! # Schema
//!
//! Each multimodal collection uses a Lance table with the following columns:
//!
//! | Column             | Arrow Type          | Description                    |
//! |--------------------|---------------------|--------------------------------|
//! | id                 | Utf8                | Primary key                    |
//! | media_type         | Utf8                | "image"/"audio"/"video"        |
//! | mime_type          | Utf8                | MIME type                      |
//! | data               | LargeBinary         | Raw media bytes                |
//! | thumbnail          | Binary              | Preview image (≤64KB)          |
//! | text_content       | Utf8                | Caption/transcript             |
//! | metadata_json      | Utf8                | MediaMetadata as JSON          |
//! | tags_json          | Utf8                | Tags as JSON array             |
//! | user_metadata_json | Utf8                | User metadata as JSON          |
//! | embeddings_json    | Utf8                | Per-modality embeddings as JSON|
//! | created_at         | Int64               | Unix milliseconds              |
//! | last_accessed_at   | Int64               | Unix milliseconds              |
//! | file_size_bytes    | UInt64              | File size for quick query      |

use std::path::PathBuf;
use std::sync::Arc;
use std::collections::HashMap;

use arrow_array::{
    Array, RecordBatch, RecordBatchIterator,
    StringArray, LargeBinaryArray, BinaryArray,
    Int64Array, UInt64Array, Float32Array, FixedSizeListArray,
};
use arrow_schema::{Schema, Field, DataType};
use lance::Dataset;
use lance::dataset::builder::DatasetBuilder;
use tokio::runtime::Runtime;

use crate::types::multimodal::{
    MultimodalRecord, MediaType, MediaMetadata, MultimodalBackend, CrossModalSearchResult,
    MultimodalVectorConfig, VectorColumnDef, MultimodalIndexInfo,
};
use crate::storage::backend::{StorageError, StorageResult};
use crate::storage::vector_backend::VectorIndexConfig;

// ============================================================================
// Arrow Schema for Multimodal Records
// ============================================================================

/// Create the base Arrow schema for the multimodal Lance table (without vector columns).
pub fn multimodal_base_arrow_schema() -> Vec<Field> {
    vec![
        Field::new("id", DataType::Utf8, false),
        Field::new("media_type", DataType::Utf8, false),
        Field::new("mime_type", DataType::Utf8, false),
        Field::new("data", DataType::LargeBinary, true),
        Field::new("thumbnail", DataType::Binary, true),
        Field::new("text_content", DataType::Utf8, true),
        Field::new("metadata_json", DataType::Utf8, false),
        Field::new("tags_json", DataType::Utf8, false),
        Field::new("user_metadata_json", DataType::Utf8, false),
        Field::new("embeddings_json", DataType::Utf8, false),
        Field::new("created_at", DataType::Int64, false),
        Field::new("last_accessed_at", DataType::Int64, false),
        Field::new("file_size_bytes", DataType::UInt64, false),
    ]
}

/// Create the Arrow schema for the multimodal Lance table (legacy, no native vector columns).
pub fn multimodal_arrow_schema() -> Arc<Schema> {
    Arc::new(Schema::new(multimodal_base_arrow_schema()))
}

/// Create the Arrow schema with additional native vector columns.
///
/// Appends FixedSizeList(Float32, dim) columns for each configured embedding.
/// This enables Lance ANN indexing on those columns.
pub fn multimodal_arrow_schema_with_vectors(vec_config: &MultimodalVectorConfig) -> Arc<Schema> {
    let mut fields = multimodal_base_arrow_schema();
    // Sort by column name for deterministic ordering
    let mut entries: Vec<_> = vec_config.columns.iter().collect();
    entries.sort_by_key(|(_, def)| &def.column_name);
    for (_, def) in entries {
        fields.push(Field::new(
            &def.column_name,
            DataType::FixedSizeList(
                Arc::new(Field::new("item", DataType::Float32, true)),
                def.dimension as i32,
            ),
            true, // nullable — not all records have all embeddings
        ));
    }
    Arc::new(Schema::new(fields))
}

// ============================================================================
// Convert MultimodalRecord ↔ RecordBatch
// ============================================================================

/// Convert a slice of `MultimodalRecord`s into an Arrow RecordBatch.
///
/// If `vec_config` is provided, also writes native vector columns alongside JSON.
pub fn multimodal_to_record_batch(
    records: &[MultimodalRecord],
    schema: &Arc<Schema>,
    vec_config: Option<&MultimodalVectorConfig>,
) -> Result<RecordBatch, StorageError> {
    let ids: Vec<&str> = records.iter().map(|r| r.id.as_str()).collect();
    let media_types: Vec<&str> = records.iter().map(|r| match r.media_type {
        MediaType::Image => "image",
        MediaType::Audio => "audio",
        MediaType::Video => "video",
    }).collect();
    let mime_types: Vec<&str> = records.iter().map(|r| r.mime_type.as_str()).collect();

    // Binary data (LargeBinary)
    let data_values: Vec<Option<&[u8]>> = records.iter().map(|r| {
        r.data.as_ref().map(|d| d.as_slice())
    }).collect();

    // Thumbnails (Binary)
    let thumb_values: Vec<Option<&[u8]>> = records.iter().map(|r| {
        r.thumbnail.as_ref().map(|t| t.as_slice())
    }).collect();

    // Text content
    let text_values: Vec<Option<&str>> = records.iter().map(|r| {
        r.text_content.as_deref()
    }).collect();

    // JSON columns
    let metadata_jsons: Vec<String> = records.iter().map(|r| {
        serde_json::to_string(&r.metadata).unwrap_or_default()
    }).collect();
    let tags_jsons: Vec<String> = records.iter().map(|r| {
        serde_json::to_string(&r.tags).unwrap_or_default()
    }).collect();
    let user_meta_jsons: Vec<String> = records.iter().map(|r| {
        serde_json::to_string(&r.user_metadata).unwrap_or_default()
    }).collect();
    let embed_jsons: Vec<String> = records.iter().map(|r| {
        serde_json::to_string(&r.embeddings).unwrap_or_default()
    }).collect();

    // Timestamps
    let created_ats: Vec<i64> = records.iter().map(|r| r.created_at).collect();
    let last_accessed: Vec<i64> = records.iter().map(|r| r.last_accessed_at).collect();
    let file_sizes: Vec<u64> = records.iter().map(|r| r.file_size()).collect();

    let mut columns: Vec<Arc<dyn Array>> = vec![
        Arc::new(StringArray::from(ids)),
        Arc::new(StringArray::from(media_types)),
        Arc::new(StringArray::from(mime_types)),
        Arc::new(LargeBinaryArray::from_opt_vec(data_values)),
        Arc::new(BinaryArray::from_opt_vec(thumb_values)),
        Arc::new(StringArray::from(text_values)),
        Arc::new(StringArray::from(metadata_jsons.iter().map(|s| s.as_str()).collect::<Vec<_>>())),
        Arc::new(StringArray::from(tags_jsons.iter().map(|s| s.as_str()).collect::<Vec<_>>())),
        Arc::new(StringArray::from(user_meta_jsons.iter().map(|s| s.as_str()).collect::<Vec<_>>())),
        Arc::new(StringArray::from(embed_jsons.iter().map(|s| s.as_str()).collect::<Vec<_>>())),
        Arc::new(Int64Array::from(created_ats)),
        Arc::new(Int64Array::from(last_accessed)),
        Arc::new(UInt64Array::from(file_sizes)),
    ];

    // Append native vector columns (Phase-60: dual-write)
    if let Some(vc) = vec_config {
        // Maintain deterministic column ordering (sorted by column_name)
        let mut entries: Vec<_> = vc.columns.iter().collect();
        entries.sort_by_key(|(_, def)| &def.column_name);

        for (embedding_name, def) in entries {
            let dim = def.dimension as i32;

            // Flatten all vectors into a single Float32Array, with nulls for missing
            let mut flat_values: Vec<f32> = Vec::with_capacity(records.len() * def.dimension);
            let mut validity: Vec<bool> = Vec::with_capacity(records.len());

            for record in records {
                if let Some(vec) = record.embeddings.get(embedding_name) {
                    if vec.len() == def.dimension {
                        flat_values.extend_from_slice(vec);
                        validity.push(true);
                    } else {
                        // Dimension mismatch — treat as null
                        flat_values.extend(std::iter::repeat(0.0f32).take(def.dimension));
                        validity.push(false);
                    }
                } else {
                    // No embedding for this modality — null
                    flat_values.extend(std::iter::repeat(0.0f32).take(def.dimension));
                    validity.push(false);
                }
            }

            let values_array = Float32Array::from(flat_values);
            let field = Arc::new(Field::new("item", DataType::Float32, true));
            let list_array = FixedSizeListArray::try_new(
                field, dim, Arc::new(values_array),
                Some(validity.into()),
            ).map_err(|e| StorageError::Serialization(
                format!("Failed to create vector column '{}': {}", def.column_name, e),
            ))?;

            columns.push(Arc::new(list_array));
        }
    }

    let batch = RecordBatch::try_new(schema.clone(), columns)
        .map_err(|e| StorageError::Serialization(format!("Arrow batch creation failed: {}", e)))?;

    Ok(batch)
}

/// Convert a RecordBatch row to a `MultimodalRecord`.
fn record_batch_row_to_multimodal(
    batch: &RecordBatch,
    row_idx: usize,
    include_data: bool,
) -> Result<MultimodalRecord, StorageError> {
    let id_col = batch.column(0).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("id column type mismatch".into()))?;
    let media_type_col = batch.column(1).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("media_type column type mismatch".into()))?;
    let mime_type_col = batch.column(2).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("mime_type column type mismatch".into()))?;
    let text_col = batch.column(5).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("text_content column type mismatch".into()))?;
    let metadata_col = batch.column(6).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("metadata column type mismatch".into()))?;
    let tags_col = batch.column(7).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("tags column type mismatch".into()))?;
    let user_meta_col = batch.column(8).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("user_metadata column type mismatch".into()))?;
    let embed_col = batch.column(9).as_any().downcast_ref::<StringArray>()
        .ok_or_else(|| StorageError::Serialization("embeddings column type mismatch".into()))?;
    let created_col = batch.column(10).as_any().downcast_ref::<Int64Array>()
        .ok_or_else(|| StorageError::Serialization("created_at column type mismatch".into()))?;
    let accessed_col = batch.column(11).as_any().downcast_ref::<Int64Array>()
        .ok_or_else(|| StorageError::Serialization("last_accessed_at column type mismatch".into()))?;

    let media_type_str = media_type_col.value(row_idx);
    let media_type = match media_type_str {
        "image" => MediaType::Image,
        "audio" => MediaType::Audio,
        "video" => MediaType::Video,
        other => return Err(StorageError::Serialization(
            format!("Unknown media_type: {}", other),
        )),
    };

    let data = if include_data {
        let data_col = batch.column(3).as_any().downcast_ref::<LargeBinaryArray>()
            .ok_or_else(|| StorageError::Serialization("data column type mismatch".into()))?;
        if data_col.is_null(row_idx) {
            None
        } else {
            Some(data_col.value(row_idx).to_vec())
        }
    } else {
        None
    };

    let thumbnail = {
        let thumb_col = batch.column(4).as_any().downcast_ref::<BinaryArray>()
            .ok_or_else(|| StorageError::Serialization("thumbnail column type mismatch".into()))?;
        if thumb_col.is_null(row_idx) {
            None
        } else {
            Some(thumb_col.value(row_idx).to_vec())
        }
    };

    let text_content = if text_col.is_null(row_idx) {
        None
    } else {
        Some(text_col.value(row_idx).to_string())
    };

    let metadata: MediaMetadata = serde_json::from_str(metadata_col.value(row_idx))
        .unwrap_or_default();
    let tags: Vec<String> = serde_json::from_str(tags_col.value(row_idx))
        .unwrap_or_default();
    let user_metadata: std::collections::HashMap<String, serde_json::Value> =
        serde_json::from_str(user_meta_col.value(row_idx)).unwrap_or_default();
    let embeddings: std::collections::HashMap<String, Vec<f32>> =
        serde_json::from_str(embed_col.value(row_idx)).unwrap_or_default();

    Ok(MultimodalRecord {
        id: id_col.value(row_idx).to_string(),
        media_type,
        mime_type: mime_type_col.value(row_idx).to_string(),
        data,
        thumbnail,
        text_content,
        metadata,
        tags,
        user_metadata: user_metadata,
        embeddings,
        created_at: created_col.value(row_idx),
        last_accessed_at: accessed_col.value(row_idx),
    })
}

// ============================================================================
// LanceMultimodalBackend
// ============================================================================

/// Lance-based storage backend for multimodal data.
///
/// Stores images, audio, and video as LargeBinary columns in a Lance dataset.
/// Each collection can have a separate multimodal table alongside the main
/// text/embedding table.
///
/// # Native Vector Columns (Phase-60)
///
/// When constructed with `MultimodalVectorConfig`, the backend also writes
/// native Lance FixedSizeList(Float32, dim) columns for specified embeddings.
/// This enables ANN indexing and fast search via `scanner.nearest()`.
pub struct LanceMultimodalBackend {
    /// Path to the Lance dataset for multimodal data
    uri: String,
    /// Arrow schema (includes vector columns if vec_config is set)
    schema: Arc<Schema>,
    /// Tokio runtime for async Lance operations
    rt: Arc<Runtime>,
    /// Cached dataset handle
    dataset: parking_lot::RwLock<Option<Arc<Dataset>>>,
    /// Native vector column configuration (Phase-60)
    vec_config: Option<MultimodalVectorConfig>,
}

impl LanceMultimodalBackend {
    /// Create a new multimodal backend (legacy mode, no native vector columns).
    ///
    /// `uri` is the path to the Lance dataset (e.g., "./data/collections/my_col/multimodal.lance")
    pub fn new(uri: impl Into<String>) -> StorageResult<Self> {
        let rt = Runtime::new()
            .map_err(|e| StorageError::Backend(format!("Failed to create runtime: {}", e)))?;

        Ok(Self {
            uri: uri.into(),
            schema: multimodal_arrow_schema(),
            rt: Arc::new(rt),
            dataset: parking_lot::RwLock::new(None),
            vec_config: None,
        })
    }

    /// Create a new multimodal backend with native vector column support.
    ///
    /// Native vector columns enable ANN indexing for fast search.
    ///
    /// # Example
    ///
    /// ```no_run
    /// use aifastdb_core::types::multimodal::{MultimodalVectorConfig, VectorColumnDef};
    /// use aifastdb_core::storage::lance::LanceMultimodalBackend;
    ///
    /// let vec_config = MultimodalVectorConfig::clip(512);
    /// let backend = LanceMultimodalBackend::with_vectors("./data/multimodal.lance", vec_config).unwrap();
    /// ```
    pub fn with_vectors(uri: impl Into<String>, vec_config: MultimodalVectorConfig) -> StorageResult<Self> {
        let rt = Runtime::new()
            .map_err(|e| StorageError::Backend(format!("Failed to create runtime: {}", e)))?;
        let schema = multimodal_arrow_schema_with_vectors(&vec_config);

        Ok(Self {
            uri: uri.into(),
            schema,
            rt: Arc::new(rt),
            dataset: parking_lot::RwLock::new(None),
            vec_config: Some(vec_config),
        })
    }

    /// Get the vector column configuration.
    pub fn vector_config(&self) -> Option<&MultimodalVectorConfig> {
        self.vec_config.as_ref()
    }

    /// Get or open the Lance dataset (always refreshes to latest version).
    fn get_or_open_dataset(&self) -> StorageResult<Arc<Dataset>> {
        let uri = self.uri.clone();
        match self.rt.block_on(async {
            DatasetBuilder::from_uri(&uri).load().await
        }) {
            Ok(ds) => {
                let ds = Arc::new(ds);
                *self.dataset.write() = Some(ds.clone());
                Ok(ds)
            }
            Err(_) => {
                // Dataset doesn't exist yet, will be created on first write
                Err(StorageError::NotFound(
                    "Multimodal dataset not yet created — will be created on first write".into()
                ))
            }
        }
    }

    /// Write records to the Lance dataset (create or append).
    fn write_records(&self, records: &[MultimodalRecord]) -> StorageResult<()> {
        let batch = multimodal_to_record_batch(records, &self.schema, self.vec_config.as_ref())?;
        let schema = self.schema.clone();

        let uri = self.uri.clone();
        let ds_exists = self.dataset.read().is_some();

        self.rt.block_on(async {
            if ds_exists {
                // Delete existing records with same IDs first
                let ids: Vec<String> = records.iter().map(|r| format!("'{}'", r.id)).collect();
                let delete_filter = format!("id IN ({})", ids.join(", "));
                if let Ok(ds) = DatasetBuilder::from_uri(&uri).load().await {
                    let mut ds_mut = ds;
                    let _ = ds_mut.delete(&delete_filter).await;
                }

                // Append new records with WriteMode::Append
                let write_params = lance::dataset::WriteParams {
                    mode: lance::dataset::WriteMode::Append,
                    ..Default::default()
                };
                let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
                let new_ds = Dataset::write(reader, &uri, Some(write_params))
                    .await
                    .map_err(|e| StorageError::Backend(format!("Lance append failed: {}", e)))?;
                *self.dataset.write() = Some(Arc::new(new_ds));
            } else {
                // Create new dataset
                let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
                let new_ds = Dataset::write(reader, &uri, None)
                    .await
                    .map_err(|e| StorageError::Backend(format!("Lance create failed: {}", e)))?;
                *self.dataset.write() = Some(Arc::new(new_ds));
            }

            Ok(())
        })
    }

    /// Scan the dataset with an optional filter and column projection.
    fn scan_records(
        &self,
        filter: Option<&str>,
        columns: Option<&[&str]>,
        limit: Option<usize>,
    ) -> StorageResult<Vec<RecordBatch>> {
        let ds = self.get_or_open_dataset()?;

        self.rt.block_on(async {
            let mut scanner = ds.scan();
            if let Some(f) = filter {
                scanner.filter(f)
                    .map_err(|e| StorageError::Backend(format!("Filter error: {}", e)))?;
            }
            if let Some(cols) = columns {
                scanner.project(cols)
                    .map_err(|e| StorageError::Backend(format!("Project error: {}", e)))?;
            }
            if let Some(lim) = limit {
                scanner.limit(Some(lim as i64), None)
                    .map_err(|e| StorageError::Backend(format!("Limit error: {}", e)))?;
            }

            use futures::TryStreamExt;
            let batches: Vec<RecordBatch> = scanner.try_into_stream()
                .await
                .map_err(|e| StorageError::Backend(format!("Scan stream error: {}", e)))?
                .try_collect::<Vec<RecordBatch>>()
                .await
                .map_err(|e| StorageError::Backend(format!("Collect error: {}", e)))?;

            Ok(batches)
        })
    }
}

impl MultimodalBackend for LanceMultimodalBackend {
    fn put_multimodal(
        &self,
        record: &MultimodalRecord,
    ) -> StorageResult<()> {
        self.write_records(&[record.clone()])
    }

    fn get_multimodal(
        &self,
        id: &str,
        include_data: bool,
    ) -> StorageResult<Option<MultimodalRecord>> {
        let filter = format!("id = '{}'", id.replace('\'', "''"));

        let batches = match self.scan_records(Some(&filter), None, Some(1)) {
            Ok(b) => b,
            Err(StorageError::NotFound(_)) => return Ok(None),
            Err(e) => return Err(e),
        };

        for batch in &batches {
            if batch.num_rows() > 0 {
                let record = record_batch_row_to_multimodal(batch, 0, include_data)?;
                return Ok(Some(record));
            }
        }

        Ok(None)
    }

    fn delete_multimodal(
        &self,
        id: &str,
    ) -> StorageResult<bool> {
        let _ = match self.get_or_open_dataset() {
            Ok(d) => d,
            Err(StorageError::NotFound(_)) => return Ok(false),
            Err(e) => return Err(e),
        };

        let filter = format!("id = '{}'", id.replace('\'', "''"));
        let uri = self.uri.clone();

        self.rt.block_on(async {
            // Re-open to get the latest version, then delete
            let mut ds = DatasetBuilder::from_uri(&uri).load().await
                .map_err(|e| StorageError::Backend(format!("Open failed: {}", e)))?;
            ds.delete(&filter).await
                .map_err(|e| StorageError::Backend(format!("Delete failed: {}", e)))?;
            // Refresh the cached dataset handle
            let refreshed = DatasetBuilder::from_uri(&uri).load().await
                .map_err(|e| StorageError::Backend(format!("Refresh failed: {}", e)))?;
            *self.dataset.write() = Some(Arc::new(refreshed));
            Ok(true)
        })
    }

    fn list_multimodal(
        &self,
        media_type: Option<MediaType>,
        limit: usize,
        offset: usize,
    ) -> StorageResult<Vec<String>> {
        let filter = media_type.map(|mt| format!("media_type = '{}'", mt));
        let filter_ref = filter.as_deref();

        let batches = match self.scan_records(filter_ref, Some(&["id"]), Some(limit + offset)) {
            Ok(b) => b,
            Err(StorageError::NotFound(_)) => return Ok(vec![]),
            Err(e) => return Err(e),
        };

        let mut ids = Vec::new();
        let mut skipped = 0;
        for batch in &batches {
            let id_col = batch.column(0).as_any().downcast_ref::<StringArray>()
                .ok_or_else(|| StorageError::Serialization("id column type mismatch".into()))?;
            for i in 0..batch.num_rows() {
                if skipped < offset {
                    skipped += 1;
                    continue;
                }
                if ids.len() >= limit {
                    break;
                }
                ids.push(id_col.value(i).to_string());
            }
        }

        Ok(ids)
    }

    fn multimodal_count(
        &self,
        media_type: Option<MediaType>,
    ) -> StorageResult<usize> {
        let filter = media_type.map(|mt| format!("media_type = '{}'", mt));
        let filter_ref = filter.as_deref();

        let batches = match self.scan_records(filter_ref, Some(&["id"]), None) {
            Ok(b) => b,
            Err(StorageError::NotFound(_)) => return Ok(0),
            Err(e) => return Err(e),
        };

        let count: usize = batches.iter().map(|b| b.num_rows()).sum();
        Ok(count)
    }

    fn search_multimodal(
        &self,
        query_vector: &[f32],
        embedding_column: &str,
        top_k: usize,
        media_type: Option<MediaType>,
    ) -> StorageResult<Vec<CrossModalSearchResult>> {
        // Phase-60: Try ANN search first if native vector column is available
        if let Some(vc) = &self.vec_config {
            if let Some(col_def) = vc.resolve_column(embedding_column) {
                // Check if the dataset exists and the column is present
                if let Ok(ds) = self.get_or_open_dataset() {
                    let schema = ds.schema();
                    if schema.field(&col_def.column_name).is_some() {
                        let ann_result = self.search_multimodal_ann(
                            &ds, &col_def.column_name, query_vector, top_k, media_type,
                        );
                        match ann_result {
                            Ok(results) if !results.is_empty() => return Ok(results),
                            Ok(_) => { /* empty results — fall through to brute-force */ }
                            Err(_) => { /* ANN failed — fall through to brute-force */ }
                        }
                    }
                }
            }
        }

        // Fallback: brute-force search via JSON embeddings
        self.search_multimodal_brute_force(query_vector, embedding_column, top_k, media_type)
    }

    fn create_multimodal_index(
        &self,
        embedding_name: &str,
        config: &VectorIndexConfig,
    ) -> StorageResult<()> {
        use lance::index::vector::VectorIndexParams;
        use lance_index::IndexType;
        use lance_index::traits::DatasetIndexExt;
        use lance_index::vector::hnsw::builder::HnswBuildParams;
        use lance_index::vector::ivf::IvfBuildParams;
        use lance_index::vector::pq::PQBuildParams;
        use lance_index::vector::sq::builder::SQBuildParams;
        use lance_linalg::distance::DistanceType;
        use crate::storage::vector_backend::{VectorIndexType, VectorDistanceMetric};

        // Resolve the native vector column name
        let col_def = self.vec_config.as_ref()
            .and_then(|vc| vc.resolve_column(embedding_name))
            .ok_or_else(|| StorageError::Backend(format!(
                "No native vector column configured for embedding '{}'. \
                 Use LanceMultimodalBackend::with_vectors() to configure vector columns.",
                embedding_name
            )))?;

        let column_name = col_def.column_name.clone();
        let dimension = col_def.dimension;

        let ds = self.get_or_open_dataset()?;
        let num_rows = self.rt.block_on(async {
            ds.count_rows(None).await
        }).map_err(|e| StorageError::Backend(format!("Failed to count rows: {}", e)))?;

        if num_rows < 2 {
            return Err(StorageError::Backend(
                "Cannot create index: need at least 2 vectors".into()
            ));
        }

        // ================================================================
        // GPU Acceleration Path (Phase-60 + Phase-59 integration)
        // ================================================================
        // When gpu_accelerate is true and the gpu-index feature is enabled,
        // we use IndexBuildPipeline for GPU-accelerated KMeans training.
        // The trained centroids are then used to build the Lance index.
        // If GPU is unavailable, transparently falls back to CPU.
        // ================================================================
        #[cfg(feature = "gpu-index")]
        let gpu_pretrained = if config.gpu_accelerate {
            self.gpu_pretrain_centroids(
                &column_name, dimension, num_rows, config,
            )
        } else {
            None
        };

        #[cfg(not(feature = "gpu-index"))]
        let gpu_pretrained: Option<Vec<f32>> = None;

        // Resolve distance metric
        let metric_type = match config.metric {
            VectorDistanceMetric::L2 => DistanceType::L2,
            VectorDistanceMetric::Cosine => DistanceType::Cosine,
            VectorDistanceMetric::Dot => DistanceType::Dot,
        };

        // Determine index parameters
        let num_partitions = config.num_partitions
            .unwrap_or_else(|| ((num_rows as f64).sqrt() as u32).max(2));

        let build_ivf = || {
            let mut ivf = IvfBuildParams::new(num_partitions as usize);
            // If GPU pre-trained centroids are available, inject them
            if let Some(ref centroids_data) = gpu_pretrained {
                tracing::info!(
                    "Using GPU pre-trained centroids ({} partitions × {} dims = {} floats)",
                    num_partitions, dimension, centroids_data.len()
                );
                let values_array = Float32Array::from(centroids_data.clone());
                let field = Arc::new(Field::new("item", DataType::Float32, true));
                if let Ok(centroids_arr) = FixedSizeListArray::try_new(
                    field, dimension as i32, Arc::new(values_array), None,
                ) {
                    ivf.centroids = Some(Arc::new(centroids_arr));
                } else {
                    tracing::warn!("Failed to create centroids FixedSizeListArray, using default training");
                }
            }
            ivf
        };
        let build_pq = || {
            let nsv = config.num_sub_vectors.unwrap_or(16) as usize;
            PQBuildParams {
                num_sub_vectors: nsv,
                ..Default::default()
            }
        };
        let build_hnsw = || {
            let mut hnsw = HnswBuildParams::default();
            if let Some(m) = config.hnsw_m { hnsw = hnsw.num_edges(m); }
            if let Some(ef) = config.hnsw_ef_construction { hnsw = hnsw.ef_construction(ef); }
            if let Some(ml) = config.hnsw_max_level { hnsw = hnsw.max_level(ml); }
            hnsw
        };
        let build_sq = || SQBuildParams {
            num_bits: config.sq_num_bits.unwrap_or(8),
            sample_rate: config.sq_sample_rate.unwrap_or(256),
        };

        let (index_type, params) = match config.index_type {
            VectorIndexType::IvfPq => {
                (IndexType::IvfPq, VectorIndexParams::with_ivf_pq_params(
                    metric_type, build_ivf(), build_pq(),
                ))
            }
            VectorIndexType::IvfHnswSq => {
                (IndexType::IvfHnswSq, VectorIndexParams::with_ivf_hnsw_sq_params(
                    metric_type, build_ivf(), build_hnsw(), build_sq(),
                ))
            }
            VectorIndexType::IvfHnswPq => {
                (IndexType::IvfHnswPq, VectorIndexParams::with_ivf_hnsw_pq_params(
                    metric_type, build_ivf(), build_hnsw(), build_pq(),
                ))
            }
            VectorIndexType::Cagra => {
                // CAGRA is GPU-exclusive. Use IVF_HNSW_SQ as Lance storage format
                // with GPU pre-trained centroids if available.
                tracing::info!("CAGRA index requested for multimodal; using IVF_HNSW_SQ as Lance storage format");
                (IndexType::IvfHnswSq, VectorIndexParams::with_ivf_hnsw_sq_params(
                    metric_type, build_ivf(), build_hnsw(), build_sq(),
                ))
            }
            _ => {
                // Default to IVF_PQ for unsupported types
                (IndexType::IvfPq, VectorIndexParams::with_ivf_pq_params(
                    metric_type, build_ivf(), build_pq(),
                ))
            }
        };

        // Create index on the dataset
        let uri = self.uri.clone();
        self.rt.block_on(async {
            let ds = DatasetBuilder::from_uri(&uri).load().await
                .map_err(|e| StorageError::Backend(format!("Failed to load dataset: {}", e)))?;
            let mut ds = ds;

            ds.create_index(
                &[&column_name],
                index_type,
                None,
                &params,
                true, // replace existing
            ).await.map_err(|e| StorageError::Backend(
                format!("Failed to create multimodal index on '{}': {}", column_name, e),
            ))?;

            // Refresh cached dataset
            let refreshed = DatasetBuilder::from_uri(&uri).load().await
                .map_err(|e| StorageError::Backend(format!("Failed to refresh dataset: {}", e)))?;
            *self.dataset.write() = Some(Arc::new(refreshed));

            Ok(())
        })
    }

    fn has_multimodal_index(
        &self,
        embedding_name: &str,
    ) -> StorageResult<bool> {
        use lance_index::traits::DatasetIndexExt;

        let col_def = self.vec_config.as_ref()
            .and_then(|vc| vc.resolve_column(embedding_name));

        let col_def = match col_def {
            Some(d) => d,
            None => return Ok(false),
        };

        let ds = match self.get_or_open_dataset() {
            Ok(ds) => ds,
            Err(_) => return Ok(false),
        };

        // Use load_indices (from DatasetIndexExt) to check index existence
        let indices = self.rt.block_on(async {
            ds.load_indices().await
        }).map_err(|e| StorageError::Backend(format!("Failed to load indices: {}", e)))?;

        if indices.is_empty() {
            return Ok(false);
        }

        // Check if any index covers this column by matching field IDs in schema
        let schema = ds.schema();
        let column_name = &col_def.column_name;
        let has_idx = indices.iter().any(|idx| {
            idx.fields.iter().any(|&field_id| {
                schema.fields.iter().any(|f| {
                    f.id == field_id as i32 && f.name == *column_name
                })
            })
        });

        Ok(has_idx)
    }

    fn multimodal_index_info(
        &self,
    ) -> StorageResult<Vec<MultimodalIndexInfo>> {
        let vc = match &self.vec_config {
            Some(vc) => vc,
            None => return Ok(vec![]),
        };

        let mut infos = Vec::new();
        for (embedding_name, def) in &vc.columns {
            let has_index = self.has_multimodal_index(embedding_name).unwrap_or(false);
            infos.push(MultimodalIndexInfo {
                embedding_name: embedding_name.clone(),
                column_name: def.column_name.clone(),
                dimension: def.dimension,
                has_index,
                num_vectors: None, // Could be populated with a scan count
            });
        }

        Ok(infos)
    }
}

// ============================================================================
// GPU Accelerated Index Building (Phase-60 T60.4)
// ============================================================================

impl LanceMultimodalBackend {
    /// GPU pre-train KMeans centroids for IVF index building.
    ///
    /// When `gpu-index` feature is enabled and GPU is available, this method:
    /// 1. Scans the native vector column to collect all vectors
    /// 2. Uses `IndexBuildPipeline` to train KMeans centroids on GPU
    /// 3. Returns the trained centroids as `Vec<f32>`
    ///
    /// Returns `None` if:
    /// - GPU is not available (transparent fallback)
    /// - Data is too small for GPU benefit (< 10K vectors)
    /// - Any error occurs during GPU training (logs warning, falls back)
    #[cfg(feature = "gpu-index")]
    fn gpu_pretrain_centroids(
        &self,
        vector_column: &str,
        dimension: usize,
        num_rows: usize,
        config: &VectorIndexConfig,
    ) -> Option<Vec<f32>> {
        use crate::gpu::{
            GpuIndexConfig, GpuBuildMode, IndexBuildPipeline, should_use_gpu,
        };

        // Check if GPU would be beneficial for this dataset size
        if !should_use_gpu(num_rows, dimension) {
            tracing::info!(
                "Multimodal GPU index: dataset too small ({} vectors × {} dims), using CPU path",
                num_rows, dimension
            );
            return None;
        }

        // Build GpuIndexConfig from VectorIndexConfig GPU parameters
        let gpu_config = {
            let mut gc = match config.gpu_device_id {
                Some(id) if id >= 0 => GpuIndexConfig::full_gpu(id),
                _ => GpuIndexConfig::auto(),
            };
            if let Some(budget) = config.gpu_memory_budget {
                gc = gc.with_memory_budget(budget);
            }
            gc
        };

        // Create the pipeline (probes GPU automatically)
        let pipeline = IndexBuildPipeline::new(gpu_config);

        if !pipeline.will_use_gpu() {
            tracing::info!("Multimodal GPU index: no GPU available, falling back to CPU");
            return None;
        }

        tracing::info!(
            "Multimodal GPU index: using GPU for KMeans training ({} vectors × {} dims)",
            num_rows, dimension
        );

        // Collect vectors from the native vector column
        let vectors = match self.collect_vectors_for_gpu(vector_column, dimension) {
            Ok(v) => v,
            Err(e) => {
                tracing::warn!("Multimodal GPU index: failed to collect vectors: {}, falling back to CPU", e);
                return None;
            }
        };

        let n_vectors = vectors.len() / dimension;
        let num_partitions = config.num_partitions
            .unwrap_or_else(|| ((n_vectors as f64).sqrt() as u32).max(2));

        // Train centroids using GPU pipeline
        match pipeline.build_ivf(&vectors, n_vectors, dimension, num_partitions as usize) {
            Ok(result) => {
                tracing::info!(
                    "Multimodal GPU index: KMeans training completed in {:.1}ms, used_gpu={}",
                    result.total_time_ms, result.device_used
                );
                Some(result.centroids)
            }
            Err(e) => {
                tracing::warn!(
                    "Multimodal GPU index: GPU KMeans training failed: {}, falling back to CPU",
                    e
                );
                None
            }
        }
    }

    /// Collect all vectors from a native vector column for GPU training.
    ///
    /// Scans the Lance dataset and extracts the FixedSizeList vector column
    /// into a flat `Vec<f32>` suitable for GPU processing.
    #[cfg(feature = "gpu-index")]
    fn collect_vectors_for_gpu(
        &self,
        vector_column: &str,
        dimension: usize,
    ) -> StorageResult<Vec<f32>> {
        use futures::stream::TryStreamExt;
        use arrow_array::cast::AsArray;

        let ds = self.get_or_open_dataset()?;

        self.rt.block_on(async {
            let mut scanner = ds.scan();
            scanner.project(&[vector_column])
                .map_err(|e| StorageError::Backend(format!(
                    "Failed to project vector column '{}': {}", vector_column, e
                )))?;

            let stream = scanner.try_into_stream().await
                .map_err(|e| StorageError::Backend(format!(
                    "Failed to create scan stream: {}", e
                )))?;

            let batches: Vec<RecordBatch> = stream.try_collect::<Vec<RecordBatch>>().await
                .map_err(|e| StorageError::Backend(format!(
                    "Failed to collect batches: {}", e
                )))?;

            let mut all_vectors: Vec<f32> = Vec::new();

            for batch in &batches {
                if let Some(col) = batch.column_by_name(vector_column) {
                    // The column is FixedSizeList<Float32, dim>
                    let fsl = col.as_fixed_size_list();
                    let values = fsl.values();
                    let float_arr = values.as_any().downcast_ref::<Float32Array>()
                        .ok_or_else(|| StorageError::Backend(
                            "Vector column values are not Float32".into()
                        ))?;

                    all_vectors.extend_from_slice(float_arr.values());
                }
            }

            if all_vectors.is_empty() {
                return Err(StorageError::Backend(
                    "No vectors found in native vector column".into()
                ));
            }

            let expected = (all_vectors.len() / dimension) * dimension;
            if all_vectors.len() != expected {
                tracing::warn!(
                    "Vector data size {} not evenly divisible by dimension {}, truncating",
                    all_vectors.len(), dimension
                );
                all_vectors.truncate(expected);
            }

            Ok(all_vectors)
        })
    }
}

// ============================================================================
// Private ANN Search Helpers
// ============================================================================

impl LanceMultimodalBackend {
    /// ANN search via Lance scanner.nearest() — uses vector index if available.
    ///
    /// Returns results with score = 1.0 - distance (for cosine metric).
    fn search_multimodal_ann(
        &self,
        ds: &Dataset,
        vector_column: &str,
        query: &[f32],
        k: usize,
        media_type: Option<MediaType>,
    ) -> StorageResult<Vec<CrossModalSearchResult>> {
        use futures::stream::TryStreamExt;

        let query_array = Float32Array::from(query.to_vec());
        let filter = media_type.map(|mt| format!("media_type = '{}'", mt));
        let nprobe = 20; // reasonable default for multimodal search

        let results = self.rt.block_on(async {
            let mut scanner = ds.scan();

            scanner
                .nearest(vector_column, &query_array, k)
                .map_err(|e| StorageError::Backend(format!("ANN nearest setup failed: {}", e)))?;

            scanner.nprobes(nprobe);

            // Apply media_type filter (pushed down to Lance scanner)
            if let Some(ref f) = filter {
                scanner.filter(f)
                    .map_err(|e| StorageError::Backend(format!("ANN filter failed: {}", e)))?;
            }

            // Project the columns we need for CrossModalSearchResult
            scanner
                .project(&["id", "media_type", "mime_type", "text_content",
                          "tags_json", "metadata_json", "thumbnail", "_distance"])
                .map_err(|e| StorageError::Backend(format!("ANN project failed: {}", e)))?;

            let stream = scanner
                .try_into_stream()
                .await
                .map_err(|e| StorageError::Backend(format!("ANN stream failed: {}", e)))?;

            let batches: Vec<RecordBatch> = stream
                .try_collect::<Vec<RecordBatch>>()
                .await
                .map_err(|e| StorageError::Backend(format!("ANN collect failed: {}", e)))?;

            let mut scored: Vec<CrossModalSearchResult> = Vec::new();

            for batch in &batches {
                let id_col = batch.column_by_name("id")
                    .and_then(|c| c.as_any().downcast_ref::<StringArray>());
                let media_type_col = batch.column_by_name("media_type")
                    .and_then(|c| c.as_any().downcast_ref::<StringArray>());
                let mime_type_col = batch.column_by_name("mime_type")
                    .and_then(|c| c.as_any().downcast_ref::<StringArray>());
                let text_col = batch.column_by_name("text_content")
                    .and_then(|c| c.as_any().downcast_ref::<StringArray>());
                let tags_col = batch.column_by_name("tags_json")
                    .and_then(|c| c.as_any().downcast_ref::<StringArray>());
                let metadata_col = batch.column_by_name("metadata_json")
                    .and_then(|c| c.as_any().downcast_ref::<StringArray>());
                let thumbnail_col = batch.column_by_name("thumbnail")
                    .and_then(|c| c.as_any().downcast_ref::<BinaryArray>());
                let dist_col = batch.column_by_name("_distance")
                    .and_then(|c| c.as_any().downcast_ref::<Float32Array>());

                let (id_col, media_type_col, mime_type_col, dist_col) =
                    match (id_col, media_type_col, mime_type_col, dist_col) {
                        (Some(a), Some(b), Some(c), Some(d)) => (a, b, c, d),
                        _ => continue,
                    };

                for i in 0..batch.num_rows() {
                    let distance = dist_col.value(i);
                    // Convert distance to similarity score (cosine distance → similarity)
                    let score = 1.0 - distance;

                    let parsed_media_type = match media_type_col.value(i) {
                        "image" => MediaType::Image,
                        "audio" => MediaType::Audio,
                        "video" => MediaType::Video,
                        _ => MediaType::Image,
                    };

                    let text_content = text_col
                        .filter(|tc| !tc.is_null(i))
                        .map(|tc| tc.value(i).to_string());

                    let tags: Vec<String> = tags_col
                        .map(|tc| serde_json::from_str(tc.value(i)).unwrap_or_default())
                        .unwrap_or_default();

                    let file_size = metadata_col
                        .map(|mc| {
                            let v: serde_json::Value = serde_json::from_str(mc.value(i)).unwrap_or_default();
                            v.get("file_size_bytes").and_then(|v| v.as_u64()).unwrap_or(0)
                        })
                        .unwrap_or(0);

                    let has_thumbnail = thumbnail_col
                        .map(|tc| !tc.is_null(i) && tc.value(i).len() > 0)
                        .unwrap_or(false);

                    scored.push(CrossModalSearchResult {
                        id: id_col.value(i).to_string(),
                        media_type: parsed_media_type,
                        score,
                        mime_type: mime_type_col.value(i).to_string(),
                        text_content,
                        file_size_bytes: file_size,
                        has_thumbnail,
                        tags,
                    });
                }
            }

            // Sort by score descending (highest similarity first)
            scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));

            Ok::<Vec<CrossModalSearchResult>, StorageError>(scored)
        })?;

        Ok(results)
    }

    /// Legacy brute-force search: scan all rows, parse JSON, compute cosine similarity.
    fn search_multimodal_brute_force(
        &self,
        query_vector: &[f32],
        embedding_column: &str,
        top_k: usize,
        media_type: Option<MediaType>,
    ) -> StorageResult<Vec<CrossModalSearchResult>> {
        // Build filter for media type if specified
        let filter = media_type.map(|mt| format!("media_type = '{}'", mt));
        let filter_ref = filter.as_deref();

        // Scan relevant columns (use actual Lance column names with _json suffix)
        let columns = &["id", "media_type", "mime_type", "text_content", "tags_json", "embeddings_json", "metadata_json", "thumbnail"];
        let batches = match self.scan_records(filter_ref, Some(columns), None) {
            Ok(b) => b,
            Err(StorageError::NotFound(_)) => return Ok(vec![]),
            Err(e) => return Err(e),
        };

        // Collect all records with their embeddings and compute cosine similarity
        let mut scored: Vec<CrossModalSearchResult> = Vec::new();

        for batch in &batches {
            let id_col = batch.column_by_name("id")
                .and_then(|c| c.as_any().downcast_ref::<StringArray>())
                .ok_or_else(|| StorageError::Serialization("id column missing".into()))?;
            let media_type_col = batch.column_by_name("media_type")
                .and_then(|c| c.as_any().downcast_ref::<StringArray>())
                .ok_or_else(|| StorageError::Serialization("media_type column missing".into()))?;
            let mime_type_col = batch.column_by_name("mime_type")
                .and_then(|c| c.as_any().downcast_ref::<StringArray>())
                .ok_or_else(|| StorageError::Serialization("mime_type column missing".into()))?;
            let text_content_col = batch.column_by_name("text_content")
                .and_then(|c| c.as_any().downcast_ref::<StringArray>())
                .ok_or_else(|| StorageError::Serialization("text_content column missing".into()))?;
            let tags_col = batch.column_by_name("tags_json")
                .and_then(|c| c.as_any().downcast_ref::<StringArray>())
                .ok_or_else(|| StorageError::Serialization("tags_json column missing".into()))?;
            let embeddings_col = batch.column_by_name("embeddings_json")
                .and_then(|c| c.as_any().downcast_ref::<StringArray>())
                .ok_or_else(|| StorageError::Serialization("embeddings_json column missing".into()))?;
            let metadata_col = batch.column_by_name("metadata_json")
                .and_then(|c| c.as_any().downcast_ref::<StringArray>())
                .ok_or_else(|| StorageError::Serialization("metadata_json column missing".into()))?;
            let thumbnail_col = batch.column_by_name("thumbnail")
                .and_then(|c| c.as_any().downcast_ref::<BinaryArray>());

            for i in 0..batch.num_rows() {
                // Parse embeddings JSON
                let embeddings_str = embeddings_col.value(i);
                let embeddings: HashMap<String, Vec<f32>> =
                    serde_json::from_str(embeddings_str).unwrap_or_default();

                // Look for the requested embedding column
                let record_embedding = match embeddings.get(embedding_column) {
                    Some(emb) => emb,
                    None => continue, // Skip records without this embedding
                };

                // Compute cosine similarity
                let score = cosine_similarity(query_vector, record_embedding);

                // Parse metadata for file_size
                let metadata_str = metadata_col.value(i);
                let metadata: serde_json::Value = serde_json::from_str(metadata_str).unwrap_or_default();
                let file_size = metadata.get("file_size_bytes")
                    .and_then(|v| v.as_u64())
                    .unwrap_or(0);

                // Parse tags
                let tags_str = tags_col.value(i);
                let tags: Vec<String> = serde_json::from_str(tags_str).unwrap_or_default();

                let has_thumbnail = thumbnail_col
                    .map(|tc| !tc.is_null(i) && tc.value(i).len() > 0)
                    .unwrap_or(false);

                let parsed_media_type = match media_type_col.value(i) {
                    "image" => MediaType::Image,
                    "audio" => MediaType::Audio,
                    "video" => MediaType::Video,
                    _ => MediaType::Image, // fallback
                };

                scored.push(CrossModalSearchResult {
                    id: id_col.value(i).to_string(),
                    media_type: parsed_media_type,
                    score,
                    mime_type: mime_type_col.value(i).to_string(),
                    text_content: if text_content_col.is_null(i) { None } else { Some(text_content_col.value(i).to_string()) },
                    file_size_bytes: file_size,
                    has_thumbnail,
                    tags,
                });
            }
        }

        // Sort by score descending
        scored.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));

        // Return top_k
        scored.truncate(top_k);
        Ok(scored)
    }
}

/// Compute cosine similarity between two vectors.
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
    if a.len() != b.len() || a.is_empty() {
        return 0.0;
    }
    let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
    let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
    let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
    if norm_a > 1e-12 && norm_b > 1e-12 {
        dot / (norm_a * norm_b)
    } else {
        0.0
    }
}

// ============================================================================
// Tests
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;

    fn make_backend(tmp: &TempDir) -> LanceMultimodalBackend {
        let path = tmp.path().join("test_multimodal.lance");
        LanceMultimodalBackend::new(path.to_string_lossy().to_string()).unwrap()
    }

    fn make_test_record(id: &str, media_type: MediaType) -> MultimodalRecord {
        MultimodalRecord::new(id, media_type)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]) // fake JPEG header
            .with_text_content("A test image of a cat")
            .with_tag("test")
            .with_user_metadata("source", serde_json::json!("unit-test"))
    }

    #[test]
    fn test_put_and_get_multimodal() {
        let tmp = TempDir::new().unwrap();
        let backend = make_backend(&tmp);

        let record = make_test_record("img-001", MediaType::Image);
        backend.put_multimodal(&record).unwrap();

        // Get with data
        let result = backend.get_multimodal("img-001", true).unwrap();
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.id, "img-001");
        assert_eq!(r.media_type, MediaType::Image);
        assert_eq!(r.mime_type, "image/jpeg");
        assert_eq!(r.text_content, Some("A test image of a cat".to_string()));
        assert!(r.data.is_some());
        assert_eq!(r.data.unwrap().len(), 6);
        assert_eq!(r.tags, vec!["test".to_string()]);
    }

    #[test]
    fn test_get_without_data() {
        let tmp = TempDir::new().unwrap();
        let backend = make_backend(&tmp);

        let record = make_test_record("img-002", MediaType::Image)
            .with_data(vec![0; 1024]); // 1KB of data
        backend.put_multimodal(&record).unwrap();

        // Get without data (metadata only)
        let result = backend.get_multimodal("img-002", false).unwrap();
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.id, "img-002");
        assert!(r.data.is_none()); // No binary data loaded
    }

    #[test]
    fn test_delete_multimodal() {
        let tmp = TempDir::new().unwrap();
        let backend = make_backend(&tmp);

        let record = make_test_record("img-003", MediaType::Image);
        backend.put_multimodal(&record).unwrap();

        let deleted = backend.delete_multimodal("img-003").unwrap();
        assert!(deleted);

        let result = backend.get_multimodal("img-003", false).unwrap();
        assert!(result.is_none());
    }

    #[test]
    fn test_list_multimodal() {
        let tmp = TempDir::new().unwrap();
        let backend = make_backend(&tmp);

        // Write multiple records
        let records = vec![
            make_test_record("img-010", MediaType::Image),
            MultimodalRecord::new("aud-001", MediaType::Audio)
                .with_mime_type("audio/wav")
                .with_data(vec![0; 100]),
            MultimodalRecord::new("vid-001", MediaType::Video)
                .with_mime_type("video/mp4")
                .with_data(vec![0; 200]),
        ];
        for r in &records {
            backend.put_multimodal(r).unwrap();
        }

        // List all
        let all_ids = backend.list_multimodal(None, 100, 0).unwrap();
        assert_eq!(all_ids.len(), 3);

        // List by media type
        let image_ids = backend.list_multimodal(Some(MediaType::Image), 100, 0).unwrap();
        assert_eq!(image_ids.len(), 1);
        assert_eq!(image_ids[0], "img-010");

        // Count
        let total = backend.multimodal_count(None).unwrap();
        assert_eq!(total, 3);
        let audio_count = backend.multimodal_count(Some(MediaType::Audio)).unwrap();
        assert_eq!(audio_count, 1);
    }

    #[test]
    fn test_search_multimodal_cross_modal() {
        let tmp = TempDir::new().unwrap();
        let backend = make_backend(&tmp);

        // Create records with embeddings simulating CLIP output
        let rec1 = MultimodalRecord::new("img-cat", MediaType::Image)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF, 0xD8])
            .with_text_content("A cat sitting on a chair")
            .with_embedding("clip_image", vec![0.8, 0.1, 0.0, 0.1]) // "cat-like" vector
            .with_tag("animal");

        let rec2 = MultimodalRecord::new("img-dog", MediaType::Image)
            .with_mime_type("image/png")
            .with_data(vec![0x89, 0x50])
            .with_text_content("A dog playing in the park")
            .with_embedding("clip_image", vec![0.1, 0.8, 0.0, 0.1]) // "dog-like" vector
            .with_tag("animal");

        let rec3 = MultimodalRecord::new("img-car", MediaType::Image)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF, 0xD8])
            .with_text_content("A red sports car")
            .with_embedding("clip_image", vec![0.0, 0.0, 0.9, 0.1]) // "car-like" vector
            .with_tag("vehicle");

        backend.put_multimodal(&rec1).unwrap();
        backend.put_multimodal(&rec2).unwrap();
        backend.put_multimodal(&rec3).unwrap();

        // Search with a "cat-like" query vector
        let query = vec![0.9, 0.1, 0.0, 0.0]; // similar to cat
        let results = backend.search_multimodal(&query, "clip_image", 3, None).unwrap();

        assert_eq!(results.len(), 3);
        // Cat should be the top result (highest cosine similarity to cat-like query)
        assert_eq!(results[0].id, "img-cat");
        assert!(results[0].score > results[1].score);
        assert!(results[0].score > 0.9); // High similarity

        // Search with top_k = 1
        let results = backend.search_multimodal(&query, "clip_image", 1, None).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, "img-cat");

        // Search with media_type filter (only images — all are images, so same result)
        let results = backend.search_multimodal(&query, "clip_image", 3, Some(MediaType::Image)).unwrap();
        assert_eq!(results.len(), 3);

        // Search with audio filter — no audio records
        let results = backend.search_multimodal(&query, "clip_image", 3, Some(MediaType::Audio)).unwrap();
        assert_eq!(results.len(), 0);
    }

    #[test]
    fn test_search_multimodal_no_embedding() {
        let tmp = TempDir::new().unwrap();
        let backend = make_backend(&tmp);

        // Record without any embedding
        let rec = MultimodalRecord::new("img-no-emb", MediaType::Image)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF]);
        backend.put_multimodal(&rec).unwrap();

        // Record with embedding in a different column
        let rec2 = MultimodalRecord::new("img-text-emb", MediaType::Image)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF])
            .with_embedding("clip_text", vec![0.5, 0.5, 0.0, 0.0]);
        backend.put_multimodal(&rec2).unwrap();

        // Search for "clip_image" — neither record has it
        let query = vec![0.5, 0.5, 0.0, 0.0];
        let results = backend.search_multimodal(&query, "clip_image", 10, None).unwrap();
        assert_eq!(results.len(), 0);

        // Search for "clip_text" — only rec2 has it
        let results = backend.search_multimodal(&query, "clip_text", 10, None).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, "img-text-emb");
    }

    // ========================================================================
    // Phase-60: Native Vector Column + ANN Search Tests
    // ========================================================================

    fn make_vec_config(dim: usize) -> MultimodalVectorConfig {
        MultimodalVectorConfig::new()
            .with_column("clip_image", VectorColumnDef::new("vec_clip_image", dim))
    }

    fn make_backend_with_vectors(tmp: &TempDir, dim: usize) -> LanceMultimodalBackend {
        let path = tmp.path().join("test_multimodal_vec.lance");
        LanceMultimodalBackend::with_vectors(
            path.to_string_lossy().to_string(),
            make_vec_config(dim),
        ).unwrap()
    }

    #[test]
    fn test_native_vector_column_dual_write() {
        // T60.1 verification: records with embeddings get written to both
        // embeddings_json AND the native vector column
        let tmp = TempDir::new().unwrap();
        let dim = 4;
        let backend = make_backend_with_vectors(&tmp, dim);

        let rec = MultimodalRecord::new("img-vec-001", MediaType::Image)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF, 0xD8])
            .with_text_content("Cat on roof")
            .with_embedding("clip_image", vec![0.8, 0.1, 0.0, 0.1])
            .with_tag("animal");

        backend.put_multimodal(&rec).unwrap();

        // Verify we can read it back with embeddings via JSON path
        let result = backend.get_multimodal("img-vec-001", true).unwrap().unwrap();
        assert_eq!(result.id, "img-vec-001");
        let emb = result.embeddings.get("clip_image").unwrap();
        assert_eq!(emb.len(), dim);
        assert!((emb[0] - 0.8).abs() < 1e-5);
    }

    #[test]
    fn test_brute_force_search_with_vec_config() {
        // When vec_config is set but no index exists, search_multimodal
        // should still work via brute-force fallback
        let tmp = TempDir::new().unwrap();
        let dim = 4;
        let backend = make_backend_with_vectors(&tmp, dim);

        let records = vec![
            MultimodalRecord::new("cat-01", MediaType::Image)
                .with_mime_type("image/jpeg")
                .with_data(vec![0xFF])
                .with_embedding("clip_image", vec![0.9, 0.1, 0.0, 0.0]),
            MultimodalRecord::new("dog-01", MediaType::Image)
                .with_mime_type("image/png")
                .with_data(vec![0x89])
                .with_embedding("clip_image", vec![0.1, 0.9, 0.0, 0.0]),
            MultimodalRecord::new("car-01", MediaType::Image)
                .with_mime_type("image/jpeg")
                .with_data(vec![0xFF])
                .with_embedding("clip_image", vec![0.0, 0.0, 0.9, 0.1]),
        ];

        for r in &records {
            backend.put_multimodal(r).unwrap();
        }

        // No index created yet — should use brute-force
        let has_idx = backend.has_multimodal_index("clip_image").unwrap();
        assert!(!has_idx, "No index should exist before create_multimodal_index");

        // Search: cat-like query should find cat first
        let query = vec![0.85, 0.15, 0.0, 0.0];
        let results = backend.search_multimodal(&query, "clip_image", 3, None).unwrap();
        assert_eq!(results.len(), 3);
        assert_eq!(results[0].id, "cat-01");
        assert!(results[0].score > 0.9);
    }

    #[test]
    fn test_create_index_and_ann_search() {
        // T60.2 + T60.3: Create index on native vector column, then search
        use crate::storage::vector_backend::{VectorIndexConfig, VectorDistanceMetric};

        let tmp = TempDir::new().unwrap();
        let dim = 4;
        let backend = make_backend_with_vectors(&tmp, dim);

        // Need enough data for PQ training (at least 256 rows)
        let records: Vec<MultimodalRecord> = (0..300).map(|i| {
            // Generate diverse vectors for a meaningful index
            let angle = (i as f32) * std::f32::consts::PI / 10.0;
            let vec = vec![angle.cos(), angle.sin(), 0.1 * (i as f32), 0.05];
            MultimodalRecord::new(&format!("item-{:03}", i), MediaType::Image)
                .with_mime_type("image/jpeg")
                .with_data(vec![0xFF])
                .with_embedding("clip_image", vec)
                .with_tag("test-ann")
        }).collect();

        for r in &records {
            backend.put_multimodal(r).unwrap();
        }

        // Verify no index initially
        assert!(!backend.has_multimodal_index("clip_image").unwrap());

        // Create IVF_PQ index
        let idx_config = VectorIndexConfig {
            metric: VectorDistanceMetric::Cosine,
            num_partitions: Some(2),
            num_sub_vectors: Some(2),
            ..Default::default()
        };

        backend.create_multimodal_index("clip_image", &idx_config).unwrap();

        // Verify index exists
        assert!(backend.has_multimodal_index("clip_image").unwrap());

        // ANN search with the index
        let query = vec![1.0, 0.0, 0.0, 0.0]; // similar to item-000
        let results = backend.search_multimodal(&query, "clip_image", 5, None).unwrap();

        // Should return results (ANN or brute-force fallback)
        assert!(!results.is_empty(), "ANN search should return results");
        assert!(results.len() <= 5);

        // Top result should have a reasonable score
        assert!(results[0].score > 0.0, "Top result should have positive score");
    }

    #[test]
    fn test_index_info_reports_columns() {
        // T60.3: multimodal_index_info() reports vector column status
        let tmp = TempDir::new().unwrap();
        let dim = 4;
        let backend = make_backend_with_vectors(&tmp, dim);

        let infos = backend.multimodal_index_info().unwrap();
        assert_eq!(infos.len(), 1);
        assert_eq!(infos[0].embedding_name, "clip_image");
        assert_eq!(infos[0].column_name, "vec_clip_image");
        assert_eq!(infos[0].dimension, dim);
        assert!(!infos[0].has_index); // No index yet
    }

    /// Helper: populate backend with N records for index tests.
    /// PQ training requires ≥ 256 rows, so we generate enough data.
    fn populate_for_index(backend: &LanceMultimodalBackend, n: usize, dim: usize) {
        for i in 0..n {
            let angle = (i as f32) * std::f32::consts::PI * 2.0 / (n as f32);
            let mut vec = vec![angle.cos(), angle.sin()];
            // Pad to requested dimension
            while vec.len() < dim {
                vec.push(0.1 * ((i % 10) as f32));
            }
            let rec = MultimodalRecord::new(&format!("r-{:04}", i), MediaType::Image)
                .with_mime_type("image/jpeg")
                .with_data(vec![0xFF])
                .with_embedding("clip_image", vec);
            backend.put_multimodal(&rec).unwrap();
        }
    }

    #[test]
    fn test_rebuild_multimodal_index() {
        // T60.3: rebuild (re-create) index replaces existing index
        use crate::storage::vector_backend::{VectorIndexConfig, VectorDistanceMetric};

        let tmp = TempDir::new().unwrap();
        let dim = 4;
        let backend = make_backend_with_vectors(&tmp, dim);

        // Need >= 256 rows for PQ training
        populate_for_index(&backend, 300, dim);

        let idx_config = VectorIndexConfig {
            metric: VectorDistanceMetric::Cosine,
            num_partitions: Some(2),
            num_sub_vectors: Some(2),
            ..Default::default()
        };

        // Create initial index
        backend.create_multimodal_index("clip_image", &idx_config).unwrap();
        assert!(backend.has_multimodal_index("clip_image").unwrap());

        // Rebuild (re-create with replace=true) should succeed
        backend.create_multimodal_index("clip_image", &idx_config).unwrap();
        assert!(backend.has_multimodal_index("clip_image").unwrap());
    }

    #[test]
    fn test_ann_vs_brute_force_consistency() {
        // T60.5: Verify ANN and brute-force return overlapping top results
        use crate::storage::vector_backend::{VectorIndexConfig, VectorDistanceMetric};

        let tmp = TempDir::new().unwrap();
        let dim = 4;
        let backend = make_backend_with_vectors(&tmp, dim);

        // Create well-separated clusters with enough data for PQ (>= 256 rows)
        // Cluster A: high x, Cluster B: high z, Cluster C: high y, plus fillers
        let mut records: Vec<MultimodalRecord> = Vec::new();

        // 3 named cluster-a points (very high x)
        for i in 0..3 {
            let x = 0.95 - (i as f32) * 0.05;
            records.push(
                MultimodalRecord::new(&format!("cluster-a-{}", i), MediaType::Image)
                    .with_mime_type("image/jpeg")
                    .with_data(vec![0xFF])
                    .with_embedding("clip_image", vec![x, 1.0 - x, 0.0, 0.0]),
            );
        }
        // 3 named cluster-b points (very high z)
        for i in 0..3 {
            let z = 0.95 - (i as f32) * 0.05;
            records.push(
                MultimodalRecord::new(&format!("cluster-b-{}", i), MediaType::Image)
                    .with_mime_type("image/jpeg")
                    .with_data(vec![0xFF])
                    .with_embedding("clip_image", vec![0.0, 0.0, z, 1.0 - z]),
            );
        }
        // Fill up to 300 rows with random-ish vectors
        for i in 0..294 {
            let angle = (i as f32) * std::f32::consts::PI * 2.0 / 294.0;
            records.push(
                MultimodalRecord::new(&format!("filler-{:04}", i), MediaType::Image)
                    .with_mime_type("image/jpeg")
                    .with_data(vec![0xFF])
                    .with_embedding("clip_image", vec![
                        angle.cos() * 0.3 + 0.5,
                        angle.sin() * 0.3 + 0.5,
                        0.1 * ((i % 5) as f32),
                        0.1 * ((i % 7) as f32),
                    ]),
            );
        }

        for rec in &records {
            backend.put_multimodal(rec).unwrap();
        }

        // Brute-force search (before index)
        let query = vec![0.97, 0.03, 0.0, 0.0]; // very close to cluster A
        let bf_results = backend.search_multimodal(&query, "clip_image", 3, None).unwrap();
        assert!(bf_results.len() >= 3);
        // Top result should be cluster-a
        assert!(
            bf_results[0].id.starts_with("cluster-a"),
            "Brute-force top result should be cluster-a, got {}",
            bf_results[0].id
        );

        // Create index
        let idx_config = VectorIndexConfig {
            metric: VectorDistanceMetric::Cosine,
            num_partitions: Some(4),
            num_sub_vectors: Some(2),
            ..Default::default()
        };
        backend.create_multimodal_index("clip_image", &idx_config).unwrap();

        // ANN search (after index)
        let ann_results = backend.search_multimodal(&query, "clip_image", 3, None).unwrap();
        assert!(!ann_results.is_empty(), "ANN should return results");

        // Verify ANN top result is from cluster A
        assert!(
            ann_results[0].id.starts_with("cluster-a"),
            "ANN top result should be from cluster-a, got {}",
            ann_results[0].id
        );
    }

    #[test]
    fn test_search_nonexistent_embedding_column() {
        // Edge case: search with embedding name not in vec_config
        let tmp = TempDir::new().unwrap();
        let dim = 4;
        let backend = make_backend_with_vectors(&tmp, dim);

        let rec = MultimodalRecord::new("x-001", MediaType::Image)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF])
            .with_embedding("some_other", vec![0.5, 0.5, 0.0, 0.0]);
        backend.put_multimodal(&rec).unwrap();

        // Search for "some_other" — not in vec_config, falls back to brute-force via JSON
        let query = vec![0.5, 0.5, 0.0, 0.0];
        let results = backend.search_multimodal(&query, "some_other", 3, None).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, "x-001");
    }

    #[test]
    fn test_create_index_without_vec_config_fails() {
        // Edge case: create index on backend without vec_config
        use crate::storage::vector_backend::VectorIndexConfig;

        let tmp = TempDir::new().unwrap();
        let backend = make_backend(&tmp); // no vec_config

        let rec = MultimodalRecord::new("no-vec", MediaType::Image)
            .with_mime_type("image/jpeg")
            .with_data(vec![0xFF])
            .with_embedding("clip_image", vec![0.5, 0.5, 0.0, 0.0]);
        backend.put_multimodal(&rec).unwrap();

        let result = backend.create_multimodal_index("clip_image", &VectorIndexConfig::default());
        assert!(result.is_err(), "Creating index without vec_config should fail");
    }
}
