//! Tantivy-based full-text search index implementation
//!
//! Phase-73 enhanced — provides BM25-ranked full-text search with:
//! - Chinese tokenization (jieba-rs) with user dictionary support
//! - Synonym expansion (bidirectional / unidirectional)
//! - Stop word filtering (built-in + custom)
//! - English stemming (Snowball)
//! - Per-field relevance boost (BoostQuery)
//! - Multi-language support (Chinese / English, extensible to JP/KR)
//! - Hot reload for synonyms, dictionaries, and stop words
//! - Phrase search, fuzzy matching, result snippet highlighting
//! - Disk-persisted inverted index

use std::collections::{HashMap, HashSet};
use std::io::BufRead;
use std::path::Path;
use std::sync::Arc;

use parking_lot::RwLock;
use tantivy::collector::TopDocs;
use tantivy::query::{
    BooleanQuery, BoostQuery, FuzzyTermQuery, Occur, PhraseQuery, QueryParser, TermQuery,
};
use tantivy::schema::*;
use tantivy::tokenizer::{NgramTokenizer, TextAnalyzer, RemoveLongFilter, LowerCaser, Stemmer, Language};
use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Searcher, Term};

use super::synonym::SynonymMap;
use super::traits::*;

/// Tantivy-based full-text search index (Phase-73 enhanced)
///
/// Uses BM25 scoring with synonym expansion, stop word filtering, stemming,
/// field-level boosting, and Chinese/multi-language tokenization.
pub struct TantivyFullTextIndex {
    index: Index,
    reader: IndexReader,
    writer: Arc<RwLock<IndexWriter>>,
    schema: Schema,
    config: FTSConfig,
    // Field handles for quick access
    field_id: Field,
    field_content: Field,
    field_tags: Field,
    field_metadata: Field,
    // ─── Phase-73 additions ─────────────────────────────────
    /// Synonym map for query expansion (T73.1 + T73.6 hot-reloadable)
    synonym_map: Arc<RwLock<SynonymMap>>,
    /// Stop words set (T73.3 + T73.6 hot-reloadable)
    stop_words: Arc<RwLock<HashSet<String>>>,
}

/// Name of the custom Chinese tokenizer registered with Tantivy
const CHINESE_TOKENIZER: &str = "chinese";
/// Name of the ngram tokenizer (fallback when jieba is not used)
const NGRAM_TOKENIZER: &str = "ngram3";

// ─── Default stop words ─────────────────────────────────────────────────

/// Common Chinese stop words (Phase-73 T73.3)
const DEFAULT_CHINESE_STOP_WORDS: &[&str] = &[
    "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
    "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
    "没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那", "被",
    "从", "对", "又", "与", "把", "能", "让", "因为", "所以", "但是",
    "而且", "或者", "如果", "虽然", "可以", "这个", "那个", "什么",
    "怎么", "为什么", "哪里", "哪个", "已经", "可能", "应该", "还是",
    "之", "而", "以", "及", "其", "等", "于", "中", "为",
];

/// Common English stop words (Phase-73 T73.3)
const DEFAULT_ENGLISH_STOP_WORDS: &[&str] = &[
    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
    "of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
    "being", "have", "has", "had", "do", "does", "did", "will", "would",
    "could", "should", "may", "might", "shall", "can", "need", "dare",
    "it", "its", "i", "me", "my", "we", "our", "you", "your", "he", "him",
    "his", "she", "her", "they", "them", "their", "this", "that", "these",
    "those", "not", "no", "nor", "so", "if", "then", "than", "too", "very",
    "just", "about", "above", "after", "again", "all", "also", "am", "any",
];

impl TantivyFullTextIndex {
    /// Create or open a Tantivy full-text index at the given path
    pub fn open(config: FTSConfig) -> FTSResult<Self> {
        let index_path = Path::new(&config.index_path);

        // Resolve effective language
        let lang = config.effective_language();
        let use_chinese = matches!(lang, TokenizerLanguage::Chinese);

        // Build schema
        let schema = Self::build_schema(use_chinese);

        // Get field handles
        let field_id = schema.get_field("id").unwrap();
        let field_content = schema.get_field("content").unwrap();
        let field_tags = schema.get_field("tags").unwrap();
        let field_metadata = schema.get_field("metadata").unwrap();

        // Open or create index
        let index = if index_path.exists() && index_path.join("meta.json").exists() {
            Index::open_in_dir(index_path).map_err(|e| FTSError::Internal(e.to_string()))?
        } else {
            std::fs::create_dir_all(index_path)?;
            Index::create_in_dir(index_path, schema.clone())
                .map_err(|e| FTSError::Internal(e.to_string()))?
        };

        // Register tokenizers (with optional user dict + stop words + stemmer)
        Self::register_tokenizers_enhanced(&index, &config);

        // Create reader with manual reload
        let reader = index
            .reader_builder()
            .reload_policy(ReloadPolicy::Manual)
            .try_into()
            .map_err(|e: tantivy::TantivyError| FTSError::Internal(e.to_string()))?;

        // Create writer
        let num_threads = if config.num_threads == 0 { 1 } else { config.num_threads };
        let writer = index
            .writer_with_num_threads(num_threads, config.heap_size)
            .map_err(|e| FTSError::Internal(e.to_string()))?;

        // Load synonym map (T73.1)
        let synonym_map = if let Some(ref path) = config.synonym_path {
            match SynonymMap::load_from_json(path) {
                Ok(map) => map,
                Err(e) => {
                    eprintln!("[aifastdb-fts] Warning: Failed to load synonym map from '{}': {}", path, e);
                    SynonymMap::new()
                }
            }
        } else {
            SynonymMap::new()
        };

        // Load stop words (T73.3)
        let stop_words = Self::load_stop_words(&config);

        Ok(Self {
            index,
            reader,
            writer: Arc::new(RwLock::new(writer)),
            schema,
            config,
            field_id,
            field_content,
            field_tags,
            field_metadata,
            synonym_map: Arc::new(RwLock::new(synonym_map)),
            stop_words: Arc::new(RwLock::new(stop_words)),
        })
    }

    /// Create an in-memory Tantivy index (for testing)
    pub fn open_in_memory(config: FTSConfig) -> FTSResult<Self> {
        let lang = config.effective_language();
        let use_chinese = matches!(lang, TokenizerLanguage::Chinese);

        let schema = Self::build_schema(use_chinese);

        let field_id = schema.get_field("id").unwrap();
        let field_content = schema.get_field("content").unwrap();
        let field_tags = schema.get_field("tags").unwrap();
        let field_metadata = schema.get_field("metadata").unwrap();

        let index = Index::create_in_ram(schema.clone());
        Self::register_tokenizers_enhanced(&index, &config);

        let reader = index
            .reader_builder()
            .reload_policy(ReloadPolicy::Manual)
            .try_into()
            .map_err(|e: tantivy::TantivyError| FTSError::Internal(e.to_string()))?;

        let writer = index
            .writer_with_num_threads(1, config.heap_size)
            .map_err(|e| FTSError::Internal(e.to_string()))?;

        // Load synonym map
        let synonym_map = if let Some(ref path) = config.synonym_path {
            SynonymMap::load_from_json(path).unwrap_or_default()
        } else {
            SynonymMap::new()
        };

        let stop_words = Self::load_stop_words(&config);

        Ok(Self {
            index,
            reader,
            writer: Arc::new(RwLock::new(writer)),
            schema,
            config,
            field_id,
            field_content,
            field_tags,
            field_metadata,
            synonym_map: Arc::new(RwLock::new(synonym_map)),
            stop_words: Arc::new(RwLock::new(stop_words)),
        })
    }

    // ─── Phase-73 public API ────────────────────────────────────────────

    /// Get a reference to the current synonym map (T73.1)
    pub fn synonym_map(&self) -> &Arc<RwLock<SynonymMap>> {
        &self.synonym_map
    }

    /// Set the synonym map directly (useful for testing) (T73.1)
    pub fn set_synonym_map(&self, map: SynonymMap) {
        *self.synonym_map.write() = map;
    }

    /// Get the stop words set (T73.3)
    pub fn stop_words(&self) -> &Arc<RwLock<HashSet<String>>> {
        &self.stop_words
    }

    // ─── Schema & Tokenizer ─────────────────────────────────────────────

    /// Build the Tantivy schema
    fn build_schema(enable_chinese: bool) -> Schema {
        let mut builder = Schema::builder();

        // ID field — stored, indexed as single token (not tokenized)
        builder.add_text_field("id", STRING | STORED);

        // Content field — the main searchable text
        let content_options = if enable_chinese {
            TextOptions::default()
                .set_indexing_options(
                    TextFieldIndexing::default()
                        .set_tokenizer(CHINESE_TOKENIZER)
                        .set_index_option(IndexRecordOption::WithFreqsAndPositions),
                )
                .set_stored()
        } else {
            TextOptions::default()
                .set_indexing_options(
                    TextFieldIndexing::default()
                        .set_tokenizer("default")
                        .set_index_option(IndexRecordOption::WithFreqsAndPositions),
                )
                .set_stored()
        };
        builder.add_text_field("content", content_options);

        // Tags field — stored and tokenized (space-separated tags)
        builder.add_text_field("tags", TEXT | STORED);

        // Metadata field — stored as JSON text, searchable
        let metadata_options = if enable_chinese {
            TextOptions::default()
                .set_indexing_options(
                    TextFieldIndexing::default()
                        .set_tokenizer(CHINESE_TOKENIZER)
                        .set_index_option(IndexRecordOption::WithFreqs),
                )
                .set_stored()
        } else {
            TextOptions::default()
                .set_indexing_options(
                    TextFieldIndexing::default()
                        .set_tokenizer("default")
                        .set_index_option(IndexRecordOption::WithFreqs),
                )
                .set_stored()
        };
        builder.add_text_field("metadata", metadata_options);

        builder.build()
    }

    /// Enhanced tokenizer registration with stop words + stemmer (Phase-73)
    fn register_tokenizers_enhanced(index: &Index, config: &FTSConfig) {
        let lang = config.effective_language();

        match lang {
            TokenizerLanguage::Chinese | TokenizerLanguage::Auto => {
                // Build jieba tokenizer (with optional user dict — T73.2)
                let jieba_tokenizer = if let Some(ref dict_path) = config.user_dict_path {
                    JiebaTokenizer::with_user_dict(dict_path)
                } else {
                    JiebaTokenizer::new()
                };

                // Build analyzer pipeline: jieba → RemoveLong → LowerCase [→ Stemmer]
                // Note: StopWordFilter is applied at query time via SynonymMap/manual filtering
                // because Tantivy's built-in filters apply at index time. We handle stop words
                // at search time for flexibility and hot-reload support.
                if config.enable_stemmer {
                    let analyzer = TextAnalyzer::builder(jieba_tokenizer)
                        .filter(RemoveLongFilter::limit(40))
                        .filter(LowerCaser)
                        .filter(Stemmer::new(Language::English))
                        .build();
                    index.tokenizers().register(CHINESE_TOKENIZER, analyzer);
                } else {
                    let analyzer = TextAnalyzer::builder(jieba_tokenizer)
                        .filter(RemoveLongFilter::limit(40))
                        .filter(LowerCaser)
                        .build();
                    index.tokenizers().register(CHINESE_TOKENIZER, analyzer);
                }
            }
            TokenizerLanguage::English => {
                // English uses default tokenizer; stemmer added via pipeline
                if config.enable_stemmer {
                    let analyzer = TextAnalyzer::builder(tantivy::tokenizer::SimpleTokenizer::default())
                        .filter(RemoveLongFilter::limit(40))
                        .filter(LowerCaser)
                        .filter(Stemmer::new(Language::English))
                        .build();
                    // Register as "default" is not possible, register as chinese name for schema compat
                    index.tokenizers().register(CHINESE_TOKENIZER, analyzer);
                }
            }
            TokenizerLanguage::Japanese | TokenizerLanguage::Korean => {
                // Placeholder: lindera integration will be added in T73.5
                // For now, fall back to ngram tokenizer for CJK
                eprintln!(
                    "[aifastdb-fts] Warning: {:?} tokenizer not yet available, falling back to ngram",
                    lang
                );
                let ngram = NgramTokenizer::all_ngrams(2, 3).unwrap();
                let analyzer = TextAnalyzer::builder(ngram)
                    .filter(LowerCaser)
                    .build();
                index.tokenizers().register(CHINESE_TOKENIZER, analyzer);
            }
        }

        // Also register an ngram tokenizer as fallback
        let ngram = NgramTokenizer::all_ngrams(2, 4).unwrap();
        let ngram_analyzer = TextAnalyzer::builder(ngram)
            .filter(LowerCaser)
            .build();
        index.tokenizers().register(NGRAM_TOKENIZER, ngram_analyzer);
    }

    /// Load stop words from config (T73.3)
    fn load_stop_words(config: &FTSConfig) -> HashSet<String> {
        if !config.enable_stop_words {
            return HashSet::new();
        }

        let mut words = HashSet::new();

        // Add built-in stop words
        let lang = config.effective_language();
        match lang {
            TokenizerLanguage::Chinese | TokenizerLanguage::Auto => {
                for w in DEFAULT_CHINESE_STOP_WORDS {
                    words.insert(w.to_string());
                }
                // Also add English stop words for mixed-language content
                for w in DEFAULT_ENGLISH_STOP_WORDS {
                    words.insert(w.to_string());
                }
            }
            TokenizerLanguage::English => {
                for w in DEFAULT_ENGLISH_STOP_WORDS {
                    words.insert(w.to_string());
                }
            }
            _ => {
                // For JP/KR, add English stop words as baseline
                for w in DEFAULT_ENGLISH_STOP_WORDS {
                    words.insert(w.to_string());
                }
            }
        }

        // Load custom stop words from file
        if let Some(ref path) = config.custom_stop_words_path {
            match Self::load_stop_words_file(path) {
                Ok(custom) => {
                    words.extend(custom);
                }
                Err(e) => {
                    eprintln!("[aifastdb-fts] Warning: Failed to load custom stop words from '{}': {}", path, e);
                }
            }
        }

        words
    }

    /// Load stop words from a file (one word per line)
    fn load_stop_words_file(path: &str) -> FTSResult<HashSet<String>> {
        let file = std::fs::File::open(path).map_err(|e| {
            FTSError::Io(std::io::Error::new(
                e.kind(),
                format!("Failed to read stop words file '{}': {}", path, e),
            ))
        })?;
        let reader = std::io::BufReader::new(file);
        let mut words = HashSet::new();
        for line in reader.lines() {
            let line = line?;
            let trimmed = line.trim();
            if !trimmed.is_empty() && !trimmed.starts_with('#') {
                words.insert(trimmed.to_string());
            }
        }
        Ok(words)
    }

    // ─── Internal query helpers ─────────────────────────────────────────

    /// Get a searcher for the current reader state
    fn searcher(&self) -> Searcher {
        self.reader.searcher()
    }

    /// Create a QueryParser for the searchable fields, with optional field boosts (T73.4)
    fn query_parser_with_boosts(
        &self,
        fields: Option<&[String]>,
        field_boosts: Option<&HashMap<String, f64>>,
    ) -> QueryParser {
        let search_fields = if let Some(field_names) = fields {
            field_names
                .iter()
                .filter_map(|name| self.schema.get_field(name).ok())
                .collect::<Vec<_>>()
        } else {
            vec![self.field_content, self.field_tags, self.field_metadata]
        };

        let mut parser = QueryParser::for_index(&self.index, search_fields);
        parser.set_conjunction_by_default();

        // Apply field boosts (T73.4)
        if let Some(boosts) = field_boosts {
            for (field_name, &boost) in boosts {
                if let Ok(field) = self.schema.get_field(field_name) {
                    parser.set_field_boost(field, boost as f32);
                }
            }
        }

        parser
    }

    /// Legacy query_parser (backward compat)
    #[allow(dead_code)]
    fn query_parser(&self, fields: Option<&[String]>) -> QueryParser {
        self.query_parser_with_boosts(fields, None)
    }

    /// Resolve effective field boosts: query-level > config-level > none (T73.4)
    fn resolve_field_boosts<'a>(&'a self, query: &'a FTSQuery) -> Option<&'a HashMap<String, f64>> {
        query
            .field_boosts
            .as_ref()
            .or(self.config.default_field_boosts.as_ref())
    }

    /// Apply synonym expansion to a query (T73.1)
    ///
    /// Expands query terms using the synonym map:
    /// - "数据库 高性能" → "(数据库 OR DB OR database) AND (高性能)"
    ///
    /// Returns the original text if no synonyms are found.
    fn expand_query_with_synonyms(&self, text: &str) -> String {
        let map = self.synonym_map.read();
        if map.is_empty() {
            return text.to_string();
        }

        match map.expand_query(text) {
            Some(groups) => {
                // Build expanded query string for Tantivy's query parser
                let parts: Vec<String> = groups
                    .iter()
                    .map(|group| {
                        if group.len() == 1 {
                            group[0].clone()
                        } else {
                            // Wrap synonym group in parentheses with OR
                            let joined = group.join(" OR ");
                            format!("({})", joined)
                        }
                    })
                    .collect();
                parts.join(" ")
            }
            None => text.to_string(),
        }
    }

    /// Filter stop words from a query text (T73.3)
    ///
    /// Removes stop words from the query. If all words are stop words,
    /// returns the original text to avoid empty queries.
    fn filter_stop_words(&self, text: &str) -> String {
        let sw = self.stop_words.read();
        if sw.is_empty() {
            return text.to_string();
        }

        let words: Vec<&str> = text.split_whitespace().collect();
        let filtered: Vec<&str> = words
            .iter()
            .filter(|w| !sw.contains(&w.to_lowercase()))
            .copied()
            .collect();

        if filtered.is_empty() {
            // Don't return empty query — fallback to original
            text.to_string()
        } else {
            filtered.join(" ")
        }
    }

    /// Preprocess query text: stop word filtering → synonym expansion (T73.1 + T73.3)
    fn preprocess_query(&self, text: &str, expand_synonyms: bool) -> String {
        // Step 1: Filter stop words
        let filtered = self.filter_stop_words(text);

        // Step 2: Expand synonyms
        if expand_synonyms {
            self.expand_query_with_synonyms(&filtered)
        } else {
            filtered
        }
    }

    /// Execute search with the given tantivy query
    fn execute_search(
        &self,
        query: &dyn tantivy::query::Query,
        limit: usize,
        offset: usize,
        highlight: bool,
        snippet_max_len: usize,
        search_text: &str,
    ) -> FTSResult<Vec<FTSSearchResult>> {
        let searcher = self.searcher();
        let top_docs = searcher
            .search(query, &TopDocs::with_limit(limit + offset))
            .map_err(|e| FTSError::Internal(e.to_string()))?;

        let mut results = Vec::with_capacity(limit);

        for (i, (score, doc_address)) in top_docs.into_iter().enumerate() {
            if i < offset {
                continue;
            }
            if results.len() >= limit {
                break;
            }

            let doc: tantivy::TantivyDocument = searcher
                .doc(doc_address)
                .map_err(|e| FTSError::Internal(e.to_string()))?;

            // Extract ID
            let id = doc
                .get_first(self.field_id)
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();

            // Generate snippet if requested
            let snippet = if highlight {
                self.generate_snippet(&searcher, query, &doc, snippet_max_len, search_text)
            } else {
                None
            };

            // Determine matched fields
            let mut matched_fields = Vec::new();
            if let Some(content) = doc.get_first(self.field_content).and_then(|v| v.as_str()) {
                let lower = content.to_lowercase();
                let search_lower = search_text.to_lowercase();
                if search_lower.split_whitespace().any(|term| lower.contains(term)) {
                    matched_fields.push("content".to_string());
                }
            }
            if let Some(tags) = doc.get_first(self.field_tags).and_then(|v| v.as_str()) {
                let lower = tags.to_lowercase();
                let search_lower = search_text.to_lowercase();
                if search_lower.split_whitespace().any(|term| lower.contains(term)) {
                    matched_fields.push("tags".to_string());
                }
            }

            results.push(FTSSearchResult {
                id,
                score: score as f64,
                snippet,
                matched_fields,
            });
        }

        Ok(results)
    }

    /// Generate a highlighted snippet from the document
    fn generate_snippet(
        &self,
        searcher: &Searcher,
        query: &dyn tantivy::query::Query,
        doc: &tantivy::TantivyDocument,
        max_len: usize,
        _search_text: &str,
    ) -> Option<String> {
        let snippet_gen = tantivy::SnippetGenerator::create(
            searcher,
            query,
            self.field_content,
        );

        match snippet_gen {
            Ok(mut generator) => {
                generator.set_max_num_chars(max_len);
                let snippet = generator.snippet_from_doc(doc);
                let html = snippet.to_html();
                if html.is_empty() {
                    None
                } else {
                    Some(html)
                }
            }
            Err(_) => None,
        }
    }

    /// Build a phrase query from the search text
    fn build_phrase_query(&self, text: &str, fields: Option<&[String]>) -> FTSResult<Box<dyn tantivy::query::Query>> {
        let target_fields = if let Some(field_names) = fields {
            field_names
                .iter()
                .filter_map(|name| self.schema.get_field(name).ok())
                .collect::<Vec<_>>()
        } else {
            vec![self.field_content]
        };

        // For each target field, create a phrase query and combine with OR
        let mut sub_queries: Vec<(Occur, Box<dyn tantivy::query::Query>)> = Vec::new();

        for field in target_fields {
            // Tokenize the text using the field's tokenizer
            let tokenizer_name = self.schema.get_field_entry(field)
                .field_type()
                .get_index_record_option()
                .map(|_| {
                    if self.config.enable_chinese {
                        CHINESE_TOKENIZER
                    } else {
                        "default"
                    }
                })
                .unwrap_or("default");

            let tokenizer_manager = self.index.tokenizers();
            let mut tokenizer = tokenizer_manager.get(tokenizer_name)
                .ok_or_else(|| FTSError::Internal(format!("Tokenizer '{}' not found", tokenizer_name)))?;

            let mut token_stream = tokenizer.token_stream(text);
            let mut terms = Vec::new();
            while token_stream.advance() {
                let token = token_stream.token();
                terms.push(Term::from_field_text(field, &token.text));
            }

            if terms.len() >= 2 {
                let phrase_query = PhraseQuery::new(terms);
                sub_queries.push((Occur::Should, Box::new(phrase_query)));
            } else if terms.len() == 1 {
                let term_query = TermQuery::new(terms.into_iter().next().unwrap(), IndexRecordOption::WithFreqs);
                sub_queries.push((Occur::Should, Box::new(term_query)));
            }
        }

        if sub_queries.is_empty() {
            return Err(FTSError::QueryParse("No terms extracted from phrase query".into()));
        }

        Ok(Box::new(BooleanQuery::new(sub_queries)))
    }

    /// Build a fuzzy query from the search text
    fn build_fuzzy_query(&self, text: &str, distance: u8, fields: Option<&[String]>) -> FTSResult<Box<dyn tantivy::query::Query>> {
        let target_fields = if let Some(field_names) = fields {
            field_names
                .iter()
                .filter_map(|name| self.schema.get_field(name).ok())
                .collect::<Vec<_>>()
        } else {
            vec![self.field_content, self.field_tags]
        };

        let keywords: Vec<&str> = text.split_whitespace().collect();
        let mut sub_queries: Vec<(Occur, Box<dyn tantivy::query::Query>)> = Vec::new();

        for keyword in keywords {
            for &field in &target_fields {
                let term = Term::from_field_text(field, keyword);
                let fuzzy = FuzzyTermQuery::new(term, distance, true);
                sub_queries.push((Occur::Should, Box::new(fuzzy)));
            }
        }

        if sub_queries.is_empty() {
            return Err(FTSError::QueryParse("No terms for fuzzy query".into()));
        }

        Ok(Box::new(BooleanQuery::new(sub_queries)))
    }

    /// Wrap a query with a BoostQuery if a boost factor > 1.0 is specified (T73.4)
    fn maybe_boost(
        query: Box<dyn tantivy::query::Query>,
        _field_boosts: Option<&HashMap<String, f64>>,
    ) -> Box<dyn tantivy::query::Query> {
        // Field-level boost is handled via QueryParser::set_field_boost.
        // This method is kept for potential future per-query-level boost.
        query
    }
}

// ─── FullTextIndex trait implementation ─────────────────────────────────

impl FullTextIndex for TantivyFullTextIndex {
    fn add_document(&self, ftsdoc: &FTSDocument) -> FTSResult<()> {
        let tags_text = ftsdoc.tags.join(" ");
        let metadata_text = ftsdoc.metadata
            .iter()
            .map(|(k, v)| format!("{}: {}", k, v))
            .collect::<Vec<_>>()
            .join("; ");

        let tantivy_doc = doc!(
            self.field_id => ftsdoc.id.as_str(),
            self.field_content => ftsdoc.content.as_str(),
            self.field_tags => tags_text.as_str(),
            self.field_metadata => metadata_text.as_str()
        );

        let mut writer = self.writer.write();
        writer
            .add_document(tantivy_doc)
            .map_err(|e| FTSError::Internal(e.to_string()))?;

        Ok(())
    }

    fn add_documents(&self, docs: &[FTSDocument]) -> FTSResult<usize> {
        let mut writer = self.writer.write();
        let mut count = 0;

        for ftsdoc in docs {
            let tags_text = ftsdoc.tags.join(" ");
            let metadata_text = ftsdoc.metadata
                .iter()
                .map(|(k, v)| format!("{}: {}", k, v))
                .collect::<Vec<_>>()
                .join("; ");

            let tantivy_doc = doc!(
                self.field_id => ftsdoc.id.as_str(),
                self.field_content => ftsdoc.content.as_str(),
                self.field_tags => tags_text.as_str(),
                self.field_metadata => metadata_text.as_str()
            );

            writer
                .add_document(tantivy_doc)
                .map_err(|e| FTSError::Internal(e.to_string()))?;
            count += 1;
        }

        Ok(count)
    }

    fn delete_document(&self, id: &str) -> FTSResult<()> {
        let term = Term::from_field_text(self.field_id, id);
        let mut writer = self.writer.write();
        writer.delete_term(term);
        Ok(())
    }

    fn search(&self, query: &FTSQuery) -> FTSResult<Vec<FTSSearchResult>> {
        let fields_ref = query.fields.as_ref().map(|f| f.as_slice());
        let field_boosts = self.resolve_field_boosts(query);

        let tantivy_query: Box<dyn tantivy::query::Query> = match &query.mode {
            FTSQueryMode::Standard => {
                // Phase-73: preprocess → stop words + synonym expansion
                let processed_text = self.preprocess_query(&query.text, query.expand_synonyms);
                let parser = self.query_parser_with_boosts(fields_ref, field_boosts);
                let parsed = parser
                    .parse_query(&processed_text)
                    .map_err(|e| FTSError::QueryParse(e.to_string()))?;
                Self::maybe_boost(parsed, field_boosts)
            }
            FTSQueryMode::Phrase => {
                self.build_phrase_query(&query.text, fields_ref)?
            }
            FTSQueryMode::Fuzzy { distance } => {
                self.build_fuzzy_query(&query.text, *distance, fields_ref)?
            }
            FTSQueryMode::Boolean => {
                // Boolean mode: preprocess but don't expand synonyms (user controls query structure)
                let processed_text = self.filter_stop_words(&query.text);
                let parser = self.query_parser_with_boosts(fields_ref, field_boosts);
                let parsed = parser
                    .parse_query(&processed_text)
                    .map_err(|e| FTSError::QueryParse(e.to_string()))?;
                Self::maybe_boost(parsed, field_boosts)
            }
        };

        self.execute_search(
            tantivy_query.as_ref(),
            query.limit,
            query.offset,
            query.highlight,
            query.snippet_max_len,
            &query.text,
        )
    }

    fn doc_count(&self) -> FTSResult<u64> {
        let searcher = self.searcher();
        Ok(searcher.num_docs())
    }

    fn rebuild_index(&self) -> FTSResult<()> {
        // Delete all documents and commit
        let mut writer = self.writer.write();
        writer
            .delete_all_documents()
            .map_err(|e| FTSError::Internal(e.to_string()))?;
        writer
            .commit()
            .map_err(|e| FTSError::Internal(e.to_string()))?;
        drop(writer);

        self.reader
            .reload()
            .map_err(|e| FTSError::Internal(e.to_string()))?;
        Ok(())
    }

    fn commit(&self) -> FTSResult<()> {
        let mut writer = self.writer.write();
        writer
            .commit()
            .map_err(|e| FTSError::Internal(e.to_string()))?;
        drop(writer);

        self.reader
            .reload()
            .map_err(|e| FTSError::Internal(e.to_string()))?;
        Ok(())
    }

    fn backend_name(&self) -> &str {
        "tantivy"
    }

    // ─── Phase-73 hot reload (T73.6) ────────────────────────────────────

    fn reload_synonyms(&self, path: &str) -> FTSResult<usize> {
        let new_map = SynonymMap::load_from_json(path)?;
        let count = new_map.len();
        *self.synonym_map.write() = new_map;
        Ok(count)
    }

    fn reload_user_dict(&self, path: &str) -> FTSResult<()> {
        // Validate the file exists and is readable
        if !Path::new(path).exists() {
            return Err(FTSError::Io(std::io::Error::new(
                std::io::ErrorKind::NotFound,
                format!("User dictionary file not found: {}", path),
            )));
        }

        // Re-register tokenizers with the new user dict
        // Note: this updates new documents' tokenization; existing indexed docs are not re-tokenized
        let mut config_clone = self.config.clone();
        config_clone.user_dict_path = Some(path.to_string());
        Self::register_tokenizers_enhanced(&self.index, &config_clone);

        Ok(())
    }

    fn update_stop_words(&self, words: Vec<String>) -> FTSResult<()> {
        let mut sw = self.stop_words.write();
        sw.clear();
        for w in words {
            sw.insert(w);
        }
        Ok(())
    }

    fn get_config(&self) -> Option<&FTSConfig> {
        Some(&self.config)
    }
}

// ─── Jieba Tokenizer for Tantivy ────────────────────────────────────────

/// A Tantivy Tokenizer that uses jieba-rs for Chinese text segmentation
///
/// Phase-73 T73.2: supports loading user dictionaries for domain-specific terms
#[derive(Clone)]
struct JiebaTokenizer {
    jieba: Arc<jieba_rs::Jieba>,
}

impl JiebaTokenizer {
    /// Create with default dictionary
    fn new() -> Self {
        Self {
            jieba: Arc::new(jieba_rs::Jieba::new()),
        }
    }

    /// Create with custom user dictionary (T73.2)
    ///
    /// User dict format: one entry per line — `词语 词频 词性`
    /// Example: `向量数据库 5 n`
    fn with_user_dict(dict_path: &str) -> Self {
        let mut jieba = jieba_rs::Jieba::new();

        match std::fs::File::open(dict_path) {
            Ok(file) => {
                let reader = std::io::BufReader::new(file);
                match jieba.load_dict(&mut reader.lines().filter_map(|l| l.ok()).collect::<Vec<_>>().join("\n").as_bytes()) {
                    Ok(_) => {
                        // Successfully loaded user dict
                    }
                    Err(e) => {
                        eprintln!("[aifastdb-fts] Warning: Failed to load user dict '{}': {}", dict_path, e);
                    }
                }
            }
            Err(e) => {
                eprintln!("[aifastdb-fts] Warning: Failed to open user dict '{}': {}", dict_path, e);
            }
        }

        Self {
            jieba: Arc::new(jieba),
        }
    }
}

impl tantivy::tokenizer::Tokenizer for JiebaTokenizer {
    type TokenStream<'a> = JiebaTokenStream;

    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
        let words = self.jieba.cut(text, true);
        let mut tokens = Vec::new();
        let mut offset = 0;

        for word in words {
            let trimmed = word.trim();
            if trimmed.is_empty() {
                offset += word.len();
                continue;
            }
            // Skip single ASCII punctuation / whitespace
            if trimmed.len() == 1 && trimmed.as_bytes()[0].is_ascii_punctuation() {
                offset += word.len();
                continue;
            }

            tokens.push(tantivy::tokenizer::Token {
                offset_from: offset,
                offset_to: offset + word.len(),
                position: tokens.len(),
                text: trimmed.to_lowercase(),
                position_length: 1,
            });
            offset += word.len();
        }

        JiebaTokenStream {
            tokens,
            index: 0,
            token: tantivy::tokenizer::Token::default(),
        }
    }
}

struct JiebaTokenStream {
    tokens: Vec<tantivy::tokenizer::Token>,
    index: usize,
    token: tantivy::tokenizer::Token,
}

impl tantivy::tokenizer::TokenStream for JiebaTokenStream {
    fn advance(&mut self) -> bool {
        if self.index < self.tokens.len() {
            self.token = self.tokens[self.index].clone();
            self.index += 1;
            true
        } else {
            false
        }
    }

    fn token(&self) -> &tantivy::tokenizer::Token {
        &self.token
    }

    fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
        &mut self.token
    }
}

// ─── Tests ──────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::HashMap;

    fn create_test_index() -> TantivyFullTextIndex {
        TantivyFullTextIndex::open_in_memory(FTSConfig::default()).unwrap()
    }

    fn make_doc(id: &str, content: &str, tags: Vec<&str>) -> FTSDocument {
        FTSDocument {
            id: id.to_string(),
            content: content.to_string(),
            tags: tags.into_iter().map(|s| s.to_string()).collect(),
            metadata: HashMap::new(),
        }
    }

    // ─── Original tests (preserved) ─────────────────────────────────────

    #[test]
    fn test_add_and_search_basic() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "The quick brown fox jumps over the lazy dog", vec!["animal"])).unwrap();
        index.add_document(&make_doc("2", "Rust is a systems programming language", vec!["programming"])).unwrap();
        index.add_document(&make_doc("3", "The fox and the hound are friends", vec!["animal"])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("fox")).unwrap();
        assert_eq!(results.len(), 2);
        let ids: Vec<&str> = results.iter().map(|r| r.id.as_str()).collect();
        assert!(ids.contains(&"1"));
        assert!(ids.contains(&"3"));
    }

    #[test]
    fn test_bm25_ranking() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "Rust programming in Rust is amazing", vec!["rust"])).unwrap();
        index.add_document(&make_doc("2", "Python is great for scripting", vec!["python"])).unwrap();
        index.add_document(&make_doc("3", "Rust is fast", vec!["rust"])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("rust")).unwrap();
        assert!(results.len() >= 2);
        assert_eq!(results[0].id, "1");
    }

    #[test]
    fn test_chinese_search() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "人工智能数据库是未来的趋势", vec!["技术"])).unwrap();
        index.add_document(&make_doc("2", "深度学习模型需要大量数据", vec!["AI"])).unwrap();
        index.add_document(&make_doc("3", "高性能数据库引擎使用Rust开发", vec!["数据库"])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("数据库")).unwrap();
        assert!(results.len() >= 2, "Expected at least 2 results for '数据库', got {}", results.len());
        let ids: Vec<&str> = results.iter().map(|r| r.id.as_str()).collect();
        assert!(ids.contains(&"1") || ids.contains(&"3"));
    }

    #[test]
    fn test_phrase_search() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "quick brown fox", vec![])).unwrap();
        index.add_document(&make_doc("2", "brown quick fox", vec![])).unwrap();
        index.add_document(&make_doc("3", "the quick brown fox jumps", vec![])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::phrase("quick brown")).unwrap();
        let ids: Vec<&str> = results.iter().map(|r| r.id.as_str()).collect();
        assert!(ids.contains(&"1") || ids.contains(&"3"));
    }

    #[test]
    fn test_fuzzy_search() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "programming language", vec![])).unwrap();
        index.add_document(&make_doc("2", "database engine", vec![])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::fuzzy("programing", 1)).unwrap();
        assert!(!results.is_empty(), "Fuzzy search should find 'programming' for 'programing'");
        assert_eq!(results[0].id, "1");
    }

    #[test]
    fn test_delete_document() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "hello world", vec![])).unwrap();
        index.add_document(&make_doc("2", "hello rust", vec![])).unwrap();
        index.commit().unwrap();

        assert_eq!(index.doc_count().unwrap(), 2);

        index.delete_document("1").unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("hello")).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, "2");
    }

    #[test]
    fn test_update_document() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "old content about cats", vec![])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("cats")).unwrap();
        assert_eq!(results.len(), 1);

        index.update_document(&make_doc("1", "new content about dogs", vec![])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("cats")).unwrap();
        assert_eq!(results.len(), 0);

        let results = index.search(&FTSQuery::new("dogs")).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, "1");
    }

    #[test]
    fn test_batch_add() {
        let index = create_test_index();

        let docs: Vec<FTSDocument> = (0..100)
            .map(|i| make_doc(&format!("doc-{}", i), &format!("Document number {} about various topics", i), vec!["batch"]))
            .collect();

        let count = index.add_documents(&docs).unwrap();
        assert_eq!(count, 100);
        index.commit().unwrap();

        assert_eq!(index.doc_count().unwrap(), 100);

        let results = index.search(&FTSQuery::new("document topics")).unwrap();
        assert!(!results.is_empty());
    }

    #[test]
    fn test_tag_search() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "some content", vec!["rust", "database"])).unwrap();
        index.add_document(&make_doc("2", "other content", vec!["python", "ml"])).unwrap();
        index.commit().unwrap();

        let results = index.search(
            &FTSQuery::new("rust").with_fields(vec!["tags".into()])
        ).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, "1");
    }

    #[test]
    fn test_snippet_generation() {
        let index = create_test_index();

        let long_content = "Rust is a multi-paradigm, general-purpose programming language. \
            Rust emphasizes performance, type safety, and concurrency. \
            Rust enforces memory safety without using a garbage collector.";
        index.add_document(&make_doc("1", long_content, vec![])).unwrap();
        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("rust programming")).unwrap();
        assert!(!results.is_empty());
        if let Some(snippet) = &results[0].snippet {
            assert!(snippet.contains("<b>") || snippet.len() > 0);
        }
    }

    #[test]
    fn test_empty_index_search() {
        let index = create_test_index();

        let results = index.search(&FTSQuery::new("anything")).unwrap();
        assert!(results.is_empty());
        assert_eq!(index.doc_count().unwrap(), 0);
    }

    #[test]
    fn test_rebuild_index() {
        let index = create_test_index();

        index.add_document(&make_doc("1", "hello world", vec![])).unwrap();
        index.add_document(&make_doc("2", "hello rust", vec![])).unwrap();
        index.commit().unwrap();

        assert_eq!(index.doc_count().unwrap(), 2);

        index.rebuild_index().unwrap();
        assert_eq!(index.doc_count().unwrap(), 0);
    }

    #[test]
    fn test_pagination() {
        let index = create_test_index();

        for i in 0..20 {
            index.add_document(&make_doc(
                &format!("doc-{}", i),
                &format!("test document about topic {}", i),
                vec![],
            )).unwrap();
        }
        index.commit().unwrap();

        let page1 = index.search(
            &FTSQuery::new("test document").with_limit(5).with_offset(0)
        ).unwrap();
        assert_eq!(page1.len(), 5);

        let page2 = index.search(
            &FTSQuery::new("test document").with_limit(5).with_offset(5)
        ).unwrap();
        assert_eq!(page2.len(), 5);

        let page1_ids: Vec<&str> = page1.iter().map(|r| r.id.as_str()).collect();
        let page2_ids: Vec<&str> = page2.iter().map(|r| r.id.as_str()).collect();
        for id in &page2_ids {
            assert!(!page1_ids.contains(id), "Page 2 should not overlap with page 1");
        }
    }

    #[test]
    fn test_metadata_search() {
        let index = create_test_index();

        let mut metadata1 = HashMap::new();
        metadata1.insert("author".to_string(), "Alice".to_string());
        metadata1.insert("category".to_string(), "science".to_string());

        let mut metadata2 = HashMap::new();
        metadata2.insert("author".to_string(), "Bob".to_string());
        metadata2.insert("category".to_string(), "technology".to_string());

        index.add_document(&FTSDocument {
            id: "1".to_string(),
            content: "An article about physics".to_string(),
            tags: vec![],
            metadata: metadata1,
        }).unwrap();

        index.add_document(&FTSDocument {
            id: "2".to_string(),
            content: "An article about coding".to_string(),
            tags: vec![],
            metadata: metadata2,
        }).unwrap();

        index.commit().unwrap();

        let results = index.search(&FTSQuery::new("Alice")).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].id, "1");
    }

    #[test]
    fn test_backend_name() {
        let index = create_test_index();
        assert_eq!(index.backend_name(), "tantivy");
    }

    // ─── Phase-73 new tests ─────────────────────────────────────────────

    #[test]
    fn test_synonym_expansion_search() {
        let index = create_test_index();

        // Set up synonyms: 数据库 ↔ DB ↔ database
        let mut syn_map = SynonymMap::new();
        syn_map.add_bidirectional("数据库", vec!["db", "database"]);
        index.set_synonym_map(syn_map);

        index.add_document(&make_doc("1", "这是一个高性能的数据库引擎", vec![])).unwrap();
        index.add_document(&make_doc("2", "This is a fast DB engine", vec![])).unwrap();
        index.add_document(&make_doc("3", "Python is a programming language", vec![])).unwrap();
        index.commit().unwrap();

        // Search for "database" should also find "数据库" and "DB" docs
        let results = index.search(&FTSQuery::new("database")).unwrap();
        assert!(results.len() >= 2, "Synonym expansion should find at least 2 results, got {}", results.len());
    }

    #[test]
    fn test_synonym_disabled_per_query() {
        let index = create_test_index();

        let mut syn_map = SynonymMap::new();
        syn_map.add_bidirectional("hello", vec!["hi", "hey"]);
        index.set_synonym_map(syn_map);

        index.add_document(&make_doc("1", "hello world", vec![])).unwrap();
        index.add_document(&make_doc("2", "hi there", vec![])).unwrap();
        index.commit().unwrap();

        // With synonyms disabled, "hello" should not match "hi"
        let results = index.search(&FTSQuery::new("hello").no_synonym_expansion()).unwrap();
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].id, "1");
    }

    #[test]
    fn test_stop_words_filtering() {
        // Create index with stop words enabled
        let config = FTSConfig::default().with_stop_words();
        let index = TantivyFullTextIndex::open_in_memory(config).unwrap();

        index.add_document(&make_doc("1", "Rust programming language", vec![])).unwrap();
        index.add_document(&make_doc("2", "Python scripting tool", vec![])).unwrap();
        index.commit().unwrap();

        // "is a" are stop words — should be filtered, searching for "Rust"
        let results = index.search(&FTSQuery::new("Rust is a language")).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].id, "1");
    }

    #[test]
    fn test_field_boost_tags_priority() {
        let index = create_test_index();

        // Doc 1: "rust" in content only
        index.add_document(&make_doc("1", "rust is a programming language", vec!["systems"])).unwrap();
        // Doc 2: "rust" in tags (should rank higher with tag boost)
        index.add_document(&make_doc("2", "a programming language", vec!["rust"])).unwrap();
        index.commit().unwrap();

        // Boost tags field
        let mut boosts = HashMap::new();
        boosts.insert("tags".to_string(), 3.0);
        boosts.insert("content".to_string(), 1.0);

        let results = index.search(
            &FTSQuery::new("rust").with_field_boosts(boosts)
        ).unwrap();

        assert!(results.len() >= 2);
        // Doc 2 should rank higher because "rust" is in boosted tags field
        assert_eq!(results[0].id, "2", "Tag-boosted doc should rank first");
    }

    #[test]
    fn test_config_default_field_boosts() {
        let mut default_boosts = HashMap::new();
        default_boosts.insert("tags".to_string(), 2.0);

        let config = FTSConfig::default().with_default_boosts(default_boosts.clone());
        let index = TantivyFullTextIndex::open_in_memory(config).unwrap();

        // The config should have default boosts set
        assert!(index.get_config().unwrap().default_field_boosts.is_some());
    }

    #[test]
    fn test_update_stop_words_at_runtime() {
        let config = FTSConfig::default().with_stop_words();
        let index = TantivyFullTextIndex::open_in_memory(config).unwrap();

        // Initially has default stop words
        assert!(!index.stop_words().read().is_empty());

        // Update to custom stop words
        index.update_stop_words(vec!["custom".to_string(), "stop".to_string()]).unwrap();

        let sw = index.stop_words().read();
        assert_eq!(sw.len(), 2);
        assert!(sw.contains("custom"));
        assert!(sw.contains("stop"));
    }

    #[test]
    fn test_reload_synonyms_from_json() {
        let index = create_test_index();
        assert!(index.synonym_map().read().is_empty());

        // Write a temp synonym file
        let dir = tempfile::tempdir().unwrap();
        let syn_file = dir.path().join("synonyms.json");
        std::fs::write(&syn_file, r#"{
            "entries": [
                { "term": "fast", "synonyms": ["quick", "rapid"], "bidirectional": true }
            ]
        }"#).unwrap();

        let count = index.reload_synonyms(syn_file.to_str().unwrap()).unwrap();
        assert!(count > 0);
        assert!(!index.synonym_map().read().is_empty());
    }

    #[test]
    fn test_english_language_config() {
        let config = FTSConfig::default()
            .with_language(TokenizerLanguage::English)
            .with_stemmer();
        let index = TantivyFullTextIndex::open_in_memory(config).unwrap();

        index.add_document(&make_doc("1", "running and jumping", vec![])).unwrap();
        index.commit().unwrap();

        // With English stemmer, "run" should match "running"
        let results = index.search(&FTSQuery::new("run")).unwrap();
        // Note: stemmer matches depend on exact stemming; this tests that config works
        assert_eq!(index.backend_name(), "tantivy");
    }

    #[test]
    fn test_get_config() {
        let index = create_test_index();
        let config = index.get_config().unwrap();
        assert!(config.enable_chinese);
        assert_eq!(config.language, TokenizerLanguage::Auto);
    }

    #[test]
    fn test_effective_language() {
        let config = FTSConfig::default();
        assert_eq!(config.effective_language(), TokenizerLanguage::Chinese);

        let config_en = FTSConfig {
            enable_chinese: false,
            ..Default::default()
        };
        assert_eq!(config_en.effective_language(), TokenizerLanguage::English);

        let config_jp = FTSConfig::default().with_language(TokenizerLanguage::Japanese);
        assert_eq!(config_jp.effective_language(), TokenizerLanguage::Japanese);
    }

    #[test]
    fn test_stop_words_dont_empty_query() {
        let config = FTSConfig::default().with_stop_words();
        let index = TantivyFullTextIndex::open_in_memory(config).unwrap();

        index.add_document(&make_doc("1", "the is a", vec![])).unwrap();
        index.commit().unwrap();

        // Query with only stop words should fallback to original text, not empty
        let results = index.search(&FTSQuery::new("the is"));
        // Should not error even if all words are stop words
        assert!(results.is_ok());
    }
}
