// Obsidian [[wikilinks]] in doc comments are intentional — they link to
// design and reference docs in docs/. Rustdoc doesn't understand them.
#![allow(rustdoc::broken_intra_doc_links)]

//! `luci-analysis` — text analysis pipeline for Luci.
//!
//! Transforms raw text into indexed terms via a three-stage pipeline:
//!
//! ```text
//! Raw Text → Tokenizer → Token Filters → Indexed Terms
//! ```
//!
//! Provides the `Tokenizer` and `TokenFilter` traits with built-in
//! implementations matching Elasticsearch's analyzer model: `standard`,
//! `simple`, `whitespace`, and `keyword` analyzers.
//!
//! See [[analyzers]] for the full specification.

mod filter;
mod token;
mod tokenizer;

pub use filter::{LowercaseFilter, StemmerAlgorithm, StemmerFilter, StopFilter, TokenFilter};
pub use token::Token;
pub use tokenizer::{KeywordTokenizer, LetterTokenizer, StandardTokenizer, Tokenizer, WhitespaceTokenizer};

use std::collections::HashMap;

/// A complete text analysis pipeline: tokenizer + token filters.
///
/// Combines a single [`Tokenizer`] with zero or more [`TokenFilter`]s.
/// The tokenizer breaks text into tokens, then each filter transforms the
/// token stream in order.
///
/// See [[analyzers#Pipeline Stages]].
pub struct Analyzer {
    name: String,
    tokenizer: Box<dyn Tokenizer>,
    filters: Vec<Box<dyn TokenFilter>>,
}

impl Analyzer {
    /// Create a new analyzer with the given name, tokenizer, and filters.
    pub fn new(
        name: impl Into<String>,
        tokenizer: impl Tokenizer + 'static,
        filters: Vec<Box<dyn TokenFilter>>,
    ) -> Self {
        Self {
            name: name.into(),
            tokenizer: Box::new(tokenizer),
            filters,
        }
    }

    /// Run the full analysis pipeline on the input text.
    ///
    /// Tokenizes the text, then applies each filter in order.
    pub fn analyze(&self, text: &str) -> Vec<Token> {
        let mut tokens = Vec::new();
        self.tokenizer.tokenize(text, &mut tokens);
        for filter in &self.filters {
            filter.apply(&mut tokens);
        }
        tokens
    }

    /// The analyzer's name (e.g., `"standard"`, `"simple"`).
    pub fn name(&self) -> &str {
        &self.name
    }
}

/// Registry of named analyzers with fallback resolution.
///
/// Implements the analyzer resolution chain from [[analyzers#Analyzer Resolution]]:
/// 1. Look up by exact name
/// 2. Fall back to the `standard` analyzer
///
/// All built-in analyzers are registered on construction.
pub struct AnalyzerRegistry {
    analyzers: HashMap<String, Analyzer>,
}

impl AnalyzerRegistry {
    /// Create a new registry with all built-in analyzers pre-registered.
    pub fn new() -> Self {
        let mut registry = Self {
            analyzers: HashMap::new(),
        };
        registry.register(standard_analyzer());
        registry.register(simple_analyzer());
        registry.register(whitespace_analyzer());
        registry.register(keyword_analyzer());
        registry.register(stop_analyzer());
        registry
    }

    /// Register a custom analyzer. Overwrites any existing analyzer with
    /// the same name.
    pub fn register(&mut self, analyzer: Analyzer) {
        self.analyzers.insert(analyzer.name.clone(), analyzer);
    }

    /// Look up an analyzer by name, falling back to `standard`.
    pub fn get(&self, name: &str) -> &Analyzer {
        self.analyzers
            .get(name)
            .unwrap_or_else(|| self.analyzers.get("standard").unwrap())
    }

    /// Look up an analyzer by name. Returns `None` if not found.
    pub fn try_get(&self, name: &str) -> Option<&Analyzer> {
        self.analyzers.get(name)
    }

    /// List all registered analyzer names.
    pub fn names(&self) -> Vec<&str> {
        self.analyzers.keys().map(String::as_str).collect()
    }
}

impl Default for AnalyzerRegistry {
    fn default() -> Self {
        Self::new()
    }
}

// --- Built-in analyzer constructors ---

/// `standard` analyzer: UAX#29 tokenizer + lowercase filter.
///
/// The default analyzer for `text` fields.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn standard_analyzer() -> Analyzer {
    Analyzer::new(
        "standard",
        StandardTokenizer,
        vec![Box::new(LowercaseFilter)],
    )
}

/// `simple` analyzer: letter tokenizer + lowercase filter.
///
/// Splits on non-letter characters and lowercases.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn simple_analyzer() -> Analyzer {
    Analyzer::new(
        "simple",
        LetterTokenizer,
        vec![Box::new(LowercaseFilter)],
    )
}

/// `whitespace` analyzer: whitespace tokenizer, no filters.
///
/// Splits on whitespace only, preserving case and punctuation.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn whitespace_analyzer() -> Analyzer {
    Analyzer::new("whitespace", WhitespaceTokenizer, vec![])
}

/// `keyword` analyzer: keyword tokenizer, no filters.
///
/// Emits the entire input as a single token. Used for exact-match fields.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn keyword_analyzer() -> Analyzer {
    Analyzer::new("keyword", KeywordTokenizer, vec![])
}

/// `stop` analyzer: UAX#29 tokenizer + lowercase + English stop words.
///
/// Like `standard` but removes common English stop words.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn stop_analyzer() -> Analyzer {
    Analyzer::new(
        "stop",
        StandardTokenizer,
        vec![Box::new(LowercaseFilter), Box::new(StopFilter::english())],
    )
}

/// `language` analyzer: UAX#29 tokenizer + lowercase + stop words + stemmer.
///
/// The most aggressive built-in analyzer — normalizes, removes stop words,
/// and stems. Best recall for free-text search.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn language_analyzer(algorithm: StemmerAlgorithm) -> Analyzer {
    Analyzer::new(
        "language",
        StandardTokenizer,
        vec![
            Box::new(LowercaseFilter),
            Box::new(StopFilter::english()),
            Box::new(StemmerFilter::new(algorithm)),
        ],
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    // --- Analyzer ---

    #[test]
    fn standard_analyzer_basic() {
        let analyzer = standard_analyzer();
        let tokens = analyzer.analyze("The Quick Brown Fox");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["the", "quick", "brown", "fox"]);
    }

    #[test]
    fn standard_analyzer_name() {
        let analyzer = standard_analyzer();
        assert_eq!(analyzer.name(), "standard");
    }

    #[test]
    fn simple_analyzer_strips_numbers() {
        let analyzer = simple_analyzer();
        let tokens = analyzer.analyze("Hello123World");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["hello", "world"]);
    }

    #[test]
    fn whitespace_analyzer_preserves_everything() {
        let analyzer = whitespace_analyzer();
        let tokens = analyzer.analyze("Hello, World!");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["Hello,", "World!"]);
    }

    #[test]
    fn keyword_analyzer_single_token() {
        let analyzer = keyword_analyzer();
        let tokens = analyzer.analyze("Hello, World!");
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].text, "Hello, World!");
    }

    #[test]
    fn stop_analyzer_removes_stop_words() {
        let analyzer = stop_analyzer();
        let tokens = analyzer.analyze("The quick brown fox is a test");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["quick", "brown", "fox", "test"]);
    }

    #[test]
    fn language_analyzer_stems() {
        let analyzer = language_analyzer(StemmerAlgorithm::English);
        let tokens = analyzer.analyze("The cats are running quickly");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["cat", "run", "quick"]);
    }

    #[test]
    fn analyzer_preserves_positions() {
        let analyzer = stop_analyzer();
        let tokens = analyzer.analyze("the quick brown fox");
        // "the" removed, but positions preserved from tokenization.
        assert_eq!(tokens[0].text, "quick");
        assert_eq!(tokens[0].position, 1); // position 0 was "the"
    }

    #[test]
    fn analyzer_empty_input() {
        let analyzer = standard_analyzer();
        let tokens = analyzer.analyze("");
        assert!(tokens.is_empty());
    }

    // --- AnalyzerRegistry ---

    #[test]
    fn registry_has_builtins() {
        let registry = AnalyzerRegistry::new();
        let names = registry.names();
        assert!(names.contains(&"standard"));
        assert!(names.contains(&"simple"));
        assert!(names.contains(&"whitespace"));
        assert!(names.contains(&"keyword"));
        assert!(names.contains(&"stop"));
    }

    #[test]
    fn registry_get_standard() {
        let registry = AnalyzerRegistry::new();
        let analyzer = registry.get("standard");
        assert_eq!(analyzer.name(), "standard");
    }

    #[test]
    fn registry_fallback_to_standard() {
        let registry = AnalyzerRegistry::new();
        let analyzer = registry.get("nonexistent");
        assert_eq!(analyzer.name(), "standard");
    }

    #[test]
    fn registry_try_get_returns_none() {
        let registry = AnalyzerRegistry::new();
        assert!(registry.try_get("nonexistent").is_none());
        assert!(registry.try_get("standard").is_some());
    }

    #[test]
    fn registry_custom_analyzer() {
        let mut registry = AnalyzerRegistry::new();
        registry.register(Analyzer::new(
            "custom",
            WhitespaceTokenizer,
            vec![Box::new(LowercaseFilter)],
        ));

        let analyzer = registry.get("custom");
        assert_eq!(analyzer.name(), "custom");
        let tokens = analyzer.analyze("Hello World");
        assert_eq!(tokens[0].text, "hello");
    }

    // --- End-to-end ---

    #[test]
    fn analyze_realistic_document() {
        let analyzer = standard_analyzer();
        let text = "Elasticsearch is a distributed, RESTful search and \
                    analytics engine. It centrally stores your data for \
                    lightning fast search.";
        let tokens = analyzer.analyze(text);

        // Should produce lowercased terms without punctuation.
        assert!(tokens.len() > 10);
        assert!(tokens.iter().all(|t| t.text == t.text.to_lowercase()));

        // Offsets should point back to the original text.
        for token in &tokens {
            assert_eq!(
                text[token.offset_from..token.offset_to].to_lowercase(),
                token.text
            );
        }
    }

    #[test]
    fn stop_analyzer_realistic() {
        let analyzer = stop_analyzer();
        let text = "The quick brown fox jumps over the lazy dog";
        let tokens = analyzer.analyze(text);
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();

        // "the" (appears twice) and "over" (not a stop word) should be handled correctly.
        assert!(!texts.contains(&"the"));
        assert!(texts.contains(&"quick"));
        assert!(texts.contains(&"over")); // "over" is not in default stop words
    }

    #[test]
    fn language_analyzer_realistic() {
        let analyzer = language_analyzer(StemmerAlgorithm::English);
        let text = "The users were searching for documents containing these keywords";
        let tokens = analyzer.analyze(text);
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();

        // Stop words removed, remaining words stemmed.
        assert!(!texts.contains(&"the"));
        assert!(!texts.contains(&"these")); // stop word
        assert!(texts.contains(&"user")); // "users" → "user"
        assert!(texts.contains(&"search")); // "searching" → "search"
        assert!(texts.contains(&"document")); // "documents" → "document"
        assert!(texts.contains(&"keyword")); // "keywords" → "keyword"
    }
}
