/// Tokenizers break input text into a sequence of tokens.
///
/// Each tokenizer defines word boundaries differently. The tokenizer is
/// responsible for setting `text`, `offset_from`, `offset_to`, and `position`
/// on each output token.
///
/// See [[analyzers#Tokenizer]].

use crate::token::Token;

/// Breaks input text into a sequence of tokens.
///
/// Implementations must be thread-safe (`Send + Sync`) so that analyzers can
/// be shared across indexing threads.
pub trait Tokenizer: Send + Sync {
    /// Tokenize `text`, appending tokens to `output`.
    ///
    /// The `output` buffer is caller-owned and reusable — callers may
    /// pre-allocate it and clear between calls to avoid repeated allocation.
    fn tokenize(&self, text: &str, output: &mut Vec<Token>);
}

/// Unicode Text Segmentation tokenizer (UAX#29 word boundaries).
///
/// The default tokenizer for `standard` and `stop` analyzers. Splits text
/// on Unicode word boundaries, keeping only "word" segments (skipping
/// whitespace and punctuation).
///
/// See [[analyzers#Tokenizer]] and
/// [UAX#29](https://unicode.org/reports/tr29/).
pub struct StandardTokenizer;

impl Tokenizer for StandardTokenizer {
    fn tokenize(&self, text: &str, output: &mut Vec<Token>) {
        use unicode_segmentation::UnicodeSegmentation;

        let mut position = output.last().map_or(0, |t| t.position + 1);

        for (byte_offset, word) in text.unicode_word_indices() {
            output.push(Token::new(
                word,
                byte_offset,
                byte_offset + word.len(),
                position,
            ));
            position += 1;
        }
    }
}

/// Splits text on Unicode whitespace.
///
/// Preserves punctuation attached to words (e.g., `"hello,"` is one token).
/// Used by the `whitespace` analyzer.
///
/// See [[analyzers#Tokenizer]].
pub struct WhitespaceTokenizer;

impl Tokenizer for WhitespaceTokenizer {
    fn tokenize(&self, text: &str, output: &mut Vec<Token>) {
        let mut position = output.last().map_or(0, |t| t.position + 1);

        for token_text in text.split_whitespace() {
            // Compute byte offset from pointer arithmetic.
            let byte_offset = token_text.as_ptr() as usize - text.as_ptr() as usize;
            output.push(Token::new(
                token_text,
                byte_offset,
                byte_offset + token_text.len(),
                position,
            ));
            position += 1;
        }
    }
}

/// Splits text on non-letter characters.
///
/// Groups consecutive Unicode letters into tokens. Numbers, punctuation,
/// and whitespace are treated as delimiters. Used by the `simple` analyzer.
///
/// See [[analyzers#Tokenizer]].
pub struct LetterTokenizer;

impl Tokenizer for LetterTokenizer {
    fn tokenize(&self, text: &str, output: &mut Vec<Token>) {
        let mut position = output.last().map_or(0, |t| t.position + 1);
        let mut start = None;

        for (i, ch) in text.char_indices() {
            if ch.is_alphabetic() {
                if start.is_none() {
                    start = Some(i);
                }
            } else if let Some(s) = start.take() {
                output.push(Token::new(&text[s..i], s, i, position));
                position += 1;
            }
        }

        // Emit trailing token.
        if let Some(s) = start {
            output.push(Token::new(&text[s..], s, text.len(), position));
        }
    }
}

/// Emits the entire input as a single token.
///
/// Used for keyword fields where the full value should be indexed as-is
/// (e.g., status codes, tags, identifiers). Used by the `keyword` analyzer.
///
/// See [[analyzers#Tokenizer]].
pub struct KeywordTokenizer;

impl Tokenizer for KeywordTokenizer {
    fn tokenize(&self, text: &str, output: &mut Vec<Token>) {
        if text.is_empty() {
            return;
        }
        let position = output.last().map_or(0, |t| t.position + 1);
        output.push(Token::new(text, 0, text.len(), position));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // --- StandardTokenizer ---

    #[test]
    fn standard_basic() {
        let mut tokens = Vec::new();
        StandardTokenizer.tokenize("Hello, world!", &mut tokens);
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].text, "Hello");
        assert_eq!(tokens[1].text, "world");
    }

    #[test]
    fn standard_positions() {
        let mut tokens = Vec::new();
        StandardTokenizer.tokenize("the quick brown fox", &mut tokens);
        assert_eq!(tokens.len(), 4);
        for (i, token) in tokens.iter().enumerate() {
            assert_eq!(token.position, i as u32);
        }
    }

    #[test]
    fn standard_offsets() {
        let mut tokens = Vec::new();
        StandardTokenizer.tokenize("Hello world", &mut tokens);
        assert_eq!(tokens[0].offset_from, 0);
        assert_eq!(tokens[0].offset_to, 5);
        assert_eq!(tokens[1].offset_from, 6);
        assert_eq!(tokens[1].offset_to, 11);
    }

    #[test]
    fn standard_empty() {
        let mut tokens = Vec::new();
        StandardTokenizer.tokenize("", &mut tokens);
        assert!(tokens.is_empty());
    }

    #[test]
    fn standard_punctuation_only() {
        let mut tokens = Vec::new();
        StandardTokenizer.tokenize("!!! ... ???", &mut tokens);
        assert!(tokens.is_empty());
    }

    #[test]
    fn standard_numbers() {
        let mut tokens = Vec::new();
        StandardTokenizer.tokenize("test123 456abc", &mut tokens);
        // UAX#29 treats alphanumeric sequences as words.
        assert!(tokens.len() >= 2);
    }

    #[test]
    fn standard_apostrophe() {
        let mut tokens = Vec::new();
        StandardTokenizer.tokenize("it's a test", &mut tokens);
        // UAX#29 keeps apostrophes within words.
        assert!(tokens.iter().any(|t| t.text.contains("it")));
    }

    // --- WhitespaceTokenizer ---

    #[test]
    fn whitespace_basic() {
        let mut tokens = Vec::new();
        WhitespaceTokenizer.tokenize("Hello, world!", &mut tokens);
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].text, "Hello,");
        assert_eq!(tokens[1].text, "world!");
    }

    #[test]
    fn whitespace_preserves_punctuation() {
        let mut tokens = Vec::new();
        WhitespaceTokenizer.tokenize("price=$100.00", &mut tokens);
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].text, "price=$100.00");
    }

    #[test]
    fn whitespace_multiple_spaces() {
        let mut tokens = Vec::new();
        WhitespaceTokenizer.tokenize("a  b\t\nc", &mut tokens);
        assert_eq!(tokens.len(), 3);
        assert_eq!(tokens[0].text, "a");
        assert_eq!(tokens[1].text, "b");
        assert_eq!(tokens[2].text, "c");
    }

    #[test]
    fn whitespace_offsets() {
        let mut tokens = Vec::new();
        WhitespaceTokenizer.tokenize("Hello world", &mut tokens);
        assert_eq!(tokens[0].offset_from, 0);
        assert_eq!(tokens[0].offset_to, 5);
        assert_eq!(tokens[1].offset_from, 6);
        assert_eq!(tokens[1].offset_to, 11);
    }

    #[test]
    fn whitespace_empty() {
        let mut tokens = Vec::new();
        WhitespaceTokenizer.tokenize("", &mut tokens);
        assert!(tokens.is_empty());
    }

    // --- LetterTokenizer ---

    #[test]
    fn letter_basic() {
        let mut tokens = Vec::new();
        LetterTokenizer.tokenize("Hello, world!", &mut tokens);
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].text, "Hello");
        assert_eq!(tokens[1].text, "world");
    }

    #[test]
    fn letter_strips_numbers() {
        let mut tokens = Vec::new();
        LetterTokenizer.tokenize("test123data", &mut tokens);
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].text, "test");
        assert_eq!(tokens[1].text, "data");
    }

    #[test]
    fn letter_unicode() {
        let mut tokens = Vec::new();
        LetterTokenizer.tokenize("café résumé", &mut tokens);
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].text, "café");
        assert_eq!(tokens[1].text, "résumé");
    }

    #[test]
    fn letter_offsets() {
        let mut tokens = Vec::new();
        LetterTokenizer.tokenize("abc 123 def", &mut tokens);
        assert_eq!(tokens[0].offset_from, 0);
        assert_eq!(tokens[0].offset_to, 3);
        assert_eq!(tokens[1].offset_from, 8);
        assert_eq!(tokens[1].offset_to, 11);
    }

    #[test]
    fn letter_empty() {
        let mut tokens = Vec::new();
        LetterTokenizer.tokenize("", &mut tokens);
        assert!(tokens.is_empty());
    }

    #[test]
    fn letter_no_letters() {
        let mut tokens = Vec::new();
        LetterTokenizer.tokenize("12345 !@#$%", &mut tokens);
        assert!(tokens.is_empty());
    }

    // --- KeywordTokenizer ---

    #[test]
    fn keyword_basic() {
        let mut tokens = Vec::new();
        KeywordTokenizer.tokenize("Hello, world!", &mut tokens);
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].text, "Hello, world!");
    }

    #[test]
    fn keyword_offsets() {
        let mut tokens = Vec::new();
        KeywordTokenizer.tokenize("test value", &mut tokens);
        assert_eq!(tokens[0].offset_from, 0);
        assert_eq!(tokens[0].offset_to, 10);
        assert_eq!(tokens[0].position, 0);
    }

    #[test]
    fn keyword_empty() {
        let mut tokens = Vec::new();
        KeywordTokenizer.tokenize("", &mut tokens);
        assert!(tokens.is_empty());
    }
}
