# createsonline/nlp/text_processing.py
"""
CREATESONLINE Text Processing Utilities

Sentence splitting, text cleaning, n-gram extraction,
and text statistics — all pure Python.
"""

import re
from typing import List, Dict, Tuple, Optional
from collections import Counter


class TextProcessor:
    """
    Text processing utilities for NLP pipelines.
    
    Usage:
        tp = TextProcessor()
        sentences = tp.split_sentences("Hello world. How are you?")
        cleaned = tp.clean("  Hello   World!!  ")
        ngrams = tp.get_ngrams(["hello", "world", "foo"], n=2)
    """
    
    # Common English stop words
    STOP_WORDS = frozenset({
        'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
        'of', 'with', 'by', 'is', 'am', 'are', 'was', 'were', 'be', 'been',
        'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
        'could', 'should', 'may', 'might', 'shall', 'can', 'it', 'its',
        'this', 'that', 'these', 'those', 'i', 'me', 'my', 'we', 'our',
        'you', 'your', 'he', 'him', 'his', 'she', 'her', 'they', 'them',
        'their', 'what', 'which', 'who', 'whom', 'not', 'no', 'nor',
        'so', 'if', 'then', 'than', 'too', 'very', 'just', 'about',
        'up', 'out', 'off', 'over', 'under', 'again', 'further',
        'once', 'here', 'there', 'when', 'where', 'why', 'how',
        'all', 'each', 'every', 'both', 'few', 'more', 'most',
        'other', 'some', 'such', 'only', 'own', 'same',
        'as', 'into', 'through', 'during', 'before', 'after',
        'above', 'below', 'from', 'down', 'between', 'because',
    })
    
    # Sentence boundary regex
    _SENT_RE = re.compile(r'(?<=[.!?])\s+(?=[A-Z])')
    
    def __init__(self, lowercase: bool = True, remove_stopwords: bool = False):
        self.lowercase = lowercase
        self.remove_stopwords = remove_stopwords
    
    def clean(self, text: str) -> str:
        """Clean text: normalize whitespace, optionally lowercase"""
        text = re.sub(r'\s+', ' ', text).strip()
        if self.lowercase:
            text = text.lower()
        return text
    
    def split_sentences(self, text: str) -> List[str]:
        """Split text into sentences"""
        sentences = self._SENT_RE.split(text)
        return [s.strip() for s in sentences if s.strip()]
    
    def tokenize_words(self, text: str) -> List[str]:
        """Simple word tokenization"""
        text = self.clean(text)
        tokens = re.findall(r"[a-zA-Z]+|[0-9]+|[^\s\w]", text)
        if self.remove_stopwords:
            tokens = [t for t in tokens if t.lower() not in self.STOP_WORDS]
        return tokens
    
    def get_ngrams(self, tokens: List[str], n: int = 2) -> List[Tuple[str, ...]]:
        """Extract n-grams from token list"""
        if len(tokens) < n:
            return []
        return [tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)]
    
    def get_ngram_counts(self, tokens: List[str], n: int = 2) -> Counter:
        """Count n-gram frequencies"""
        return Counter(self.get_ngrams(tokens, n))
    
    def word_frequencies(self, text: str) -> Counter:
        """Get word frequency distribution"""
        tokens = self.tokenize_words(text)
        return Counter(tokens)
    
    def text_statistics(self, text: str) -> Dict:
        """Compute basic text statistics"""
        sentences = self.split_sentences(text)
        words = self.tokenize_words(text)
        chars = len(text)
        word_count = len(words)
        sent_count = len(sentences)
        unique_words = len(set(words))
        avg_word_len = sum(len(w) for w in words) / max(1, word_count)
        avg_sent_len = word_count / max(1, sent_count)
        
        # Lexical diversity
        lexical_diversity = unique_words / max(1, word_count)
        
        return {
            "characters": chars,
            "words": word_count,
            "sentences": sent_count,
            "unique_words": unique_words,
            "avg_word_length": round(avg_word_len, 2),
            "avg_sentence_length": round(avg_sent_len, 2),
            "lexical_diversity": round(lexical_diversity, 4),
        }
    
    def pad_sequence(self, ids: List[int], max_length: int, pad_id: int = 0,
                     truncate: bool = True) -> List[int]:
        """Pad or truncate a sequence to fixed length"""
        if truncate and len(ids) > max_length:
            return ids[:max_length]
        return ids + [pad_id] * max(0, max_length - len(ids))
    
    def pad_sequences(self, sequences: List[List[int]], max_length: Optional[int] = None,
                      pad_id: int = 0, truncate: bool = True) -> List[List[int]]:
        """Pad multiple sequences to the same length"""
        if max_length is None:
            max_length = max(len(s) for s in sequences) if sequences else 0
        return [self.pad_sequence(s, max_length, pad_id, truncate) for s in sequences]
    
    def create_attention_mask(self, ids: List[int], pad_id: int = 0) -> List[int]:
        """Create attention mask (1 for real tokens, 0 for padding)"""
        return [0 if token_id == pad_id else 1 for token_id in ids]
