"""
Text sanitization and preprocessing utilities.

This module provides robust text cleaning functions to handle:
- Invalid UTF-8 characters that break tokenizers
- Control characters and non-printable characters  
- Surrogate pairs and malformed Unicode sequences
- Special characters from PDF extraction

The "can't convert negative int to unsigned" error is typically caused by
tokenizers (especially HuggingFace transformers) encountering invalid byte
sequences or special characters they can't encode.
"""

import re
import unicodedata
from typing import List, Optional

from ..infra.logging import get_logger

logger = get_logger(__name__)


# Characters that commonly cause tokenization issues
PROBLEMATIC_CHARS = {
    '\x00': '',      # Null byte
    '\ufeff': '',    # BOM
    '\ufffd': '',    # Replacement character  
    '\u200b': '',    # Zero-width space
    '\u200c': '',    # Zero-width non-joiner
    '\u200d': '',    # Zero-width joiner
    '\u2028': '\n',  # Line separator
    '\u2029': '\n',  # Paragraph separator
    '\xa0': ' ',     # Non-breaking space
    '\xad': '',      # Soft hyphen
    '\u034f': '',    # Combining grapheme joiner
}

# Regex patterns for problematic sequences
CONTROL_CHARS_PATTERN = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]')
SURROGATE_PATTERN = re.compile(r'[\ud800-\udfff]')
MULTIPLE_SPACES_PATTERN = re.compile(r' {3,}')
MULTIPLE_NEWLINES_PATTERN = re.compile(r'\n{4,}')


def sanitize_text(
    text: str,
    remove_control_chars: bool = True,
    normalize_whitespace: bool = True,
    fix_encoding: bool = True,
    max_length: Optional[int] = None
) -> str:
    """
    Sanitize text to prevent tokenization errors.
    
    This is the main entry point for text sanitization. It handles:
    - Invalid/malformed Unicode sequences
    - Control characters that break tokenizers
    - Whitespace normalization
    - Character encoding issues
    
    Args:
        text: Input text to sanitize
        remove_control_chars: Remove ASCII/Unicode control characters
        normalize_whitespace: Normalize excessive whitespace
        fix_encoding: Fix common encoding issues
        max_length: Optional maximum length to truncate to
        
    Returns:
        Sanitized text safe for tokenization
        
    Example:
        >>> sanitize_text("Hello\\x00World")
        'HelloWorld'
    """
    if not text:
        return ""
    
    original_length = len(text)
    
    try:
        # Step 1: Handle encoding issues
        if fix_encoding:
            text = _fix_encoding_issues(text)
        
        # Step 2: Remove/replace problematic characters
        text = _replace_problematic_chars(text)
        
        # Step 3: Remove surrogate pairs (invalid in UTF-8)
        text = _remove_surrogates(text)
        
        # Step 4: Remove control characters
        if remove_control_chars:
            text = _remove_control_chars(text)
        
        # Step 5: Normalize Unicode (NFC form)
        text = unicodedata.normalize('NFC', text)
        
        # Step 6: Normalize whitespace
        if normalize_whitespace:
            text = _normalize_whitespace(text)
        
        # Step 7: Truncate if needed
        if max_length and len(text) > max_length:
            text = text[:max_length]
        
        # Log if significant changes were made
        if original_length > 0:
            change_ratio = abs(original_length - len(text)) / original_length
            if change_ratio > 0.1:  # More than 10% change
                logger.debug(
                    f"Text sanitization made significant changes: "
                    f"{original_length} -> {len(text)} chars ({change_ratio:.1%} change)"
                )
        
        return text
        
    except Exception as e:
        logger.warning(f"Error during text sanitization: {e}. Using fallback.")
        return _fallback_sanitize(text, max_length)


def sanitize_batch(
    texts: List[str],
    **kwargs
) -> List[str]:
    """
    Sanitize a batch of texts.
    
    Args:
        texts: List of texts to sanitize
        **kwargs: Arguments passed to sanitize_text
        
    Returns:
        List of sanitized texts
    """
    return [sanitize_text(text, **kwargs) for text in texts]


def _fix_encoding_issues(text: str) -> str:
    """Fix common encoding issues (e.g., mojibake)."""
    try:
        # Try to detect and fix UTF-8 encoded as Latin-1
        text_bytes = text.encode('utf-8', errors='surrogatepass')
        text = text_bytes.decode('utf-8', errors='replace')
    except (UnicodeEncodeError, UnicodeDecodeError):
        # If encoding fails, just replace problematic chars
        text = text.encode('utf-8', errors='replace').decode('utf-8', errors='replace')
    
    return text


def _replace_problematic_chars(text: str) -> str:
    """Replace known problematic characters."""
    for char, replacement in PROBLEMATIC_CHARS.items():
        if char in text:
            text = text.replace(char, replacement)
    return text


def _remove_surrogates(text: str) -> str:
    """Remove surrogate pairs that cause tokenization errors."""
    return SURROGATE_PATTERN.sub('', text)


def _remove_control_chars(text: str) -> str:
    """Remove control characters (keeping newlines and tabs)."""
    # Keep \t and \n, remove other control chars
    text = CONTROL_CHARS_PATTERN.sub('', text)
    return text


def _normalize_whitespace(text: str) -> str:
    """Normalize excessive whitespace while preserving structure."""
    # Replace multiple spaces with double space (preserves some formatting)
    text = MULTIPLE_SPACES_PATTERN.sub('  ', text)
    # Replace excessive newlines
    text = MULTIPLE_NEWLINES_PATTERN.sub('\n\n\n', text)
    return text


def _fallback_sanitize(text: str, max_length: Optional[int] = None) -> str:
    """
    Fallback sanitization using ASCII-safe encoding.
    
    This is a last resort that converts to ASCII, losing some characters
    but guaranteeing tokenization will work.
    """
    try:
        # Encode to ASCII, ignoring non-ASCII chars, then decode back
        text = text.encode('ascii', errors='ignore').decode('ascii')
        if max_length:
            text = text[:max_length]
        return text
    except Exception:
        # Absolute fallback: empty string
        logger.error("Fallback sanitization failed, returning empty string")
        return ""


def is_tokenizer_safe(text: str) -> bool:
    """
    Check if text is safe for tokenization.
    
    Args:
        text: Text to check
        
    Returns:
        True if text appears safe for tokenization
    """
    if not text:
        return True
    
    # Check for surrogates
    if SURROGATE_PATTERN.search(text):
        return False
    
    # Check for null bytes
    if '\x00' in text:
        return False
    
    # Check for control characters
    if CONTROL_CHARS_PATTERN.search(text):
        return False
    
    # Try encoding to ensure no issues
    try:
        text.encode('utf-8')
        return True
    except UnicodeEncodeError:
        return False


def safe_truncate(text: str, max_tokens: int, avg_chars_per_token: float = 4.0) -> str:
    """
    Safely truncate text to approximate token count.
    
    This is a fast approximation that doesn't require actual tokenization.
    
    Args:
        text: Text to truncate
        max_tokens: Maximum number of tokens (approximate)
        avg_chars_per_token: Average characters per token (default 4.0)
        
    Returns:
        Truncated text
    """
    if not text:
        return ""
    
    max_chars = int(max_tokens * avg_chars_per_token)
    
    if len(text) <= max_chars:
        return text
    
    # Try to truncate at a sentence boundary
    truncated = text[:max_chars]
    
    # Find last sentence boundary
    for end_char in ['. ', '.\n', '! ', '? ', '\n\n']:
        last_boundary = truncated.rfind(end_char)
        if last_boundary > max_chars * 0.7:  # At least 70% of max
            return truncated[:last_boundary + 1].strip()
    
    # Fall back to word boundary
    last_space = truncated.rfind(' ')
    if last_space > max_chars * 0.8:
        return truncated[:last_space].strip()
    
    return truncated.strip()
