"""
Deep behavioral tests for text processing patterns.
Tests tokenization, stemming, normalization,
and text similarity logic.
"""

import re
from collections import Counter


# --- Tokenization patterns ---


class TestBasicTokenization:
    """Tests for basic tokenization patterns."""

    def _tokenize_whitespace(self, text):
        """Split text by whitespace."""
        if not text:
            return []
        return text.split()

    def _tokenize_words(self, text):
        """Extract only words (alphanumeric)."""
        if not text:
            return []
        return re.findall(r"\b\w+\b", text)

    def _tokenize_sentences(self, text):
        """Split text into sentences."""
        if not text:
            return []
        return re.split(r"[.!?]+\s*", text.strip())

    def test_whitespace_tokenize(self):
        result = self._tokenize_whitespace("hello world  foo")
        assert result == ["hello", "world", "foo"]

    def test_whitespace_empty(self):
        assert self._tokenize_whitespace("") == []

    def test_words_ignores_punctuation(self):
        result = self._tokenize_words("Hello, world!")
        assert result == ["Hello", "world"]

    def test_sentence_tokenize(self):
        text = "Hello world. How are you? Fine."
        result = self._tokenize_sentences(text)
        assert "Hello world" in result


class TestNGramTokenization:
    """Tests for n-gram tokenization patterns."""

    def _ngrams(self, tokens, n):
        """Generate n-grams from tokens."""
        if len(tokens) < n:
            return []
        return [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)]

    def _char_ngrams(self, text, n):
        """Generate character n-grams."""
        if len(text) < n:
            return []
        return [text[i : i + n] for i in range(len(text) - n + 1)]

    def test_bigrams(self):
        tokens = ["the", "quick", "brown", "fox"]
        result = self._ngrams(tokens, 2)
        assert ("the", "quick") in result
        assert ("quick", "brown") in result
        assert len(result) == 3

    def test_trigrams(self):
        tokens = ["a", "b", "c", "d"]
        result = self._ngrams(tokens, 3)
        assert len(result) == 2

    def test_char_ngrams(self):
        result = self._char_ngrams("hello", 2)
        assert result == ["he", "el", "ll", "lo"]


# --- Text normalization patterns ---


class TestTextNormalization:
    """Tests for text normalization patterns."""

    def _lowercase(self, text):
        """Convert to lowercase."""
        return text.lower() if text else ""

    def _remove_punctuation(self, text):
        """Remove punctuation."""
        if not text:
            return ""
        return re.sub(r"[^\w\s]", "", text)

    def _collapse_whitespace(self, text):
        """Collapse multiple whitespace to single space."""
        if not text:
            return ""
        return re.sub(r"\s+", " ", text).strip()

    def _normalize(self, text):
        """Apply all normalizations."""
        text = self._lowercase(text)
        text = self._remove_punctuation(text)
        text = self._collapse_whitespace(text)
        return text

    def test_lowercase(self):
        assert self._lowercase("Hello World") == "hello world"

    def test_remove_punctuation(self):
        assert self._remove_punctuation("Hello, world!") == "Hello world"

    def test_collapse_whitespace(self):
        assert self._collapse_whitespace("  hello   world  ") == "hello world"

    def test_full_normalize(self):
        result = self._normalize("  Hello,  World!  ")
        assert result == "hello world"


class TestUnicodeNormalization:
    """Tests for unicode normalization patterns."""

    def _remove_accents(self, text):
        """Remove diacritical marks."""
        import unicodedata

        normalized = unicodedata.normalize("NFD", text)
        return "".join(c for c in normalized if unicodedata.category(c) != "Mn")

    def _ascii_only(self, text):
        """Keep only ASCII characters."""
        return "".join(c for c in text if ord(c) < 128)

    def test_remove_accents(self):
        result = self._remove_accents("café")
        assert result == "cafe"

    def test_ascii_only(self):
        result = self._ascii_only("hello café ñ")
        assert result == "hello caf "


# --- Stop word removal patterns ---


class TestStopWordRemoval:
    """Tests for stop word removal patterns."""

    def _get_stopwords(self):
        """Get common English stop words."""
        return {
            "the",
            "a",
            "an",
            "is",
            "are",
            "was",
            "were",
            "be",
            "been",
            "being",
            "have",
            "has",
            "had",
            "do",
            "does",
            "did",
            "will",
            "would",
            "could",
            "should",
            "may",
            "might",
            "must",
            "and",
            "but",
            "or",
            "of",
            "at",
            "by",
            "for",
            "with",
            "about",
            "to",
            "from",
            "in",
            "on",
            "it",
            "its",
            "this",
            "that",
        }

    def _remove_stopwords(self, tokens):
        """Remove stop words from tokens."""
        stopwords = self._get_stopwords()
        return [t for t in tokens if t.lower() not in stopwords]

    def test_removes_stopwords(self):
        tokens = ["the", "quick", "brown", "fox"]
        result = self._remove_stopwords(tokens)
        assert "the" not in result
        assert "quick" in result

    def test_case_insensitive(self):
        tokens = ["The", "Quick", "THE"]
        result = self._remove_stopwords(tokens)
        assert "The" not in result
        assert "THE" not in result
        assert "Quick" in result


# --- Stemming patterns ---


class TestSimpleStemming:
    """Tests for simple stemming patterns."""

    def _stem_suffix(self, word):
        """Simple suffix-based stemming."""
        if len(word) < 4:
            return word
        suffixes = ["ing", "ed", "ly", "es", "s", "er", "est", "ment", "ness"]
        for suffix in suffixes:
            if word.endswith(suffix) and len(word) - len(suffix) >= 3:
                return word[: -len(suffix)]
        return word

    def test_stem_ing(self):
        assert self._stem_suffix("running") == "runn"
        assert self._stem_suffix("jumping") == "jump"

    def test_stem_ed(self):
        assert self._stem_suffix("walked") == "walk"

    def test_stem_ly(self):
        assert self._stem_suffix("quickly") == "quick"

    def test_short_word_unchanged(self):
        assert self._stem_suffix("run") == "run"


# --- Text similarity patterns ---


class TestJaccardSimilarity:
    """Tests for Jaccard similarity patterns."""

    def _jaccard(self, set1, set2):
        """Calculate Jaccard similarity."""
        if not set1 and not set2:
            return 1.0
        intersection = len(set1 & set2)
        union = len(set1 | set2)
        if union == 0:
            return 0.0
        return intersection / union

    def _jaccard_tokens(self, text1, text2):
        """Calculate Jaccard similarity for texts."""
        tokens1 = set(text1.lower().split())
        tokens2 = set(text2.lower().split())
        return self._jaccard(tokens1, tokens2)

    def test_identical_sets(self):
        s1 = {"a", "b", "c"}
        assert self._jaccard(s1, s1) == 1.0

    def test_no_overlap(self):
        s1 = {"a", "b"}
        s2 = {"c", "d"}
        assert self._jaccard(s1, s2) == 0.0

    def test_partial_overlap(self):
        s1 = {"a", "b", "c"}
        s2 = {"b", "c", "d"}
        # Intersection: {b, c} = 2, Union: {a, b, c, d} = 4
        assert self._jaccard(s1, s2) == 0.5

    def test_text_similarity(self):
        result = self._jaccard_tokens("the cat sat", "the cat ran")
        assert 0 < result < 1


class TestCosineSimilarity:
    """Tests for cosine similarity patterns."""

    def _cosine(self, vec1, vec2):
        """Calculate cosine similarity between vectors."""
        dot = sum(a * b for a, b in zip(vec1, vec2))
        norm1 = sum(a * a for a in vec1) ** 0.5
        norm2 = sum(b * b for b in vec2) ** 0.5
        if norm1 == 0 or norm2 == 0:
            return 0.0
        return dot / (norm1 * norm2)

    def _text_to_vector(self, text, vocab):
        """Convert text to term frequency vector."""
        tokens = text.lower().split()
        counter = Counter(tokens)
        return [counter.get(word, 0) for word in vocab]

    def test_identical_vectors(self):
        vec = [1, 2, 3]
        result = self._cosine(vec, vec)
        assert abs(result - 1.0) < 0.001

    def test_orthogonal_vectors(self):
        vec1 = [1, 0, 0]
        vec2 = [0, 1, 0]
        result = self._cosine(vec1, vec2)
        assert abs(result) < 0.001

    def test_zero_vector(self):
        vec1 = [1, 2, 3]
        vec2 = [0, 0, 0]
        assert self._cosine(vec1, vec2) == 0.0


class TestEditDistance:
    """Tests for edit distance (Levenshtein) patterns."""

    def _edit_distance(self, s1, s2):
        """Calculate edit distance between strings."""
        m, n = len(s1), len(s2)
        dp = [[0] * (n + 1) for _ in range(m + 1)]
        for i in range(m + 1):
            dp[i][0] = i
        for j in range(n + 1):
            dp[0][j] = j
        for i in range(1, m + 1):
            for j in range(1, n + 1):
                if s1[i - 1] == s2[j - 1]:
                    dp[i][j] = dp[i - 1][j - 1]
                else:
                    dp[i][j] = 1 + min(
                        dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]
                    )
        return dp[m][n]

    def _similarity_from_distance(self, s1, s2):
        """Calculate similarity from edit distance."""
        if not s1 and not s2:
            return 1.0
        max_len = max(len(s1), len(s2))
        if max_len == 0:
            return 1.0
        distance = self._edit_distance(s1, s2)
        return 1 - (distance / max_len)

    def test_identical_strings(self):
        assert self._edit_distance("hello", "hello") == 0

    def test_one_insertion(self):
        assert self._edit_distance("cat", "cats") == 1

    def test_one_deletion(self):
        assert self._edit_distance("cats", "cat") == 1

    def test_one_substitution(self):
        assert self._edit_distance("cat", "bat") == 1

    def test_similarity(self):
        sim = self._similarity_from_distance("hello", "hallo")
        assert 0.7 < sim < 0.9


# --- Keyword extraction patterns ---


class TestKeywordExtraction:
    """Tests for keyword extraction patterns."""

    def _extract_keywords(self, text, top_n=5):
        """Extract top keywords by frequency."""
        tokens = re.findall(r"\b\w+\b", text.lower())
        stopwords = {
            "the",
            "a",
            "an",
            "is",
            "are",
            "in",
            "on",
            "of",
            "to",
            "and",
        }
        tokens = [t for t in tokens if t not in stopwords and len(t) > 2]
        counter = Counter(tokens)
        return [word for word, _ in counter.most_common(top_n)]

    def _extract_phrases(self, text, min_freq=2):
        """Extract common bigram phrases."""
        tokens = re.findall(r"\b\w+\b", text.lower())
        bigrams = [
            f"{tokens[i]} {tokens[i + 1]}" for i in range(len(tokens) - 1)
        ]
        counter = Counter(bigrams)
        return [
            phrase for phrase, count in counter.items() if count >= min_freq
        ]

    def test_extracts_frequent_words(self):
        text = "python python python java java ruby"
        keywords = self._extract_keywords(text, top_n=2)
        assert "python" in keywords

    def test_ignores_stopwords(self):
        text = "the the the cat cat dog"
        keywords = self._extract_keywords(text, top_n=3)
        assert "the" not in keywords

    def test_extract_phrases(self):
        text = "machine learning machine learning deep learning"
        phrases = self._extract_phrases(text, min_freq=2)
        assert "machine learning" in phrases


# --- Text truncation patterns ---


class TestTextTruncation:
    """Tests for text truncation patterns."""

    def _truncate(self, text, max_length, suffix="..."):
        """Truncate text to max length."""
        if not text or len(text) <= max_length:
            return text
        return text[: max_length - len(suffix)] + suffix

    def _truncate_words(self, text, max_words, suffix="..."):
        """Truncate text to max words."""
        words = text.split()
        if len(words) <= max_words:
            return text
        return " ".join(words[:max_words]) + suffix

    def _smart_truncate(self, text, max_length, suffix="..."):
        """Truncate at word boundary."""
        if not text or len(text) <= max_length:
            return text
        truncated = text[: max_length - len(suffix)]
        # Find last space
        last_space = truncated.rfind(" ")
        if last_space > max_length // 2:
            truncated = truncated[:last_space]
        return truncated + suffix

    def test_short_text_unchanged(self):
        result = self._truncate("hello", max_length=10)
        assert result == "hello"

    def test_long_text_truncated(self):
        result = self._truncate("hello world", max_length=8)
        assert len(result) == 8
        assert result.endswith("...")

    def test_truncate_words(self):
        result = self._truncate_words("one two three four five", max_words=3)
        assert result == "one two three..."

    def test_smart_truncate(self):
        result = self._smart_truncate("hello wonderful world", max_length=15)
        assert result.endswith("...")
        assert "wonderful" not in result or result.endswith("...")


# --- Text cleaning patterns ---


class TestTextCleaning:
    """Tests for text cleaning patterns."""

    def _remove_html_tags(self, text):
        """Remove HTML tags."""
        return re.sub(r"<[^>]+>", "", text)

    def _remove_urls(self, text):
        """Remove URLs."""
        return re.sub(r"https?://\S+", "", text)

    def _remove_emails(self, text):
        """Remove email addresses."""
        return re.sub(r"\S+@\S+\.\S+", "", text)

    def _remove_numbers(self, text):
        """Remove numbers."""
        return re.sub(r"\d+", "", text)

    def test_remove_html(self):
        result = self._remove_html_tags("<p>Hello</p> <b>World</b>")
        assert result == "Hello World"

    def test_remove_urls(self):
        result = self._remove_urls("Visit https://example.com for more")
        assert "https" not in result
        assert "Visit" in result

    def test_remove_emails(self):
        result = self._remove_emails("Contact user@example.com today")
        assert "@" not in result

    def test_remove_numbers(self):
        result = self._remove_numbers("Order 12345 is ready")
        assert "12345" not in result


# --- Text splitting patterns ---


class TestTextSplitting:
    """Tests for text splitting patterns."""

    def _split_into_chunks(self, text, chunk_size):
        """Split text into fixed-size chunks."""
        return [
            text[i : i + chunk_size] for i in range(0, len(text), chunk_size)
        ]

    def _split_into_paragraphs(self, text):
        """Split text into paragraphs."""
        return [p.strip() for p in text.split("\n\n") if p.strip()]

    def _split_preserving_words(self, text, max_chunk_size):
        """Split text into chunks at word boundaries."""
        words = text.split()
        chunks = []
        current_chunk = []
        current_size = 0
        for word in words:
            if current_size + len(word) + 1 > max_chunk_size and current_chunk:
                chunks.append(" ".join(current_chunk))
                current_chunk = [word]
                current_size = len(word)
            else:
                current_chunk.append(word)
                current_size += len(word) + 1
        if current_chunk:
            chunks.append(" ".join(current_chunk))
        return chunks

    def test_fixed_size_chunks(self):
        text = "abcdefghij"
        chunks = self._split_into_chunks(text, 3)
        assert chunks == ["abc", "def", "ghi", "j"]

    def test_paragraphs(self):
        text = "First paragraph.\n\nSecond paragraph.\n\nThird."
        paragraphs = self._split_into_paragraphs(text)
        assert len(paragraphs) == 3

    def test_word_boundary_chunks(self):
        text = "one two three four five"
        chunks = self._split_preserving_words(text, 10)
        # Each chunk should respect word boundaries
        for chunk in chunks:
            assert len(chunk) <= 10 or " " not in chunk


# --- Text hashing patterns ---


class TestTextHashing:
    """Tests for text hashing patterns."""

    def _simple_hash(self, text):
        """Simple hash for text."""
        return hash(text.lower().strip())

    def _fingerprint(self, text, k=3):
        """Create text fingerprint using k-shingles."""
        text = text.lower()
        shingles = {text[i : i + k] for i in range(len(text) - k + 1)}
        return frozenset(shingles)

    def _simhash_bits(self, tokens, num_bits=32):
        """Simple simhash implementation."""
        v = [0] * num_bits
        for token in tokens:
            h = hash(token) & ((1 << num_bits) - 1)
            for i in range(num_bits):
                if h & (1 << i):
                    v[i] += 1
                else:
                    v[i] -= 1
        fingerprint = 0
        for i in range(num_bits):
            if v[i] > 0:
                fingerprint |= 1 << i
        return fingerprint

    def test_same_text_same_hash(self):
        h1 = self._simple_hash("Hello World")
        h2 = self._simple_hash("hello world")
        assert h1 == h2

    def test_fingerprint_identical(self):
        fp1 = self._fingerprint("hello world")
        fp2 = self._fingerprint("hello world")
        assert fp1 == fp2

    def test_fingerprint_similar_overlap(self):
        fp1 = self._fingerprint("hello world")
        fp2 = self._fingerprint("hello there")
        # Should have some overlap
        assert len(fp1 & fp2) > 0
