"""
Topic Clustering Analyzer

Uses TF-IDF and K-means clustering to analyze content topics and extract
LSI (Latent Semantic Indexing) keywords.

Requires scikit-learn: pip install turboseo[ml]
"""

import re
from typing import Any

from pydantic import BaseModel

# Check for sklearn availability
try:
    from sklearn.cluster import KMeans
    from sklearn.feature_extraction.text import TfidfVectorizer

    SKLEARN_AVAILABLE = True
except ImportError:
    SKLEARN_AVAILABLE = False
    KMeans = None  # type: ignore
    TfidfVectorizer = None  # type: ignore


# Common stop words for LSI keyword filtering
STOP_WORDS = {
    "a", "an", "and", "are", "as", "at", "be", "by", "for", "from",
    "has", "he", "in", "is", "it", "its", "of", "on", "that", "the",
    "to", "was", "will", "with", "you", "your", "this", "their", "but",
    "or", "not", "can", "have", "all", "when", "there", "been", "if",
    "more", "so", "about", "what", "which", "who", "would", "could",
    "into", "than", "then", "them", "these", "they", "those", "through",
    "very", "just", "also", "how", "out", "up", "down", "only", "some",
    "such", "any", "each", "few", "most", "other", "our", "own", "same",
}


class TopicCluster(BaseModel):
    """A topic cluster from content analysis."""

    cluster_id: int
    top_terms: list[str]  # Most significant terms in this cluster
    section_indices: list[int]  # Which sections belong to this cluster
    section_count: int  # Number of sections in this cluster


class LSIKeywords(BaseModel):
    """Latent semantic keywords extracted from content."""

    single_words: list[str]  # Top single-word keywords
    phrases: list[str]  # Top multi-word phrases


class TopicClusteringResult(BaseModel):
    """Result of topic clustering analysis."""

    section_count: int
    cluster_count: int
    clusters: list[TopicCluster]
    lsi_keywords: LSIKeywords
    recommendations: list[str]


def _split_into_sections(content: str) -> list[str]:
    """
    Split content into sections by H2 headings.

    Returns:
        List of section texts (each with at least 20 words)
    """
    # Split by H2 headers
    parts = re.split(r"^##\s+", content, flags=re.MULTILINE)

    sections = []
    for part in parts:
        # Clean the section text
        text = part.strip()
        if not text:
            continue

        # Remove the header line itself (first line)
        lines = text.split("\n", 1)
        if len(lines) > 1:
            text = lines[1].strip()
        elif not text.startswith("#"):
            text = lines[0].strip()
        else:
            continue

        # Clean markdown formatting
        text = re.sub(r"^#+\s+.*$", "", text, flags=re.MULTILINE)  # Headers
        text = re.sub(r"```[\s\S]*?```", "", text)  # Code blocks
        text = re.sub(r"`[^`]+`", "", text)  # Inline code
        text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)  # Links
        text = re.sub(r"[*_]{1,2}([^*_]+)[*_]{1,2}", r"\1", text)  # Bold/italic
        text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE)  # Lists

        # Only include sections with enough words
        words = text.split()
        if len(words) >= 20:
            sections.append(" ".join(words))

    return sections


def _extract_tfidf_features(
    sections: list[str],
) -> tuple[Any, Any, list[str]]:
    """
    Extract TF-IDF features from sections.

    Returns:
        Tuple of (tfidf_matrix, vectorizer, feature_names)
    """
    vectorizer = TfidfVectorizer(
        max_features=100,
        ngram_range=(1, 2),  # Unigrams and bigrams
        stop_words="english",
        min_df=1,
        max_df=0.95,
    )
    tfidf_matrix = vectorizer.fit_transform(sections)
    feature_names = vectorizer.get_feature_names_out().tolist()
    return tfidf_matrix, vectorizer, feature_names


def _determine_cluster_count(section_count: int) -> int:
    """Determine optimal number of clusters."""
    # Formula: min(5, max(2, sections // 2))
    return min(5, max(2, section_count // 2))


def _cluster_sections(tfidf_matrix: Any, n_clusters: int) -> Any:
    """Apply K-means clustering to TF-IDF matrix."""
    kmeans = KMeans(
        n_clusters=n_clusters,
        random_state=42,
        n_init=10,
    )
    kmeans.fit(tfidf_matrix)
    return kmeans


def _extract_cluster_terms(
    kmeans: Any, feature_names: list[str], top_n: int = 5
) -> list[list[str]]:
    """Extract top terms for each cluster based on cluster centers."""
    cluster_terms = []
    centers = kmeans.cluster_centers_

    for i in range(len(centers)):
        # Get indices of top terms for this cluster
        center = centers[i]
        top_indices = center.argsort()[::-1][:top_n]
        terms = [feature_names[idx] for idx in top_indices]
        cluster_terms.append(terms)

    return cluster_terms


def _extract_lsi_keywords(
    tfidf_matrix: Any,
    feature_names: list[str],
    max_single: int = 10,
    max_phrases: int = 5,
) -> LSIKeywords:
    """
    Extract top single words and phrases based on TF-IDF scores.

    Filters out stop words and very short words.
    """
    # Sum TF-IDF scores across all documents
    summed_scores = tfidf_matrix.sum(axis=0).A1  # Convert to 1D array

    # Sort by score
    top_indices = summed_scores.argsort()[::-1]

    single_words = []
    phrases = []

    for idx in top_indices:
        term = feature_names[idx]

        # Check if it's a phrase (contains space) or single word
        if " " in term:
            # Filter phrases: must be >8 chars, no stop words only
            words_in_phrase = term.split()
            content_words = [w for w in words_in_phrase if w.lower() not in STOP_WORDS]
            if len(term) > 8 and content_words and len(phrases) < max_phrases:
                phrases.append(term)
        else:
            # Filter single words: must be >3 chars, not a stop word
            if (
                len(term) > 3
                and term.lower() not in STOP_WORDS
                and len(single_words) < max_single
            ):
                single_words.append(term)

        # Stop if we have enough
        if len(single_words) >= max_single and len(phrases) >= max_phrases:
            break

    return LSIKeywords(single_words=single_words, phrases=phrases)


def _generate_recommendations(
    section_count: int,
    cluster_count: int,
    clusters: list[TopicCluster],
) -> list[str]:
    """Generate recommendations based on clustering results."""
    recommendations = []

    if section_count < 3:
        recommendations.append(
            "Add more sections (H2 headings) for better topic analysis. "
            "Minimum 3 sections recommended."
        )

    if cluster_count == 1 and section_count >= 3:
        recommendations.append(
            "All sections cluster together, indicating a single focused topic. "
            "Consider if more topic diversity would benefit readers."
        )

    # Check for unbalanced clusters
    if clusters:
        sizes = [c.section_count for c in clusters]
        max_size = max(sizes)
        min_size = min(sizes)

        if max_size > min_size * 3 and len(clusters) > 1:
            recommendations.append(
                "Topic clusters are unbalanced. Some topics have much more coverage. "
                "Consider expanding underrepresented topics."
            )

    return recommendations


def analyze_topics(content: str) -> TopicClusteringResult:
    """
    Analyze content for topic clusters and LSI keywords.

    Uses TF-IDF vectorization and K-means clustering to identify
    topic groupings in the content.

    Args:
        content: Text to analyze (markdown supported)

    Returns:
        TopicClusteringResult with clusters and keywords

    Raises:
        ImportError: If scikit-learn is not installed
    """
    if not SKLEARN_AVAILABLE:
        raise ImportError(
            "Topic clustering requires scikit-learn. "
            "Install with: pip install turboseo[ml]"
        )

    sections = _split_into_sections(content)

    # Handle insufficient sections
    if len(sections) < 2:
        return TopicClusteringResult(
            section_count=len(sections),
            cluster_count=0,
            clusters=[],
            lsi_keywords=LSIKeywords(single_words=[], phrases=[]),
            recommendations=[
                "Add more sections (H2 headings) with at least 20 words each "
                "for topic clustering analysis."
            ],
        )

    # TF-IDF extraction
    tfidf_matrix, vectorizer, feature_names = _extract_tfidf_features(sections)

    # Determine cluster count and perform clustering
    n_clusters = _determine_cluster_count(len(sections))
    kmeans = _cluster_sections(tfidf_matrix, n_clusters)

    # Extract cluster information
    cluster_terms = _extract_cluster_terms(kmeans, feature_names)
    clusters = []

    for i, terms in enumerate(cluster_terms):
        section_indices = [
            j for j, label in enumerate(kmeans.labels_) if label == i
        ]
        clusters.append(
            TopicCluster(
                cluster_id=i,
                top_terms=terms,
                section_indices=section_indices,
                section_count=len(section_indices),
            )
        )

    # Extract LSI keywords
    lsi_keywords = _extract_lsi_keywords(tfidf_matrix, feature_names)

    # Generate recommendations
    recommendations = _generate_recommendations(
        len(sections), len(clusters), clusters
    )

    return TopicClusteringResult(
        section_count=len(sections),
        cluster_count=len(clusters),
        clusters=clusters,
        lsi_keywords=lsi_keywords,
        recommendations=recommendations,
    )
