"""
Linguistic analysis tools for sign language research.

Provides tools commonly used by sign language linguists:
- Concordancer (KWIC - Key Word In Context)
- Co-occurrence analysis
- N-gram extraction
- Sign duration patterns
- Signer variation analysis
- Minimal pairs detection (with phonological and string-based modes)
"""

from __future__ import annotations

import logging
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any

import numpy as np

if TYPE_CHECKING:
    from sltk.linguistics.phonology import PhonologicalForm

logger = logging.getLogger(__name__)


@dataclass
class ConcordanceLine:
    """A single concordance line showing a gloss in context."""

    file_id: str
    segment_idx: int
    gloss: str
    left_context: list[str]
    right_context: list[str]
    start_time: float
    end_time: float
    duration: float
    tier: str
    signer_id: str | None = None

    def to_kwic_string(self, context_width: int = 5) -> str:
        """Format as KWIC (Key Word In Context) string."""
        left = " ".join(self.left_context[-context_width:]).rjust(40)
        right = " ".join(self.right_context[:context_width]).ljust(40)
        return f"{left} | {self.gloss:^15} | {right}"


@dataclass
class NGram:
    """An n-gram (sequence) of glosses."""

    glosses: tuple[str, ...]
    count: int
    examples: list[dict[str, Any]] = field(default_factory=list)

    @property
    def n(self) -> int:
        return len(self.glosses)

    def __str__(self) -> str:
        return " → ".join(self.glosses)


@dataclass
class CooccurrenceResult:
    """Results from co-occurrence analysis."""

    target_gloss: str
    collocates: dict[str, int]  # gloss -> count
    left_collocates: dict[str, int]
    right_collocates: dict[str, int]
    total_occurrences: int
    window_size: int

    def top_collocates(self, n: int = 20) -> list[tuple[str, int]]:
        """Get top n most frequent collocates."""
        return sorted(self.collocates.items(), key=lambda x: -x[1])[:n]

    def pmi(self, collocate: str, corpus_freq: dict[str, int], total_tokens: int) -> float:
        """
        Calculate Pointwise Mutual Information for a collocate.

        PMI = log2(P(target, collocate) / (P(target) * P(collocate)))

        Args:
            collocate: The collocate gloss to calculate PMI for
            corpus_freq: Dictionary mapping glosses to their corpus frequency
            total_tokens: Total number of tokens in the corpus

        Returns:
            PMI value, or 0.0 if calculation is not possible

        Raises:
            ValueError: If total_tokens is not positive
        """
        # Validate total_tokens
        if total_tokens <= 0:
            raise ValueError(f"total_tokens must be positive, got {total_tokens}")

        if collocate not in self.collocates or collocate not in corpus_freq:
            return 0.0

        # Guard against zero frequencies
        collocate_freq = corpus_freq.get(collocate, 0)
        joint_freq = self.collocates.get(collocate, 0)

        if self.total_occurrences == 0 or collocate_freq == 0 or joint_freq == 0:
            logger.debug(
                "PMI calculation returned 0.0 due to zero frequency: "
                "target_occurrences=%d, collocate_freq=%d, joint_freq=%d",
                self.total_occurrences,
                collocate_freq,
                joint_freq,
            )
            return 0.0

        p_target = self.total_occurrences / total_tokens
        p_collocate = collocate_freq / total_tokens
        p_joint = joint_freq / total_tokens

        # Additional safeguard for product being zero (should not happen given checks above)
        denominator = p_target * p_collocate
        if denominator == 0:
            return 0.0

        return np.log2(p_joint / denominator)


@dataclass
class DurationAnalysis:
    """Analysis of sign duration patterns."""

    gloss: str
    count: int
    mean_duration: float
    std_duration: float
    min_duration: float
    max_duration: float
    median_duration: float
    quartiles: tuple[float, float, float]  # Q1, Q2, Q3
    histogram_bins: list[float]
    histogram_counts: list[int]
    by_signer: dict[str, dict[str, float]] = field(default_factory=dict)


@dataclass
class SegmentReference:
    """Reference to a specific segment occurrence in the corpus."""

    file_id: str
    segment_idx: int
    start_time: float
    end_time: float
    tier: str = "gloss"

    def to_dict(self) -> dict[str, Any]:
        """Convert to dictionary for JSON serialization."""
        return {
            "file_id": self.file_id,
            "segment_idx": self.segment_idx,
            "start_time": self.start_time,
            "end_time": self.end_time,
            "tier": self.tier,
        }


@dataclass
class MinimalPair:
    """
    A minimal pair of signs.

    A minimal pair consists of two signs that differ in exactly one
    phonological parameter (for phonological mode) or have high string
    similarity (for legacy mode).

    Attributes:
        sign1: First sign gloss
        sign2: Second sign gloss
        differing_parameter: The phonological parameter that differs
            (e.g., "handshape_dom", "location"). None for legacy mode.
        distance: Phonological distance (integer) or string dissimilarity (float).
            For phonological mode, 1 means exact minimal pair.
            For string similarity mode, lower values mean more similar.
        method: Either "phonological" or "string_similarity"
        examples_sign1: List of segment references for sign1 occurrences
        examples_sign2: List of segment references for sign2 occurrences

    Example:
        >>> pair = MinimalPair(
        ...     sign1="MOTHER",
        ...     sign2="FATHER",
        ...     differing_parameter="location",
        ...     distance=1,
        ...     method="phonological",
        ...     examples_sign1=[SegmentReference("file1", 0, 1.0, 1.5)],
        ...     examples_sign2=[SegmentReference("file1", 5, 3.0, 3.5)],
        ... )
    """

    sign1: str
    sign2: str
    differing_parameter: str | None  # e.g., "handshape", "location", None for legacy
    distance: int | float  # Phonological distance or string dissimilarity
    method: str  # "phonological" or "string_similarity"
    examples_sign1: list[SegmentReference] = field(default_factory=list)
    examples_sign2: list[SegmentReference] = field(default_factory=list)

    def to_dict(self) -> dict[str, Any]:
        """Convert to dictionary for JSON serialization."""
        return {
            "sign1": self.sign1,
            "sign2": self.sign2,
            "differing_parameter": self.differing_parameter,
            "distance": self.distance,
            "method": self.method,
            "examples_sign1": [e.to_dict() for e in self.examples_sign1],
            "examples_sign2": [e.to_dict() for e in self.examples_sign2],
        }

    @property
    def is_true_minimal_pair(self) -> bool:
        """Check if this is a true minimal pair (differs in exactly one parameter)."""
        return self.method == "phonological" and self.distance == 1


def build_concordance(
    segments_by_file: dict[str, list[Any]],
    target_gloss: str,
    context_window: int = 5,
    case_sensitive: bool = False,
    signer_ids: dict[str, str] | None = None,
) -> list[ConcordanceLine]:
    """
    Build a concordance (KWIC view) for a target gloss.

    Args:
        segments_by_file: Dictionary mapping file IDs to segment lists
        target_gloss: The gloss to search for
        context_window: Number of glosses on each side
        case_sensitive: Whether to match case
        signer_ids: Optional mapping of file IDs to signer IDs

    Returns:
        List of ConcordanceLine objects
    """
    results = []
    target = target_gloss if case_sensitive else target_gloss.lower()

    for file_id, segments in segments_by_file.items():
        # Extract glosses with their metadata
        glosses = []
        for i, seg in enumerate(segments):
            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if label:
                glosses.append(
                    {
                        "idx": i,
                        "gloss": label,
                        "start": (
                            getattr(seg, "start", 0)
                            if hasattr(seg, "start")
                            else seg.get("start", 0)
                        ),
                        "end": getattr(seg, "end", 0) if hasattr(seg, "end") else seg.get("end", 0),
                        "tier": (
                            getattr(seg, "tier", "default")
                            if hasattr(seg, "tier")
                            else seg.get("tier", "default")
                        ),
                    }
                )

        # Find target occurrences
        for i, g in enumerate(glosses):
            gloss_text = g["gloss"] if case_sensitive else g["gloss"].lower()
            if gloss_text == target:
                # Get context
                left_context = [glosses[j]["gloss"] for j in range(max(0, i - context_window), i)]
                right_context = [
                    glosses[j]["gloss"]
                    for j in range(i + 1, min(len(glosses), i + context_window + 1))
                ]

                results.append(
                    ConcordanceLine(
                        file_id=file_id,
                        segment_idx=g["idx"],
                        gloss=g["gloss"],
                        left_context=left_context,
                        right_context=right_context,
                        start_time=g["start"],
                        end_time=g["end"],
                        duration=g["end"] - g["start"],
                        tier=g["tier"],
                        signer_id=signer_ids.get(file_id) if signer_ids else None,
                    )
                )

    return results


def analyze_cooccurrence(
    segments_by_file: dict[str, list[Any]],
    target_gloss: str,
    window_size: int = 3,
    case_sensitive: bool = False,
) -> CooccurrenceResult:
    """
    Analyze co-occurrence patterns for a target gloss.

    Args:
        segments_by_file: Dictionary mapping file IDs to segment lists
        target_gloss: The gloss to analyze
        window_size: Context window size
        case_sensitive: Whether to match case

    Returns:
        CooccurrenceResult with collocate frequencies
    """
    collocates: Counter[str] = Counter()
    left_collocates: Counter[str] = Counter()
    right_collocates: Counter[str] = Counter()
    total_occurrences = 0

    target = target_gloss if case_sensitive else target_gloss.lower()

    for segments in segments_by_file.values():
        glosses = []
        for seg in segments:
            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if label:
                glosses.append(label if case_sensitive else label.lower())

        for i, gloss in enumerate(glosses):
            if gloss == target:
                total_occurrences += 1

                # Left context
                for j in range(max(0, i - window_size), i):
                    collocates[glosses[j]] += 1
                    left_collocates[glosses[j]] += 1

                # Right context
                for j in range(i + 1, min(len(glosses), i + window_size + 1)):
                    collocates[glosses[j]] += 1
                    right_collocates[glosses[j]] += 1

    return CooccurrenceResult(
        target_gloss=target_gloss,
        collocates=dict(collocates),
        left_collocates=dict(left_collocates),
        right_collocates=dict(right_collocates),
        total_occurrences=total_occurrences,
        window_size=window_size,
    )


def extract_ngrams(
    segments_by_file: dict[str, list[Any]],
    n: int = 2,
    min_frequency: int = 2,
    max_ngrams: int = 100,
    include_examples: bool = True,
) -> list[NGram]:
    """
    Extract frequent n-grams (sign sequences) from corpus.

    Args:
        segments_by_file: Dictionary mapping file IDs to segment lists
        n: N-gram size (2 = bigrams, 3 = trigrams, etc.)
        min_frequency: Minimum frequency to include
        max_ngrams: Maximum number of n-grams to return
        include_examples: Whether to include example locations

    Returns:
        List of NGram objects sorted by frequency
    """
    ngram_counts: Counter[tuple[str, ...]] = Counter()
    ngram_examples: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list)

    for file_id, segments in segments_by_file.items():
        glosses = []
        for i, seg in enumerate(segments):
            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if label:
                glosses.append((label, i))

        # Extract n-grams
        for i in range(len(glosses) - n + 1):
            ngram = tuple(g[0] for g in glosses[i : i + n])
            ngram_counts[ngram] += 1

            if include_examples and len(ngram_examples[ngram]) < 5:
                ngram_examples[ngram].append(
                    {
                        "file_id": file_id,
                        "start_idx": glosses[i][1],
                        "end_idx": glosses[i + n - 1][1],
                    }
                )

    # Filter and sort
    results = []
    for ngram, count in ngram_counts.most_common(max_ngrams):
        if count >= min_frequency:
            results.append(
                NGram(
                    glosses=ngram,
                    count=count,
                    examples=ngram_examples[ngram] if include_examples else [],
                )
            )

    return results


def analyze_duration_patterns(
    segments_by_file: dict[str, list[Any]],
    gloss: str | None = None,
    min_occurrences: int = 5,
    signer_ids: dict[str, str] | None = None,
) -> dict[str, DurationAnalysis]:
    """
    Analyze sign duration patterns.

    Args:
        segments_by_file: Dictionary mapping file IDs to segment lists
        gloss: Specific gloss to analyze (None = all glosses)
        min_occurrences: Minimum occurrences for analysis
        signer_ids: Optional mapping of file IDs to signer IDs

    Returns:
        Dictionary mapping glosses to DurationAnalysis
    """
    # Collect durations by gloss and signer
    durations_by_gloss = defaultdict(list)
    durations_by_gloss_signer: dict[str, dict[str, list[float]]] = defaultdict(
        lambda: defaultdict(list)
    )

    for file_id, segments in segments_by_file.items():
        signer = signer_ids.get(file_id, "unknown") if signer_ids else "unknown"

        for seg in segments:
            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if not label:
                continue

            if gloss and label.lower() != gloss.lower():
                continue

            start = getattr(seg, "start", 0) if hasattr(seg, "start") else seg.get("start", 0)
            end = getattr(seg, "end", 0) if hasattr(seg, "end") else seg.get("end", 0)
            duration = end - start

            if duration > 0:
                durations_by_gloss[label].append(duration)
                durations_by_gloss_signer[label][signer].append(duration)

    # Compute statistics
    results = {}
    for g, durations in durations_by_gloss.items():
        if len(durations) < min_occurrences:
            continue

        durations_array = np.array(durations)

        # Skip if array is empty (shouldn't happen after min_occurrences check, but be safe)
        if durations_array.size == 0:
            continue

        # Histogram
        max_duration = float(durations_array.max())
        bins = np.linspace(0, min(2.0, max_duration), 21)
        hist, bin_edges = np.histogram(durations_array, bins=bins)

        # Per-signer stats
        by_signer = {}
        for signer, signer_durations in durations_by_gloss_signer[g].items():
            if len(signer_durations) >= 3:
                sd = np.array(signer_durations)
                by_signer[signer] = {
                    "mean": float(sd.mean()),
                    "std": float(sd.std()),
                    "count": len(signer_durations),
                }

        results[g] = DurationAnalysis(
            gloss=g,
            count=len(durations),
            mean_duration=float(durations_array.mean()),
            std_duration=float(durations_array.std()),
            min_duration=float(durations_array.min()),
            max_duration=float(durations_array.max()),
            median_duration=float(np.median(durations_array)),
            quartiles=(
                float(np.percentile(durations_array, 25)),
                float(np.percentile(durations_array, 50)),
                float(np.percentile(durations_array, 75)),
            ),
            histogram_bins=bin_edges.tolist(),
            histogram_counts=hist.tolist(),
            by_signer=by_signer,
        )

    return results


def find_minimal_pairs(
    vocabulary: list[str] | None = None,
    segments_by_file: dict[str, list[Any]] | None = None,
    tier: str = "gloss",
    phonological_forms: dict[str, PhonologicalForm] | None = None,
    use_phonology: bool = True,
    max_distance: int = 1,
    legacy_threshold: float = 0.8,
    max_pairs: int = 100,
    max_vocab_size: int = 2000,
    include_examples: bool = True,
) -> list[MinimalPair]:
    """
    Find minimal pairs in the corpus.

    If phonological_forms is provided and use_phonology=True, uses true
    phonological comparison (signs differing in exactly one parameter).

    Otherwise, falls back to string similarity (legacy behavior).

    Args:
        vocabulary: List of glosses (optional if segments_by_file provided)
        segments_by_file: Dictionary mapping file IDs to segment lists
            (used for extracting vocabulary and examples)
        tier: Tier to extract labels from (only used with segments_by_file)
        phonological_forms: Optional dict mapping glosses to PhonologicalForm
        use_phonology: Whether to use phonological comparison when available
        max_distance: Maximum phonological distance for near-minimal pairs
            (1 = true minimal pairs, 2+ = near-minimal pairs)
        legacy_threshold: String similarity threshold for legacy mode
        max_pairs: Maximum number of pairs to return
        max_vocab_size: Maximum vocabulary size to process (for performance)
        include_examples: Whether to include example segment references

    Returns:
        List of MinimalPair objects with sign pairs and their relationship

    Examples:
        >>> # Legacy mode with vocabulary list
        >>> pairs = find_minimal_pairs(vocabulary=["HELLO", "HALLO", "HELP"])

        >>> # Phonological mode with forms
        >>> forms = {"MOTHER": mother_form, "FATHER": father_form}
        >>> pairs = find_minimal_pairs(phonological_forms=forms)

        >>> # Mixed mode - phonological where available, fallback for others
        >>> pairs = find_minimal_pairs(
        ...     segments_by_file=segments,
        ...     phonological_forms=partial_forms,
        ...     use_phonology=True,
        ... )
    """
    # Extract vocabulary from segments if not provided
    if vocabulary is None and segments_by_file is not None:
        vocabulary = _extract_vocabulary_from_segments(segments_by_file, tier)
    elif vocabulary is None:
        vocabulary = list(phonological_forms.keys()) if phonological_forms else []

    if not vocabulary:
        return []

    # Build example lookup if needed
    examples_by_gloss: dict[str, list[SegmentReference]] = {}
    if include_examples and segments_by_file is not None:
        examples_by_gloss = _build_example_lookup(segments_by_file, tier)

    # Determine which mode to use
    use_phon_mode = use_phonology and phonological_forms is not None and len(phonological_forms) > 0

    if use_phon_mode:
        return _find_phonological_minimal_pairs(
            vocabulary=vocabulary,
            phonological_forms=phonological_forms,  # type: ignore
            max_distance=max_distance,
            max_pairs=max_pairs,
            examples_by_gloss=examples_by_gloss,
            legacy_threshold=legacy_threshold,
            max_vocab_size=max_vocab_size,
        )
    else:
        return _find_string_similarity_pairs(
            vocabulary=vocabulary,
            similarity_threshold=legacy_threshold,
            max_pairs=max_pairs,
            max_vocab_size=max_vocab_size,
            examples_by_gloss=examples_by_gloss,
        )


def _extract_vocabulary_from_segments(
    segments_by_file: dict[str, list[Any]], tier: str
) -> list[str]:
    """Extract unique vocabulary from segments."""
    vocab = set()
    for segments in segments_by_file.values():
        for seg in segments:
            # Get tier if available
            seg_tier = (
                getattr(seg, "tier", "gloss")
                if hasattr(seg, "tier")
                else seg.get("tier", "gloss") if isinstance(seg, dict) else "gloss"
            )
            if seg_tier != tier:
                continue

            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if label:
                vocab.add(label)
    return list(vocab)


def _build_example_lookup(
    segments_by_file: dict[str, list[Any]], tier: str, max_examples: int = 5
) -> dict[str, list[SegmentReference]]:
    """Build lookup of example segment references by gloss."""
    examples: dict[str, list[SegmentReference]] = defaultdict(list)

    for file_id, segments in segments_by_file.items():
        for i, seg in enumerate(segments):
            # Get tier if available
            seg_tier = (
                getattr(seg, "tier", "gloss")
                if hasattr(seg, "tier")
                else seg.get("tier", "gloss") if isinstance(seg, dict) else "gloss"
            )
            if seg_tier != tier:
                continue

            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if not label:
                continue

            if len(examples[label]) < max_examples:
                start = getattr(seg, "start", 0) if hasattr(seg, "start") else seg.get("start", 0)
                end = getattr(seg, "end", 0) if hasattr(seg, "end") else seg.get("end", 0)
                examples[label].append(
                    SegmentReference(
                        file_id=file_id,
                        segment_idx=i,
                        start_time=start,
                        end_time=end,
                        tier=tier,
                    )
                )

    return dict(examples)


def _find_phonological_minimal_pairs(
    vocabulary: list[str],
    phonological_forms: dict[str, PhonologicalForm],
    max_distance: int,
    max_pairs: int,
    examples_by_gloss: dict[str, list[SegmentReference]],
    legacy_threshold: float,
    max_vocab_size: int,
) -> list[MinimalPair]:
    """
    Find minimal pairs using phonological comparison.

    Signs with phonological forms are compared phonologically.
    Signs without phonological forms fall back to string similarity.
    """
    # Lazy import to avoid circular dependencies
    from sltk.linguistics.phonology import (
        get_differing_parameters,
        phonological_distance,
    )

    pairs: list[MinimalPair] = []
    vocab_list = sorted(set(vocabulary))

    # Limit vocabulary size for performance
    if len(vocab_list) > max_vocab_size:
        vocab_list = vocab_list[:max_vocab_size]

    # Separate signs with and without phonological forms
    signs_with_forms = [g for g in vocab_list if g in phonological_forms]
    signs_without_forms = [g for g in vocab_list if g not in phonological_forms]

    # Compare signs with phonological forms
    for i, g1 in enumerate(signs_with_forms):
        for g2 in signs_with_forms[i + 1 :]:
            form1 = phonological_forms[g1]
            form2 = phonological_forms[g2]

            dist = phonological_distance(form1, form2)

            if dist <= max_distance:
                differing = get_differing_parameters(form1, form2)
                diff_param = differing[0] if len(differing) == 1 else ",".join(differing)

                # Sort names alphabetically for consistent ordering
                if g1 > g2:
                    g1, g2 = g2, g1

                pairs.append(
                    MinimalPair(
                        sign1=g1,
                        sign2=g2,
                        differing_parameter=diff_param,
                        distance=dist,
                        method="phonological",
                        examples_sign1=examples_by_gloss.get(g1, []),
                        examples_sign2=examples_by_gloss.get(g2, []),
                    )
                )

    # Fall back to string similarity for signs without phonological forms
    if signs_without_forms:
        legacy_pairs = _find_string_similarity_pairs(
            vocabulary=signs_without_forms,
            similarity_threshold=legacy_threshold,
            max_pairs=max_pairs - len(pairs),
            max_vocab_size=max_vocab_size,
            examples_by_gloss=examples_by_gloss,
        )
        pairs.extend(legacy_pairs)

    # Sort by distance (phonological) or similarity (string)
    # For phonological pairs, lower distance is better
    # For string pairs, distance is 1-similarity, so lower is also better
    pairs.sort(key=lambda x: x.distance)
    return pairs[:max_pairs]


def _find_string_similarity_pairs(
    vocabulary: list[str],
    similarity_threshold: float,
    max_pairs: int,
    max_vocab_size: int,
    examples_by_gloss: dict[str, list[SegmentReference]],
) -> list[MinimalPair]:
    """
    Find minimal pairs using string similarity (legacy mode).

    Uses string edit distance as a proxy for phonological similarity.
    """
    from difflib import SequenceMatcher

    pairs: list[MinimalPair] = []
    vocab_list = sorted(set(vocabulary))

    # Limit vocabulary size for performance (O(n^2) algorithm)
    if len(vocab_list) > max_vocab_size:
        vocab_list = vocab_list[:max_vocab_size]

    # Group glosses by length for faster filtering
    by_length: dict[int, list[str]] = defaultdict(list)
    for g in vocab_list:
        by_length[len(g)].append(g)

    # Only compare glosses of similar length (within 3 characters)
    for length, glosses in by_length.items():
        # Compare within same length
        for i, g1 in enumerate(glosses):
            for g2 in glosses[i + 1 :]:
                similarity = SequenceMatcher(None, g1.lower(), g2.lower()).ratio()
                if similarity >= similarity_threshold:
                    # Sort names alphabetically for consistent ordering
                    if g1 > g2:
                        g1, g2 = g2, g1

                    pairs.append(
                        MinimalPair(
                            sign1=g1,
                            sign2=g2,
                            differing_parameter=None,
                            distance=1.0 - similarity,  # Convert to distance
                            method="string_similarity",
                            examples_sign1=examples_by_gloss.get(g1, []),
                            examples_sign2=examples_by_gloss.get(g2, []),
                        )
                    )

        # Compare with adjacent lengths (within 3)
        for offset in range(1, 4):
            if length + offset in by_length:
                adjacent_glosses = by_length[length + offset]
                for g1 in glosses:
                    for g2 in adjacent_glosses:
                        similarity = SequenceMatcher(None, g1.lower(), g2.lower()).ratio()
                        if similarity >= similarity_threshold:
                            # Sort names alphabetically for consistent ordering
                            if g1 > g2:
                                g1, g2 = g2, g1

                            pairs.append(
                                MinimalPair(
                                    sign1=g1,
                                    sign2=g2,
                                    differing_parameter=None,
                                    distance=1.0 - similarity,
                                    method="string_similarity",
                                    examples_sign1=examples_by_gloss.get(g1, []),
                                    examples_sign2=examples_by_gloss.get(g2, []),
                                )
                            )

    # Sort by distance (lower is more similar)
    pairs.sort(key=lambda x: x.distance)
    return pairs[:max_pairs]


def analyze_signer_variation(
    segments_by_file: dict[str, list[Any]],
    gloss: str,
    signer_ids: dict[str, str],
) -> dict[str, Any]:
    """
    Analyze variation in how different signers produce a sign.

    Args:
        segments_by_file: Dictionary mapping file IDs to segment lists
        gloss: The gloss to analyze
        signer_ids: Mapping of file IDs to signer IDs

    Returns:
        Dictionary with per-signer statistics and examples
    """
    signer_data: dict[str, dict[str, list[Any]]] = defaultdict(
        lambda: {
            "occurrences": [],
            "durations": [],
            "examples": [],
        }
    )

    target = gloss.lower()

    for file_id, segments in segments_by_file.items():
        signer = signer_ids.get(file_id, "unknown")

        for i, seg in enumerate(segments):
            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if not label or label.lower() != target:
                continue

            start = getattr(seg, "start", 0) if hasattr(seg, "start") else seg.get("start", 0)
            end = getattr(seg, "end", 0) if hasattr(seg, "end") else seg.get("end", 0)

            signer_data[signer]["occurrences"].append(
                {
                    "file_id": file_id,
                    "segment_idx": i,
                    "start": start,
                    "end": end,
                }
            )
            signer_data[signer]["durations"].append(end - start)

            if len(signer_data[signer]["examples"]) < 5:
                signer_data[signer]["examples"].append(
                    {
                        "file_id": file_id,
                        "start": start,
                        "end": end,
                    }
                )

    # Compute statistics
    results: dict[str, Any] = {
        "gloss": gloss,
        "signers": {},
        "summary": {
            "num_signers": len(signer_data),
            "total_occurrences": sum(len(d["occurrences"]) for d in signer_data.values()),
        },
    }

    all_durations = []
    for signer, data in signer_data.items():
        durations = np.array(data["durations"]) if data["durations"] else np.array([0])
        all_durations.extend(data["durations"])

        results["signers"][signer] = {
            "count": len(data["occurrences"]),
            "mean_duration": float(durations.mean()),
            "std_duration": float(durations.std()) if len(durations) > 1 else 0,
            "examples": data["examples"],
        }

    # Overall statistics
    if all_durations:
        all_dur = np.array(all_durations)
        results["summary"]["mean_duration"] = float(all_dur.mean())
        results["summary"]["std_duration"] = float(all_dur.std())
        results["summary"]["duration_range"] = [float(all_dur.min()), float(all_dur.max())]

    return results


def compute_cooccurrence_matrix(
    segments_by_file: dict[str, list[Any]],
    top_n_glosses: int = 50,
    window_size: int = 2,
) -> dict[str, Any]:
    """
    Compute a co-occurrence matrix for top glosses.

    Args:
        segments_by_file: Dictionary mapping file IDs to segment lists
        top_n_glosses: Number of top glosses to include
        window_size: Context window for co-occurrence

    Returns:
        Dictionary with matrix, glosses, and raw counts
    """
    # Count glosses
    gloss_counts: Counter[str] = Counter()
    for segments in segments_by_file.values():
        for seg in segments:
            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if label:
                gloss_counts[label] += 1

    # Get top glosses
    top_glosses = [g for g, _ in gloss_counts.most_common(top_n_glosses)]
    gloss_set = set(top_glosses)
    gloss_to_idx = {g: i for i, g in enumerate(top_glosses)}

    # Build co-occurrence matrix
    n = len(top_glosses)
    matrix = np.zeros((n, n), dtype=np.int32)

    for segments in segments_by_file.values():
        glosses = []
        for seg in segments:
            label = getattr(seg, "label", None) or (
                seg.get("label") if isinstance(seg, dict) else None
            )
            if label and label in gloss_set:
                glosses.append(label)

        for i, g1 in enumerate(glosses):
            idx1 = gloss_to_idx[g1]
            for j in range(max(0, i - window_size), min(len(glosses), i + window_size + 1)):
                if i != j:
                    g2 = glosses[j]
                    idx2 = gloss_to_idx[g2]
                    matrix[idx1, idx2] += 1

    return {
        "glosses": top_glosses,
        "matrix": matrix.tolist(),
        "frequencies": {g: gloss_counts[g] for g in top_glosses},
    }
