"""
Non-manual signal (NMS) analysis tools for sign language research.

Non-manual signals include facial expressions, mouth patterns, eye gaze,
head movements, and body postures. In sign languages, these carry crucial
grammatical information:
- Eyebrow raise: yes/no questions, topics, conditionals
- Eyebrow furrow: wh-questions, negation
- Head shake: negation
- Head nod: affirmation
- Mouth patterns: adverbial modification

This module provides tools for:
- NMS co-occurrence analysis with manual signs
- NMS scope analysis (how many signs an NMS spans)
- Grammatical pattern detection
- NMS duration and timing statistics
- NMS inventory analysis
"""

from __future__ import annotations

import logging
import math
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from typing import Any, Literal

import numpy as np

from sltk.data.core import Segment, SegmentList

logger = logging.getLogger(__name__)

# Standard tier names for non-manual features with common aliases
NMS_TIERS: dict[str, list[str]] = {
    "eyebrows": ["eyebrow", "brow", "eb", "eyebrows", "brows"],
    "mouth": ["mouth", "mouthing", "mp", "mouth_pattern", "oral"],
    "head": ["head", "head_mvt", "head_movement", "head_motion"],
    "eye_gaze": ["gaze", "eye_gaze", "eyes", "eye", "look"],
    "body": ["body", "body_lean", "torso", "trunk", "body_posture"],
}

# Grammatical markers typically associated with specific NMS
GRAMMATICAL_MARKERS: dict[str, dict[str, Any]] = {
    "wh_question": {
        "nms": ["furrowed", "furrow", "lowered", "squint", "brow_down"],
        "tier_preference": ["eyebrows", "head"],
        "manual_signs": ["who", "what", "where", "when", "why", "how", "which"],
    },
    "yn_question": {
        "nms": ["raised", "raise", "brow_up", "up"],
        "tier_preference": ["eyebrows"],
        "manual_signs": [],  # Often spans entire clause
    },
    "negation": {
        "nms": ["shake", "headshake", "neg", "no"],
        "tier_preference": ["head"],
        "manual_signs": ["not", "no", "never", "nothing", "don't", "can't", "won't"],
    },
    "topic": {
        "nms": ["raised", "raise", "brow_up", "topic"],
        "tier_preference": ["eyebrows", "head"],
        "manual_signs": [],  # Marked by position and NMS, not specific signs
    },
    "conditional": {
        "nms": ["raised", "raise", "brow_up", "tilt", "forward"],
        "tier_preference": ["eyebrows", "head", "body"],
        "manual_signs": ["if", "suppose", "imagine"],
    },
    "affirmation": {
        "nms": ["nod", "headnod", "yes", "affirm"],
        "tier_preference": ["head"],
        "manual_signs": ["yes", "right", "correct", "true"],
    },
}


# ============================================================================
# Result dataclasses
# ============================================================================


@dataclass
class ScopeExample:
    """An example of NMS scope over manual signs."""

    nms_label: str
    nms_start: float
    nms_end: float
    manual_signs: list[str]
    manual_times: list[tuple[float, float]]  # (start, end) for each sign

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dictionary."""
        return {
            "nms_label": self.nms_label,
            "nms_start": self.nms_start,
            "nms_end": self.nms_end,
            "manual_signs": self.manual_signs,
            "manual_times": self.manual_times,
        }


@dataclass
class NMSCooccurrenceResult:
    """Results from NMS co-occurrence analysis."""

    manual_to_nms: dict[str, list[tuple[str, float]]]  # sign -> [(nms, association)]
    nms_to_manual: dict[str, list[tuple[str, float]]]  # nms -> [(sign, association)]
    total_manual_signs: int
    total_nms_tokens: int
    chi_square_results: dict[str, dict[str, float]] = field(default_factory=dict)
    pmi_results: dict[str, dict[str, float]] = field(default_factory=dict)

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dictionary."""
        return {
            "manual_to_nms": self.manual_to_nms,
            "nms_to_manual": self.nms_to_manual,
            "total_manual_signs": self.total_manual_signs,
            "total_nms_tokens": self.total_nms_tokens,
            "chi_square_results": self.chi_square_results,
            "pmi_results": self.pmi_results,
        }

    def top_associations_for_sign(self, sign: str, n: int = 10) -> list[tuple[str, float]]:
        """Get top NMS associations for a manual sign."""
        if sign not in self.manual_to_nms:
            return []
        return sorted(self.manual_to_nms[sign], key=lambda x: -x[1])[:n]

    def top_associations_for_nms(self, nms: str, n: int = 10) -> list[tuple[str, float]]:
        """Get top manual sign associations for an NMS."""
        if nms not in self.nms_to_manual:
            return []
        return sorted(self.nms_to_manual[nms], key=lambda x: -x[1])[:n]


@dataclass
class NMSScopeResult:
    """Results from NMS scope analysis."""

    avg_scope: float  # Average number of manual signs per NMS
    scope_distribution: dict[int, int]  # scope_length -> count
    examples: list[ScopeExample]
    total_nms_spans: int
    single_sign_count: int  # NMS spanning exactly one sign
    multi_sign_count: int  # NMS spanning multiple signs

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dictionary."""
        return {
            "avg_scope": self.avg_scope,
            "scope_distribution": self.scope_distribution,
            "examples": [e.to_dict() for e in self.examples],
            "total_nms_spans": self.total_nms_spans,
            "single_sign_count": self.single_sign_count,
            "multi_sign_count": self.multi_sign_count,
            "single_sign_ratio": (
                self.single_sign_count / self.total_nms_spans if self.total_nms_spans > 0 else 0.0
            ),
        }

    @property
    def single_sign_ratio(self) -> float:
        """Proportion of NMS that span only a single sign."""
        if self.total_nms_spans == 0:
            return 0.0
        return self.single_sign_count / self.total_nms_spans


@dataclass
class GrammaticalPattern:
    """A detected grammatical pattern marked by NMS."""

    pattern_type: str  # "wh_question", "yn_question", "negation", "topic", etc.
    start: float
    end: float
    nms_markers: list[str]  # NMS labels involved
    manual_signs: list[str]  # Manual signs in the pattern
    confidence: float  # 0.0-1.0 confidence score
    tier_sources: list[str] = field(default_factory=list)  # Which tiers provided evidence

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dictionary."""
        return {
            "pattern_type": self.pattern_type,
            "start": self.start,
            "end": self.end,
            "nms_markers": self.nms_markers,
            "manual_signs": self.manual_signs,
            "confidence": self.confidence,
            "tier_sources": self.tier_sources,
            "duration": self.end - self.start,
        }

    @property
    def duration(self) -> float:
        """Duration of the pattern in seconds."""
        return self.end - self.start


@dataclass
class NMSDurationStats:
    """Statistical analysis of NMS durations."""

    nms_tier: str
    count: int
    mean: float
    median: float
    std: float
    min_val: float
    max_val: float
    quartiles: tuple[float, float, float]  # Q1, Q2 (median), Q3
    histogram_bins: list[float]
    histogram_counts: list[int]
    by_label: dict[str, dict[str, float]] = field(default_factory=dict)

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dictionary."""
        return {
            "nms_tier": self.nms_tier,
            "count": self.count,
            "mean": self.mean,
            "median": self.median,
            "std": self.std,
            "min": self.min_val,
            "max": self.max_val,
            "quartiles": list(self.quartiles),
            "histogram_bins": self.histogram_bins,
            "histogram_counts": self.histogram_counts,
            "by_label": self.by_label,
        }


@dataclass
class NMSTimingResult:
    """Results from NMS timing analysis relative to manual signs."""

    avg_onset_diff: float  # NMS onset - manual onset (negative = NMS first)
    avg_offset_diff: float  # NMS offset - manual offset
    onset_distribution: list[float]  # All onset differences
    offset_distribution: list[float]  # All offset differences
    onset_std: float
    offset_std: float
    nms_leads_count: int  # NMS starts before manual sign
    nms_lags_count: int  # NMS starts after manual sign
    coincident_count: int  # Within tolerance window
    total_comparisons: int

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dictionary."""
        return {
            "avg_onset_diff": self.avg_onset_diff,
            "avg_offset_diff": self.avg_offset_diff,
            "onset_distribution": self.onset_distribution,
            "offset_distribution": self.offset_distribution,
            "onset_std": self.onset_std,
            "offset_std": self.offset_std,
            "nms_leads_count": self.nms_leads_count,
            "nms_lags_count": self.nms_lags_count,
            "coincident_count": self.coincident_count,
            "total_comparisons": self.total_comparisons,
            "nms_leads_ratio": (
                self.nms_leads_count / self.total_comparisons if self.total_comparisons > 0 else 0.0
            ),
        }


@dataclass
class NMSInventoryResult:
    """Results from NMS inventory analysis."""

    tier_inventories: dict[str, dict[str, int]]  # tier -> {label: count}
    total_tokens: int
    total_types: int
    cross_tier_combinations: list[dict[str, Any]]  # Common NMS combinations
    frequency_distribution: dict[str, int]  # Overall label frequencies
    examples_by_label: dict[str, list[dict[str, Any]]] = field(default_factory=dict)

    def to_dict(self) -> dict[str, Any]:
        """Convert to JSON-serializable dictionary."""
        return {
            "tier_inventories": self.tier_inventories,
            "total_tokens": self.total_tokens,
            "total_types": self.total_types,
            "cross_tier_combinations": self.cross_tier_combinations,
            "frequency_distribution": self.frequency_distribution,
            "examples_by_label": self.examples_by_label,
        }

    def top_nms(self, n: int = 20) -> list[tuple[str, int]]:
        """Get the n most frequent NMS labels."""
        return sorted(self.frequency_distribution.items(), key=lambda x: -x[1])[:n]


# ============================================================================
# Helper functions
# ============================================================================


def _normalize_tier_name(tier: str) -> str | None:
    """
    Map a tier name to its canonical NMS category.

    Args:
        tier: The tier name to normalize

    Returns:
        Canonical category name or None if not an NMS tier
    """
    tier_lower = tier.lower().strip()
    for category, aliases in NMS_TIERS.items():
        if tier_lower in aliases or any(alias in tier_lower for alias in aliases):
            return category
    return None


def _find_matching_tier(available_tiers: list[str], tier_aliases: list[str]) -> str | None:
    """
    Find a tier from available_tiers that matches any of the aliases.

    Args:
        available_tiers: List of tier names in the data
        tier_aliases: List of possible tier name patterns

    Returns:
        Matching tier name or None
    """
    for tier in available_tiers:
        tier_lower = tier.lower()
        for alias in tier_aliases:
            if alias.lower() in tier_lower or tier_lower == alias.lower():
                return tier
    return None


def _get_nms_tiers(segments: SegmentList, nms_tiers: list[str] | None = None) -> list[str]:
    """
    Get the NMS tiers to analyze.

    Args:
        segments: SegmentList to analyze
        nms_tiers: Optional explicit list of tier names

    Returns:
        List of tier names that are NMS tiers
    """
    available_tiers = segments.tiers

    if nms_tiers is not None:
        # User provided explicit tier names
        return [t for t in nms_tiers if t in available_tiers]

    # Auto-detect NMS tiers based on standard names
    detected = []
    for tier in available_tiers:
        if _normalize_tier_name(tier) is not None:
            detected.append(tier)

    return detected


def _segments_overlap(seg1: Segment, seg2: Segment, tolerance: float = 0.0) -> bool:
    """Check if two segments overlap (with optional tolerance for near-overlap)."""
    return seg1.start < seg2.end + tolerance and seg2.start < seg1.end + tolerance


def _get_overlapping_segments(
    target_seg: Segment, segments: list[Segment], tolerance: float = 0.0
) -> list[Segment]:
    """Get all segments that overlap with the target segment."""
    return [s for s in segments if _segments_overlap(target_seg, s, tolerance)]


# ============================================================================
# Main analysis functions
# ============================================================================


def analyze_nms_cooccurrence(
    segments: SegmentList,
    manual_tier: str = "gloss",
    nms_tiers: list[str] | None = None,
    overlap_tolerance: float = 0.05,
) -> NMSCooccurrenceResult:
    """
    Analyze which NMS co-occur with which manual signs.

    This function examines temporal overlap between NMS segments and manual sign
    segments to determine which non-manual markers typically accompany which signs.

    Args:
        segments: SegmentList containing both manual and NMS annotations
        manual_tier: Name of the tier containing manual sign glosses
        nms_tiers: Optional list of NMS tier names (auto-detected if None)
        overlap_tolerance: Time tolerance (seconds) for considering segments as
            co-occurring even if they don't perfectly overlap

    Returns:
        NMSCooccurrenceResult containing:
        - manual_to_nms: For each sign, which NMS typically co-occur (with scores)
        - nms_to_manual: For each NMS, which signs it appears with (with scores)
        - Statistical measures (chi-square, PMI)

    Example:
        >>> segments = SegmentList.from_elan("annotated_video.eaf")
        >>> result = analyze_nms_cooccurrence(segments, manual_tier="gloss")
        >>> print(result.top_associations_for_sign("QUESTION", n=5))
        [('raised', 0.85), ('forward', 0.42), ...]
    """
    available_tiers = segments.tiers

    # Check manual tier exists
    if manual_tier not in available_tiers:
        logger.warning(f"Manual tier '{manual_tier}' not found. Available: {available_tiers}")
        return NMSCooccurrenceResult(
            manual_to_nms={},
            nms_to_manual={},
            total_manual_signs=0,
            total_nms_tokens=0,
        )

    # Get NMS tiers
    detected_nms_tiers = _get_nms_tiers(segments, nms_tiers)
    if not detected_nms_tiers:
        logger.warning(f"No NMS tiers found. Available: {available_tiers}")
        return NMSCooccurrenceResult(
            manual_to_nms={},
            nms_to_manual={},
            total_manual_signs=0,
            total_nms_tokens=0,
        )

    # Get segments by tier
    manual_segs = list(segments.filter_by_tier(manual_tier))
    nms_segs: list[Segment] = []
    for tier in detected_nms_tiers:
        nms_segs.extend(segments.filter_by_tier(tier))

    if not manual_segs or not nms_segs:
        return NMSCooccurrenceResult(
            manual_to_nms={},
            nms_to_manual={},
            total_manual_signs=len(manual_segs),
            total_nms_tokens=len(nms_segs),
        )

    # Count co-occurrences
    manual_nms_counts: dict[str, Counter] = defaultdict(Counter)
    nms_manual_counts: dict[str, Counter] = defaultdict(Counter)
    manual_counts: Counter[str] = Counter()
    nms_counts: Counter[str] = Counter()

    for manual_seg in manual_segs:
        if not manual_seg.label:
            continue

        manual_label = manual_seg.label
        manual_counts[manual_label] += 1

        # Find overlapping NMS
        overlapping = _get_overlapping_segments(manual_seg, nms_segs, overlap_tolerance)
        for nms_seg in overlapping:
            if nms_seg.label:
                nms_label = nms_seg.label
                manual_nms_counts[manual_label][nms_label] += 1
                nms_manual_counts[nms_label][manual_label] += 1

    # Count NMS totals
    for nms_seg in nms_segs:
        if nms_seg.label:
            nms_counts[nms_seg.label] += 1

    total_manual = sum(manual_counts.values())
    total_nms = sum(nms_counts.values())

    # Calculate association scores using PMI (Pointwise Mutual Information)
    manual_to_nms: dict[str, list[tuple[str, float]]] = {}
    nms_to_manual: dict[str, list[tuple[str, float]]] = {}
    pmi_results: dict[str, dict[str, float]] = defaultdict(dict)

    total_pairs = total_manual  # Each manual sign is a context window

    for manual_label, nms_counter in manual_nms_counts.items():
        associations = []
        for nms_label, joint_count in nms_counter.items():
            # Calculate PMI: log2(P(m,n) / (P(m) * P(n)))
            if total_pairs > 0 and manual_counts[manual_label] > 0 and nms_counts[nms_label] > 0:
                p_joint = joint_count / total_pairs
                p_manual = manual_counts[manual_label] / total_pairs
                p_nms = nms_counts[nms_label] / total_nms if total_nms > 0 else 0

                if p_joint > 0 and p_manual > 0 and p_nms > 0:
                    pmi = math.log2(p_joint / (p_manual * p_nms))
                    # Normalize PMI to [0, 1] range approximately
                    normalized_pmi = 1 / (1 + math.exp(-pmi))
                    associations.append((nms_label, normalized_pmi))
                    pmi_results[manual_label][nms_label] = pmi
                else:
                    associations.append((nms_label, 0.0))
            else:
                associations.append((nms_label, 0.0))

        manual_to_nms[manual_label] = sorted(associations, key=lambda x: -x[1])

    for nms_label, manual_counter in nms_manual_counts.items():
        associations = []
        for manual_label, joint_count in manual_counter.items():
            if total_pairs > 0 and nms_counts[nms_label] > 0 and manual_counts[manual_label] > 0:
                p_joint = joint_count / total_pairs
                p_nms = nms_counts[nms_label] / total_nms if total_nms > 0 else 0
                p_manual = manual_counts[manual_label] / total_pairs

                if p_joint > 0 and p_nms > 0 and p_manual > 0:
                    pmi = math.log2(p_joint / (p_nms * p_manual))
                    normalized_pmi = 1 / (1 + math.exp(-pmi))
                    associations.append((manual_label, normalized_pmi))
                else:
                    associations.append((manual_label, 0.0))
            else:
                associations.append((manual_label, 0.0))

        nms_to_manual[nms_label] = sorted(associations, key=lambda x: -x[1])

    return NMSCooccurrenceResult(
        manual_to_nms=manual_to_nms,
        nms_to_manual=nms_to_manual,
        total_manual_signs=total_manual,
        total_nms_tokens=total_nms,
        pmi_results=dict(pmi_results),
    )


def analyze_nms_scope(
    segments: SegmentList,
    nms_tier: str,
    manual_tier: str = "gloss",
    max_examples: int = 10,
) -> NMSScopeResult:
    """
    Analyze the scope of NMS over manual signs.

    NMS often spans multiple manual signs. For example, a question marker might
    span an entire clause, or negation headshake might span the verb and its
    arguments. This function quantifies these scoping patterns.

    Args:
        segments: SegmentList containing both manual and NMS annotations
        nms_tier: Name of the NMS tier to analyze
        manual_tier: Name of the tier containing manual sign glosses
        max_examples: Maximum number of scope examples to include

    Returns:
        NMSScopeResult containing:
        - avg_scope: Average number of manual signs per NMS span
        - scope_distribution: Histogram of scope lengths
        - examples: Representative examples of different scope patterns
        - single_sign vs multi_sign counts

    Example:
        >>> result = analyze_nms_scope(segments, nms_tier="head", manual_tier="gloss")
        >>> print(f"Average scope: {result.avg_scope:.2f} signs")
        >>> print(f"Multi-sign NMS: {result.multi_sign_count} ({result.single_sign_ratio:.1%})")
    """
    available_tiers = segments.tiers

    if nms_tier not in available_tiers:
        logger.warning(f"NMS tier '{nms_tier}' not found. Available: {available_tiers}")
        return NMSScopeResult(
            avg_scope=0.0,
            scope_distribution={},
            examples=[],
            total_nms_spans=0,
            single_sign_count=0,
            multi_sign_count=0,
        )

    if manual_tier not in available_tiers:
        logger.warning(f"Manual tier '{manual_tier}' not found. Available: {available_tiers}")
        return NMSScopeResult(
            avg_scope=0.0,
            scope_distribution={},
            examples=[],
            total_nms_spans=0,
            single_sign_count=0,
            multi_sign_count=0,
        )

    nms_segs = list(segments.filter_by_tier(nms_tier))
    manual_segs = sorted(segments.filter_by_tier(manual_tier), key=lambda s: s.start)

    if not nms_segs or not manual_segs:
        return NMSScopeResult(
            avg_scope=0.0,
            scope_distribution={},
            examples=[],
            total_nms_spans=0,
            single_sign_count=0,
            multi_sign_count=0,
        )

    scope_counts = []
    scope_distribution: Counter = Counter()
    examples: list[ScopeExample] = []
    examples_by_scope: dict[int, list[ScopeExample]] = defaultdict(list)

    for nms_seg in nms_segs:
        # Find manual signs that overlap with this NMS
        overlapping = _get_overlapping_segments(nms_seg, manual_segs)
        scope = len(overlapping)

        if scope > 0:
            scope_counts.append(scope)
            scope_distribution[scope] += 1

            # Collect example
            if len(examples_by_scope[scope]) < 3:
                example = ScopeExample(
                    nms_label=nms_seg.label or "",
                    nms_start=nms_seg.start,
                    nms_end=nms_seg.end,
                    manual_signs=[s.label or "" for s in overlapping],
                    manual_times=[(s.start, s.end) for s in overlapping],
                )
                examples_by_scope[scope].append(example)

    # Compile examples (prioritize diverse scope lengths)
    for scope in sorted(examples_by_scope.keys()):
        for ex in examples_by_scope[scope]:
            if len(examples) < max_examples:
                examples.append(ex)

    avg_scope = float(np.mean(scope_counts)) if scope_counts else 0.0
    single_count = scope_distribution.get(1, 0)
    multi_count = sum(c for s, c in scope_distribution.items() if s > 1)

    return NMSScopeResult(
        avg_scope=avg_scope,
        scope_distribution=dict(scope_distribution),
        examples=examples,
        total_nms_spans=len(scope_counts),
        single_sign_count=single_count,
        multi_sign_count=multi_count,
    )


def detect_grammatical_patterns(
    segments: SegmentList,
    pattern_type: Literal["question", "negation", "topic", "conditional"] | None = None,
    confidence_threshold: float = 0.3,
) -> list[GrammaticalPattern]:
    """
    Detect grammatical constructions marked by NMS.

    Sign languages use NMS to mark various grammatical constructions:
    - WH-questions: furrowed brows + wh-sign
    - Y/N questions: raised brows over clause
    - Negation: headshake + (optional) negative sign
    - Topic: raised brows over topic constituent

    Args:
        segments: SegmentList containing annotations
        pattern_type: Specific pattern type to detect (None for all types)
        confidence_threshold: Minimum confidence score to include pattern

    Returns:
        List of detected GrammaticalPattern objects with timing and confidence

    Example:
        >>> patterns = detect_grammatical_patterns(segments, pattern_type="question")
        >>> for p in patterns:
        ...     print(f"{p.pattern_type} at {p.start:.2f}-{p.end:.2f} (conf: {p.confidence:.2f})")
    """
    available_tiers = segments.tiers
    results: list[GrammaticalPattern] = []

    # Determine which pattern types to look for
    if pattern_type == "question":
        types_to_check = ["wh_question", "yn_question"]
    elif pattern_type is not None:
        types_to_check = [pattern_type]
    else:
        types_to_check = list(GRAMMATICAL_MARKERS.keys())

    # Build tier lookup
    tier_segments: dict[str, list[Segment]] = {}
    for tier in available_tiers:
        tier_segments[tier] = list(segments.filter_by_tier(tier))

    # Check each pattern type
    for ptype in types_to_check:
        if ptype not in GRAMMATICAL_MARKERS:
            continue

        marker_info = GRAMMATICAL_MARKERS[ptype]
        expected_nms = marker_info["nms"]
        tier_preferences = marker_info["tier_preference"]
        manual_sign_hints = marker_info.get("manual_signs", [])

        # Find relevant NMS tiers
        relevant_tiers = []
        for pref in tier_preferences:
            if pref in NMS_TIERS:
                for tier in available_tiers:
                    if _normalize_tier_name(tier) == pref:
                        relevant_tiers.append(tier)
                        break

        # Scan NMS segments for pattern markers
        for tier in relevant_tiers:
            for nms_seg in tier_segments.get(tier, []):
                if not nms_seg.label:
                    continue

                nms_label_lower = nms_seg.label.lower()

                # Check if this NMS matches expected markers
                match_score = 0.0
                for expected in expected_nms:
                    if expected.lower() in nms_label_lower:
                        match_score = 1.0
                        break
                    elif any(c in nms_label_lower for c in expected.lower().split("_")):
                        match_score = max(match_score, 0.5)

                if match_score == 0:
                    continue

                # Find manual signs that overlap
                manual_signs = []
                gloss_tier = None
                for t in available_tiers:
                    if "gloss" in t.lower() or t.lower() == "gloss":
                        gloss_tier = t
                        break

                if gloss_tier and gloss_tier in tier_segments:
                    overlapping = _get_overlapping_segments(nms_seg, tier_segments[gloss_tier])
                    manual_signs = [s.label or "" for s in overlapping if s.label]

                # Adjust confidence based on manual sign hints
                confidence = match_score * 0.6  # Base confidence from NMS match

                if manual_sign_hints and manual_signs:
                    for sign in manual_signs:
                        if sign.lower() in [h.lower() for h in manual_sign_hints]:
                            confidence += 0.3
                            break

                # Boost confidence if pattern spans expected scope
                if ptype in ["yn_question", "topic", "conditional"]:
                    # These typically span multiple signs
                    if len(manual_signs) > 1:
                        confidence += 0.1

                confidence = min(confidence, 1.0)

                if confidence >= confidence_threshold:
                    results.append(
                        GrammaticalPattern(
                            pattern_type=ptype,
                            start=nms_seg.start,
                            end=nms_seg.end,
                            nms_markers=[nms_seg.label],
                            manual_signs=manual_signs,
                            confidence=confidence,
                            tier_sources=[tier],
                        )
                    )

    # Sort by start time
    results.sort(key=lambda p: p.start)

    return results


def nms_duration_stats(
    segments: SegmentList,
    nms_tier: str,
) -> NMSDurationStats:
    """
    Statistical analysis of NMS durations.

    Analyzes the temporal characteristics of NMS annotations on a specific tier.

    Args:
        segments: SegmentList containing NMS annotations
        nms_tier: Name of the NMS tier to analyze

    Returns:
        NMSDurationStats with mean, median, std, min, max durations and histogram

    Example:
        >>> stats = nms_duration_stats(segments, nms_tier="head")
        >>> print(f"Mean duration: {stats.mean:.3f}s, Std: {stats.std:.3f}s")
    """
    available_tiers = segments.tiers

    if nms_tier not in available_tiers:
        logger.warning(f"NMS tier '{nms_tier}' not found. Available: {available_tiers}")
        return NMSDurationStats(
            nms_tier=nms_tier,
            count=0,
            mean=0.0,
            median=0.0,
            std=0.0,
            min_val=0.0,
            max_val=0.0,
            quartiles=(0.0, 0.0, 0.0),
            histogram_bins=[],
            histogram_counts=[],
        )

    nms_segs = list(segments.filter_by_tier(nms_tier))

    if not nms_segs:
        return NMSDurationStats(
            nms_tier=nms_tier,
            count=0,
            mean=0.0,
            median=0.0,
            std=0.0,
            min_val=0.0,
            max_val=0.0,
            quartiles=(0.0, 0.0, 0.0),
            histogram_bins=[],
            histogram_counts=[],
        )

    durations = np.array([s.duration for s in nms_segs])

    # Per-label statistics
    by_label: dict[str, dict[str, float]] = {}
    label_durations: dict[str, list[float]] = defaultdict(list)

    for seg in nms_segs:
        if seg.label:
            label_durations[seg.label].append(seg.duration)

    for label, durs in label_durations.items():
        durs_arr = np.array(durs)
        by_label[label] = {
            "count": len(durs),
            "mean": float(durs_arr.mean()),
            "std": float(durs_arr.std()) if len(durs) > 1 else 0.0,
            "median": float(np.median(durs_arr)),
        }

    # Histogram
    max_dur = float(durations.max())
    bin_max = min(5.0, max_dur * 1.1)  # Cap at 5 seconds or slightly above max
    bins = np.linspace(0, bin_max, 21)
    hist_counts, hist_edges = np.histogram(durations, bins=bins)

    return NMSDurationStats(
        nms_tier=nms_tier,
        count=len(durations),
        mean=float(durations.mean()),
        median=float(np.median(durations)),
        std=float(durations.std()),
        min_val=float(durations.min()),
        max_val=float(durations.max()),
        quartiles=(
            float(np.percentile(durations, 25)),
            float(np.percentile(durations, 50)),
            float(np.percentile(durations, 75)),
        ),
        histogram_bins=hist_edges.tolist(),
        histogram_counts=hist_counts.tolist(),
        by_label=by_label,
    )


def analyze_nms_timing(
    segments: SegmentList,
    nms_tier: str,
    manual_tier: str = "gloss",
    coincidence_tolerance: float = 0.05,
) -> NMSTimingResult:
    """
    Analyze timing relationship between NMS and manual signs.

    Examines whether NMS onset/offset precedes, coincides with, or follows
    the onset/offset of manual signs.

    Args:
        segments: SegmentList containing both manual and NMS annotations
        nms_tier: Name of the NMS tier to analyze
        manual_tier: Name of the tier containing manual sign glosses
        coincidence_tolerance: Time window (seconds) for considering onset/offset
            as coincident

    Returns:
        NMSTimingResult with:
        - avg_onset_diff: Negative means NMS typically starts before manual sign
        - avg_offset_diff: Negative means NMS typically ends before manual sign
        - Distribution of timing differences
        - Counts of leads/lags/coincident

    Example:
        >>> timing = analyze_nms_timing(segments, nms_tier="eyebrow", manual_tier="gloss")
        >>> if timing.avg_onset_diff < 0:
        ...     print(f"NMS typically leads by {abs(timing.avg_onset_diff)*1000:.0f}ms")
    """
    available_tiers = segments.tiers

    if nms_tier not in available_tiers or manual_tier not in available_tiers:
        missing = []
        if nms_tier not in available_tiers:
            missing.append(f"nms_tier '{nms_tier}'")
        if manual_tier not in available_tiers:
            missing.append(f"manual_tier '{manual_tier}'")
        logger.warning(f"Missing tiers: {', '.join(missing)}. Available: {available_tiers}")
        return NMSTimingResult(
            avg_onset_diff=0.0,
            avg_offset_diff=0.0,
            onset_distribution=[],
            offset_distribution=[],
            onset_std=0.0,
            offset_std=0.0,
            nms_leads_count=0,
            nms_lags_count=0,
            coincident_count=0,
            total_comparisons=0,
        )

    nms_segs = list(segments.filter_by_tier(nms_tier))
    manual_segs = list(segments.filter_by_tier(manual_tier))

    if not nms_segs or not manual_segs:
        return NMSTimingResult(
            avg_onset_diff=0.0,
            avg_offset_diff=0.0,
            onset_distribution=[],
            offset_distribution=[],
            onset_std=0.0,
            offset_std=0.0,
            nms_leads_count=0,
            nms_lags_count=0,
            coincident_count=0,
            total_comparisons=0,
        )

    onset_diffs = []
    offset_diffs = []
    leads = 0
    lags = 0
    coincident = 0

    for nms_seg in nms_segs:
        # Find overlapping manual signs
        overlapping = _get_overlapping_segments(nms_seg, manual_segs)

        for manual_seg in overlapping:
            # Onset difference: NMS onset - manual onset
            onset_diff = nms_seg.start - manual_seg.start
            onset_diffs.append(onset_diff)

            # Offset difference: NMS offset - manual offset
            offset_diff = nms_seg.end - manual_seg.end
            offset_diffs.append(offset_diff)

            # Classify timing relationship
            if onset_diff < -coincidence_tolerance:
                leads += 1  # NMS starts before manual
            elif onset_diff > coincidence_tolerance:
                lags += 1  # NMS starts after manual
            else:
                coincident += 1  # Approximately coincident

    if not onset_diffs:
        return NMSTimingResult(
            avg_onset_diff=0.0,
            avg_offset_diff=0.0,
            onset_distribution=[],
            offset_distribution=[],
            onset_std=0.0,
            offset_std=0.0,
            nms_leads_count=0,
            nms_lags_count=0,
            coincident_count=0,
            total_comparisons=0,
        )

    onset_arr = np.array(onset_diffs)
    offset_arr = np.array(offset_diffs)

    return NMSTimingResult(
        avg_onset_diff=float(onset_arr.mean()),
        avg_offset_diff=float(offset_arr.mean()),
        onset_distribution=onset_diffs,
        offset_distribution=offset_diffs,
        onset_std=float(onset_arr.std()),
        offset_std=float(offset_arr.std()),
        nms_leads_count=leads,
        nms_lags_count=lags,
        coincident_count=coincident,
        total_comparisons=len(onset_diffs),
    )


def analyze_nms_inventory(
    segments: SegmentList,
    nms_tiers: list[str] | None = None,
    max_examples_per_label: int = 3,
) -> NMSInventoryResult:
    """
    Analyze the inventory of NMS types in the corpus.

    Provides a comprehensive overview of NMS labels used, their frequencies,
    and how they combine across tiers.

    Args:
        segments: SegmentList containing NMS annotations
        nms_tiers: Optional list of NMS tier names (auto-detected if None)
        max_examples_per_label: Maximum number of examples to store per label

    Returns:
        NMSInventoryResult with:
        - tier_inventories: Per-tier label frequencies
        - Cross-tier combinations (e.g., "raised eyebrows + forward lean")
        - Overall frequency distribution
        - Example segments for each label

    Example:
        >>> inventory = analyze_nms_inventory(segments)
        >>> print(f"Total NMS types: {inventory.total_types}")
        >>> for nms, count in inventory.top_nms(10):
        ...     print(f"  {nms}: {count}")
    """
    detected_nms_tiers = _get_nms_tiers(segments, nms_tiers)

    if not detected_nms_tiers:
        logger.warning(f"No NMS tiers found. Available: {segments.tiers}")
        return NMSInventoryResult(
            tier_inventories={},
            total_tokens=0,
            total_types=0,
            cross_tier_combinations=[],
            frequency_distribution={},
        )

    # Per-tier inventories
    tier_inventories: dict[str, dict[str, int]] = {}
    overall_freq: Counter = Counter()
    examples_by_label: dict[str, list[dict[str, Any]]] = defaultdict(list)

    for tier in detected_nms_tiers:
        tier_segs = list(segments.filter_by_tier(tier))
        tier_counter: Counter = Counter()

        for seg in tier_segs:
            if seg.label:
                tier_counter[seg.label] += 1
                overall_freq[seg.label] += 1

                # Store examples
                if len(examples_by_label[seg.label]) < max_examples_per_label:
                    examples_by_label[seg.label].append(
                        {
                            "tier": tier,
                            "start": seg.start,
                            "end": seg.end,
                            "duration": seg.duration,
                        }
                    )

        tier_inventories[tier] = dict(tier_counter)

    # Find cross-tier combinations (NMS from different tiers that co-occur)
    cross_tier_combinations: list[dict[str, Any]] = []

    if len(detected_nms_tiers) > 1:
        # Build time-indexed lookup for efficient overlap detection
        combination_counts: Counter = Counter()

        # Get all segments with their tiers
        tier_seg_lists = {tier: list(segments.filter_by_tier(tier)) for tier in detected_nms_tiers}

        # For each segment in first tier, find overlapping segments in other tiers
        base_tier = detected_nms_tiers[0]
        for base_seg in tier_seg_lists[base_tier]:
            if not base_seg.label:
                continue

            combo_parts = [(base_tier, base_seg.label)]

            for other_tier in detected_nms_tiers[1:]:
                overlapping = _get_overlapping_segments(base_seg, tier_seg_lists[other_tier])
                for other_seg in overlapping:
                    if other_seg.label:
                        combo_parts.append((other_tier, other_seg.label))

            if len(combo_parts) > 1:
                # Sort to ensure consistent key
                combo_key = tuple(sorted(combo_parts))
                combination_counts[combo_key] += 1

        # Convert to list format
        for combo, count in combination_counts.most_common(20):
            cross_tier_combinations.append(
                {
                    "components": [{"tier": t, "label": l} for t, l in combo],
                    "count": count,
                }
            )

    total_tokens = sum(overall_freq.values())
    total_types = len(overall_freq)

    return NMSInventoryResult(
        tier_inventories=tier_inventories,
        total_tokens=total_tokens,
        total_types=total_types,
        cross_tier_combinations=cross_tier_combinations,
        frequency_distribution=dict(overall_freq),
        examples_by_label=dict(examples_by_label),
    )


# ============================================================================
# Convenience functions
# ============================================================================


def get_standard_nms_tiers() -> dict[str, list[str]]:
    """
    Get the standard NMS tier names and their aliases.

    Returns:
        Dictionary mapping canonical tier names to lists of aliases

    Example:
        >>> tiers = get_standard_nms_tiers()
        >>> print(tiers["eyebrows"])
        ['eyebrow', 'brow', 'eb', 'eyebrows', 'brows']
    """
    return NMS_TIERS.copy()


def get_grammatical_markers() -> dict[str, dict[str, Any]]:
    """
    Get the grammatical marker definitions.

    Returns:
        Dictionary mapping pattern types to their marker definitions

    Example:
        >>> markers = get_grammatical_markers()
        >>> print(markers["wh_question"]["nms"])
        ['furrowed', 'furrow', 'lowered', 'squint', 'brow_down']
    """
    return {k: v.copy() for k, v in GRAMMATICAL_MARKERS.items()}


def summarize_nms(segments: SegmentList) -> dict[str, Any]:
    """
    Generate a comprehensive summary of NMS in a SegmentList.

    Convenience function that runs multiple analyses and combines results.

    Args:
        segments: SegmentList containing NMS annotations

    Returns:
        Dictionary with summary statistics and key findings

    Example:
        >>> summary = summarize_nms(segments)
        >>> print(f"Found {summary['total_nms_tokens']} NMS annotations")
        >>> print(f"Most common: {summary['top_nms'][:5]}")
    """
    inventory = analyze_nms_inventory(segments)

    # Try to find a gloss tier
    gloss_tier = None
    for tier in segments.tiers:
        if "gloss" in tier.lower():
            gloss_tier = tier
            break

    # Get scope analysis for each detected NMS tier
    detected_tiers = _get_nms_tiers(segments, None)
    scope_summaries = {}

    for tier in detected_tiers:
        if gloss_tier:
            scope_result = analyze_nms_scope(segments, tier, gloss_tier)
            scope_summaries[tier] = {
                "avg_scope": scope_result.avg_scope,
                "single_sign_ratio": scope_result.single_sign_ratio,
            }

    # Detect grammatical patterns
    patterns = detect_grammatical_patterns(segments)
    pattern_summary = Counter(p.pattern_type for p in patterns)

    return {
        "total_nms_tokens": inventory.total_tokens,
        "total_nms_types": inventory.total_types,
        "top_nms": inventory.top_nms(20),
        "tier_inventories": inventory.tier_inventories,
        "cross_tier_combinations": inventory.cross_tier_combinations[:10],
        "scope_summaries": scope_summaries,
        "grammatical_patterns_detected": len(patterns),
        "pattern_type_counts": dict(pattern_summary),
    }
