"""
Analysis and linguistic endpoints for SLTK API.

Handles quality metrics, vocabulary analysis, and linguistic analysis from ELAN files.
"""

from __future__ import annotations

import asyncio
import json
import logging
import threading
import time
from collections import Counter
from pathlib import Path
from typing import Any

import numpy as np
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, Request

from sltk.api.dependencies import rate_limit, safe_error_detail, validate_path
from sltk.api.models import (
    AnnotationSampleSummary,
    BatchExportRequest,
    BatchSearchRequest,
    BatchSearchResponse,
    BatchSearchResultItem,
    ConcordanceRequest,
    ContextPattern,
    ContextPatternsResponse,
    CooccurrenceMatrixRequest,
    CooccurrenceRequest,
    CrossDatasetInfo,
    CrossDatasetSearchResponse,
    DatasetComparisonRequest,
    DatasetStatistics,
    DurationAnalysisRequest,
    DurationComparisonResponse,
    DurationStatistics,
    ExportTrainingDataRequest,
    GlossMatchResponse,
    GlossSearchRequest,
    MinimalPairsRequest,
    NGramRequest,
    ParallelExampleMatch,
    ParallelExamplesRequest,
    ParallelExamplesResponse,
    QualityMetricsRequest,
    QualityMetricsResponse,
    SegmentData,
    UnifiedAnnotationResponse,
    UnifiedAnnotationsResponse,
    UnifiedDatasetInfo,
    VideoQualityMetrics,
    VocabularyComparisonRequest,
    VocabularyComparisonResponse,
    VocabularyMappingRequest,
)

router = APIRouter(tags=["analysis"])
logger = logging.getLogger(__name__)

# ============================================================================
# Unified Annotations Cache
# ============================================================================

# Cache for manifests (5 minute TTL)
_manifest_cache: dict[str, tuple[dict, float]] = {}
_cache_lock = threading.Lock()
MANIFEST_CACHE_TTL = 300  # 5 minutes


def _get_cached_manifest(dataset: str) -> dict | None:
    """Get manifest from cache if fresh, else load and cache. Thread-safe."""
    from sltk.config import get_config

    # Check cache first with lock
    with _cache_lock:
        if dataset in _manifest_cache:
            manifest, cached_at = _manifest_cache[dataset]
            if time.time() - cached_at < MANIFEST_CACHE_TTL:
                return manifest

    config = get_config()
    annotation_dir = config.annotation_root / dataset

    if not annotation_dir.exists():
        return None

    manifest_path = annotation_dir / "manifest.json"
    if not manifest_path.exists():
        return None

    try:
        with open(manifest_path, encoding="utf-8") as f:
            manifest = json.load(f)
        with _cache_lock:
            _manifest_cache[dataset] = (manifest, time.time())
        return manifest
    except (OSError, json.JSONDecodeError, TypeError) as e:
        logger.debug(f"Failed to read manifest for {dataset}: {e}")
        return None


# ============================================================================
# Quality Metrics
# ============================================================================


@router.post("/api/analysis/quality", response_model=QualityMetricsResponse)
@rate_limit("30/minute")
async def get_quality_metrics(request: Request, body: QualityMetricsRequest):
    """
    Get extraction quality metrics for processed videos.

    Analyzes H5 pose files to compute detection rates, confidence scores,
    temporal gaps, and per-keypoint quality metrics.

    Rate limited to 30 requests per minute. Max 50 files per request.
    """
    from sltk.io.h5 import get_h5_metadata, load_poses_h5

    # Validate paths first (outside the thread)
    validated_paths: list[tuple[str, Path | None, str | None]] = []
    for h5_path_str in body.h5_paths:
        try:
            h5_path = validate_path(h5_path_str, must_exist=True)
            validated_paths.append((h5_path_str, h5_path, None))
        except HTTPException as e:
            validated_paths.append((h5_path_str, None, str(e.detail)))

    def _process_files():
        """Sync helper to process all H5 files."""
        metrics_list = []
        num_successful = 0
        num_failed = 0

        # Aggregated stats for summary
        all_detection_rates: list[float] = []
        all_confidences: list[float] = []
        all_gaps: list[int] = []

        confidence_threshold = 0.5  # Frames below this are "low confidence"

        for h5_path_str, h5_path, validation_error in validated_paths:
            if validation_error:
                metrics_list.append(VideoQualityMetrics(path=h5_path_str, error=validation_error))
                num_failed += 1
                continue

            try:
                # Load pose data
                get_h5_metadata(h5_path)
                poses = load_poses_h5(h5_path)

                metrics = VideoQualityMetrics(
                    path=str(h5_path),
                    format=poses.format,
                    num_frames=poses.num_frames,
                    fps=poses.fps,
                    duration_sec=(
                        poses.duration
                        if hasattr(poses, "duration")
                        else (poses.num_frames / poses.fps if poses.fps else None)
                    ),
                )

                # Detection coverage
                if body.check_coverage:
                    # Frame has detection if any keypoint is non-zero
                    data = poses.data  # Shape: (num_frames, num_keypoints, dims)
                    valid_mask = np.any(data != 0, axis=(1, 2))
                    metrics.frames_with_detection = int(valid_mask.sum())
                    metrics.detection_rate = round(
                        metrics.frames_with_detection / max(poses.num_frames, 1) * 100, 2
                    )
                    all_detection_rates.append(metrics.detection_rate)

                    # Find gaps in detection
                    if body.check_temporal:
                        gap_starts = []
                        gap_ends = []
                        in_gap = False
                        gap_start = 0

                        for i, is_valid in enumerate(valid_mask):
                            if not is_valid and not in_gap:
                                in_gap = True
                                gap_start = i
                            elif is_valid and in_gap:
                                in_gap = False
                                gap_ends.append(i - 1)
                                gap_starts.append(gap_start)

                        if in_gap:
                            gap_starts.append(gap_start)
                            gap_ends.append(len(valid_mask) - 1)

                        metrics.num_gaps = len(gap_starts)
                        if gap_starts:
                            gap_lengths = [e - s + 1 for s, e in zip(gap_starts, gap_ends)]
                            metrics.max_gap_frames = max(gap_lengths)
                            # Only store first 10 gaps
                            metrics.gap_locations = list(zip(gap_starts[:10], gap_ends[:10]))
                            all_gaps.extend(gap_lengths)

                # Confidence scores
                if body.check_confidence and poses.confidence is not None:
                    conf = poses.confidence
                    # Filter out zero/invalid confidence values
                    valid_conf = conf[conf > 0]
                    if len(valid_conf) > 0:
                        metrics.mean_confidence = round(float(valid_conf.mean()), 4)
                        metrics.min_confidence = round(float(valid_conf.min()), 4)
                        metrics.max_confidence = round(float(valid_conf.max()), 4)
                        all_confidences.append(metrics.mean_confidence)

                        # Count low confidence frames
                        frame_mean_conf = conf.mean(axis=1)
                        metrics.low_confidence_frames = int(
                            (frame_mean_conf < confidence_threshold).sum()
                        )

                # Per-keypoint detection rates
                if body.check_coverage and poses.num_keypoints > 0:
                    data = poses.data
                    keypoint_rates = {}
                    for kp_idx in range(min(poses.num_keypoints, 50)):  # Limit to 50 keypoints
                        kp_valid = np.any(data[:, kp_idx, :] != 0, axis=1)
                        rate = round(kp_valid.sum() / max(poses.num_frames, 1) * 100, 2)
                        keypoint_rates[f"kp_{kp_idx}"] = rate
                    metrics.keypoint_detection_rates = keypoint_rates

                metrics_list.append(metrics)
                num_successful += 1

            except Exception as e:
                metrics_list.append(VideoQualityMetrics(path=h5_path_str, error=str(e)))
                num_failed += 1

        # Compute summary statistics
        summary: dict[str, Any] = {
            "files_analyzed": num_successful,
            "files_failed": num_failed,
        }

        if all_detection_rates:
            summary["mean_detection_rate"] = round(np.mean(all_detection_rates), 2)
            summary["min_detection_rate"] = round(min(all_detection_rates), 2)
            summary["max_detection_rate"] = round(max(all_detection_rates), 2)

        if all_confidences:
            summary["mean_confidence"] = round(np.mean(all_confidences), 4)

        if all_gaps:
            summary["total_gaps"] = len(all_gaps)
            summary["mean_gap_length"] = round(np.mean(all_gaps), 1)
            summary["max_gap_length"] = max(all_gaps)

        return metrics_list, num_successful, num_failed, summary

    metrics_list, num_successful, num_failed, summary = await asyncio.to_thread(_process_files)

    return QualityMetricsResponse(
        num_files=len(body.h5_paths),
        num_successful=num_successful,
        num_failed=num_failed,
        metrics=metrics_list,
        summary=summary,
    )


@router.get("/api/analysis/vocabulary")
async def get_vocabulary_stats(elan_paths: list[str] = Query(default=[])):
    """Get vocabulary statistics from ELAN files."""
    from sltk.io.elan import read_eaf

    all_labels = []
    for path in elan_paths:
        try:
            segments = read_eaf(path)
            all_labels.extend([s.label for s in segments if s.label])
        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {path}: {e}")
            continue

    counter = Counter(all_labels)

    return {
        "total_tokens": len(all_labels),
        "vocabulary_size": len(counter),
        "frequencies": dict(counter.most_common(100)),
    }


# ============================================================================
# Batch Processing
# ============================================================================


@router.post("/api/batch/export")
@rate_limit("30/minute")
async def batch_export_segments(request: Request, body: BatchExportRequest):
    """
    Export multiple ELAN files to a unified format.

    Useful for creating training datasets or analysis spreadsheets.

    Rate limited to 30 requests per minute.
    """
    from sltk.io.elan import read_eaf

    # Security: Validate output path is within allowed directories
    output_path_validated = validate_path(body.output_path, must_exist=False, allow_creation=True)

    all_data = []
    errors = []

    for elan_path in body.elan_paths:
        try:
            # Security: Validate each input path
            path = validate_path(elan_path, must_exist=False)
            if not path.exists():
                errors.append({"path": elan_path, "error": "File not found"})
                continue

            segments = read_eaf(path)

            for seg in segments:
                row = {
                    "file": path.stem,
                    "start": seg.start,
                    "end": seg.end,
                    "duration": seg.duration,
                    "label": seg.label,
                    "tier": seg.tier,
                }
                if body.include_metadata and seg.metadata:
                    row.update(seg.metadata)
                all_data.append(row)

        except Exception as e:
            errors.append({"path": elan_path, "error": str(e)})

    if not all_data:
        raise HTTPException(400, "No data to export")

    # Use the validated output path
    output_path = output_path_validated
    output_path.parent.mkdir(parents=True, exist_ok=True)

    try:
        if body.output_format == "csv":
            import csv

            with open(output_path, "w", newline="", encoding="utf-8") as f:
                writer = csv.DictWriter(f, fieldnames=all_data[0].keys())
                writer.writeheader()
                writer.writerows(all_data)

        elif body.output_format == "json":
            with open(output_path, "w", encoding="utf-8") as f:
                json.dump(all_data, f, indent=2, ensure_ascii=False)

        elif body.output_format == "parquet":
            try:
                import pandas as pd

                df = pd.DataFrame(all_data)
                df.to_parquet(output_path, index=False)
            except ImportError:
                raise HTTPException(400, "pandas required for parquet export")

        return {
            "status": "success",
            "output_path": str(output_path),
            "total_segments": len(all_data),
            "files_processed": len(body.elan_paths) - len(errors),
            "errors": errors,
        }

    except Exception as e:
        raise HTTPException(500, safe_error_detail("Export failed", e))


@router.post("/api/batch/statistics", response_model=DatasetStatistics)
async def compute_batch_statistics(elan_paths: list[str]):
    """
    Compute statistics across multiple ELAN files.

    Useful for dataset analysis and quality assessment.
    """
    from sltk.io.elan import read_eaf

    total_segments = 0
    total_duration = 0.0
    all_glosses = []
    tier_counts: Counter[str] = Counter()
    num_samples = 0

    for elan_path in elan_paths:
        try:
            segments = read_eaf(elan_path)
            num_samples += 1
            total_segments += len(segments)

            for seg in segments:
                total_duration += seg.duration
                if seg.label:
                    all_glosses.append(seg.label)
                if seg.tier:
                    tier_counts[seg.tier] += 1

        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {elan_path}: {e}")
            continue

    if num_samples == 0:
        raise HTTPException(400, "No valid files processed")

    gloss_counts = Counter(all_glosses)

    return DatasetStatistics(
        num_samples=num_samples,
        total_segments=total_segments,
        total_duration_sec=round(total_duration, 2),
        vocabulary_size=len(gloss_counts),
        avg_segments_per_sample=round(total_segments / num_samples, 2),
        avg_segment_duration_sec=round(total_duration / max(total_segments, 1), 3),
        tier_counts=dict(tier_counts),
        top_glosses=dict(gloss_counts.most_common(50)),
    )


# ============================================================================
# Unified Annotations
# ============================================================================


@router.get("/api/annotations/datasets", response_model=UnifiedAnnotationsResponse)
async def list_annotation_datasets():
    """
    List all datasets with unified annotations available.

    Returns dataset names, statistics, and available splits.
    """
    from sltk.config import get_config

    config = get_config()
    annotation_root = config.annotation_root

    datasets = []
    if annotation_root.exists():
        for dataset_dir in sorted(annotation_root.iterdir()):
            if dataset_dir.is_dir():
                manifest = _get_cached_manifest(dataset_dir.name)
                if manifest:
                    stats = manifest.get("statistics", {})
                    datasets.append(
                        UnifiedDatasetInfo(
                            name=manifest.get("dataset", dataset_dir.name),
                            total_samples=stats.get("total_samples", 0),
                            unique_glosses=stats.get("unique_glosses", 0),
                            unique_signers=stats.get("unique_signers", 0),
                            tiers=stats.get("tiers", []),
                            samples_by_split=stats.get("samples_by_split", {}),
                            source_format=manifest.get("source_format", "unknown"),
                            version=manifest.get("version", "1.0.0"),
                        )
                    )

    return UnifiedAnnotationsResponse(
        datasets=datasets,
        annotation_root=str(annotation_root),
        total_datasets=len(datasets),
    )


@router.get("/api/annotations/{dataset}/samples")
async def list_annotation_samples(
    dataset: str,
    split: str | None = None,
    limit: int = Query(100, ge=1, le=1000),
    offset: int = Query(0, ge=0),
):
    """
    List samples from a dataset's unified annotations.

    Args:
        dataset: Dataset name
        split: Optional filter by split (train/val/test)
        limit: Maximum samples to return
        offset: Pagination offset
    """
    manifest = _get_cached_manifest(dataset)

    if not manifest:
        raise HTTPException(404, f"No annotations found for dataset: {dataset}")

    samples = manifest.get("samples", [])

    # Filter by split if specified
    if split:
        samples = [s for s in samples if s.get("split") == split]

    total = len(samples)
    samples = samples[offset : offset + limit]

    return {
        "dataset": dataset,
        "split": split,
        "samples": [
            AnnotationSampleSummary(
                sample_id=s["sample_id"],
                split=s.get("split"),
                num_segments=s.get("num_segments", 0),
                duration_sec=s.get("duration_sec"),
                signer_id=s.get("signer_id"),
            )
            for s in samples
        ],
        "total": total,
        "offset": offset,
        "limit": limit,
    }


@router.get(
    "/api/annotations/{dataset}/sample/{sample_id}",
    response_model=UnifiedAnnotationResponse,
)
async def get_annotation_sample(dataset: str, sample_id: str):
    """
    Get full annotation for a specific sample.

    Returns segments in the format expected by the frontend Timeline.
    """
    from sltk.config import get_config
    from sltk.io.annotations import read_unified_annotation

    config = get_config()
    annotation_dir = config.annotation_root / dataset
    annotation_path = annotation_dir / f"{sample_id}.json"

    if not annotation_path.exists():
        raise HTTPException(404, f"Sample not found: {dataset}/{sample_id}")

    try:
        sample = read_unified_annotation(annotation_path)

        # Build segments in frontend-compatible format
        segments = []
        tiers = set()
        if sample.segments:
            for seg in sample.segments:
                segments.append(
                    SegmentData(
                        start=seg.start,
                        end=seg.end,
                        label=seg.label,
                        confidence=seg.confidence,
                        tier=seg.tier,
                    )
                )
                tiers.add(seg.tier)

        return UnifiedAnnotationResponse(
            sample_id=sample.id,
            dataset=sample.dataset or dataset,
            schema_version="1.0.0",
            split=sample.split,
            signer_id=sample.signer_id,
            duration_sec=sample.metadata.get("duration_sec") if sample.metadata else None,
            fps=sample.metadata.get("fps") if sample.metadata else None,
            video_relative_path=(
                sample.metadata.get("video_relative_path") if sample.metadata else None
            ),
            glosses=sample.glosses,
            text=sample.text,
            segments=segments,
            tiers=sorted(list(tiers)),
            metadata=sample.metadata or {},
        )
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to load annotation", e))


@router.get("/api/annotations/{dataset}/sample/{sample_id}/segments")
async def get_sample_segments(
    dataset: str,
    sample_id: str,
    tier: str | None = None,
):
    """
    Get just the segments for a sample (lightweight endpoint for Timeline).

    Args:
        dataset: Dataset name
        sample_id: Sample identifier
        tier: Optional filter by tier name
    """
    from sltk.config import get_config
    from sltk.io.annotations import read_unified_annotation

    config = get_config()
    annotation_path = config.annotation_root / dataset / f"{sample_id}.json"

    if not annotation_path.exists():
        raise HTTPException(404, f"Sample not found: {dataset}/{sample_id}")

    try:
        sample = read_unified_annotation(annotation_path)

        segments = []
        tiers_found = set()
        if sample.segments:
            for seg in sample.segments:
                if tier is None or seg.tier == tier:
                    segments.append(
                        SegmentData(
                            start=seg.start,
                            end=seg.end,
                            label=seg.label,
                            confidence=seg.confidence,
                            tier=seg.tier,
                        )
                    )
                tiers_found.add(seg.tier)

        return {
            "sample_id": sample_id,
            "dataset": dataset,
            "segments": segments,
            "tiers": sorted(list(tiers_found)),
            "count": len(segments),
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to load segments", e))


@router.post("/api/annotations/cache/clear")
async def clear_annotation_cache():
    """Clear the annotation manifest cache."""
    with _cache_lock:
        _manifest_cache.clear()
    return {"status": "cleared", "message": "Annotation cache cleared"}


# ============================================================================
# Research Utilities
# ============================================================================


@router.post("/api/research/vocabulary-mapping")
async def create_vocabulary_mapping(request: VocabularyMappingRequest):
    """
    Create vocabulary mapping from ELAN files for ML training.

    Returns gloss-to-id and id-to-gloss mappings suitable for
    sequence-to-sequence models.
    """
    from sltk.utils.research import create_vocabulary_mapping

    try:
        mapping = create_vocabulary_mapping(
            elan_files=request.elan_paths,
            min_frequency=request.min_frequency,
            special_tokens=request.special_tokens,
        )
        return mapping
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to create vocabulary mapping", e))


@router.post("/api/research/compare-datasets")
async def compare_datasets_endpoint(request: DatasetComparisonRequest):
    """
    Compare two datasets by vocabulary overlap and statistics.

    Useful for understanding domain differences and transfer learning potential.
    """
    from sltk.utils.research import compare_vocabularies, compute_dataset_statistics

    try:
        stats1 = compute_dataset_statistics(
            request.dataset1_paths,
            name=request.dataset1_name,
        )
        stats2 = compute_dataset_statistics(
            request.dataset2_paths,
            name=request.dataset2_name,
        )

        comparison = compare_vocabularies(stats1, stats2)

        return {
            "dataset1_stats": stats1.to_dict(),
            "dataset2_stats": stats2.to_dict(),
            "comparison": comparison,
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to compare datasets", e))


@router.get("/api/research/pose-coverage")
async def get_pose_coverage(path: str):
    """
    Analyze pose coverage in a pose file.

    Returns statistics about detection rate, gaps, and quality.
    """
    from sltk.utils.research import compute_pose_coverage

    pose_path = validate_path(path, must_exist=True)

    try:
        coverage = compute_pose_coverage(pose_path)
        return coverage
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to compute coverage", e))


@router.post("/api/research/find-gloss-examples")
async def find_gloss_examples_endpoint(request: GlossSearchRequest):
    """
    Find examples of a specific gloss across multiple ELAN files.

    Returns timing information and surrounding context for each occurrence.
    """
    from sltk.utils.research import find_gloss_examples

    try:
        examples = find_gloss_examples(
            gloss=request.gloss,
            elan_files=request.elan_paths,
            max_examples=request.max_examples,
            context_window=request.context_window,
        )

        return {
            "gloss": request.gloss,
            "num_examples": len(examples),
            "examples": examples,
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to find examples", e))


@router.post("/api/research/export-training-data")
async def export_training_data(request: ExportTrainingDataRequest):
    """
    Export segments for ML training.

    Creates a dataset file with all segments, labels, and optional context.
    """
    from sltk.utils.research import export_segments_for_training

    try:
        output = export_segments_for_training(
            elan_files=request.elan_paths,
            output_path=request.output_path,
            format=request.format,
            include_context=request.include_context,
            context_window=request.context_window,
        )

        return {
            "status": "success",
            "output_path": str(output),
            "num_files_processed": len(request.elan_paths),
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Export failed", e))


@router.get("/api/research/dataset-statistics")
async def get_dataset_statistics(elan_paths: list[str] = Query(default=[])):
    """
    Compute comprehensive statistics for a dataset.
    """
    from sltk.utils.research import compute_dataset_statistics

    if not elan_paths:
        raise HTTPException(400, "No ELAN paths provided")

    try:
        stats = compute_dataset_statistics(elan_paths)
        return stats.to_dict()
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to compute statistics", e))


# ============================================================================
# Linguistic Analysis (from ELAN files)
# ============================================================================


@router.post("/api/linguistic/concordance")
async def get_concordance(request: ConcordanceRequest):
    """
    Build a concordance (KWIC - Key Word In Context) view for a target gloss.

    Shows each occurrence with surrounding context, useful for linguistic analysis.
    """
    from sltk.analysis.linguistic import build_concordance
    from sltk.io.elan import read_eaf

    # Load segments from ELAN files
    segments_by_file = {}
    for path in request.elan_paths:
        try:
            segments = read_eaf(path)
            segments_by_file[Path(path).stem] = list(segments)
        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {path}: {e}")
            continue

    if not segments_by_file:
        raise HTTPException(400, "No valid ELAN files found")

    concordance = build_concordance(
        segments_by_file=segments_by_file,
        target_gloss=request.target_gloss,
        context_window=request.context_window,
        case_sensitive=request.case_sensitive,
    )

    return {
        "target_gloss": request.target_gloss,
        "total_occurrences": len(concordance),
        "lines": [
            {
                "file_id": line.file_id,
                "left_context": line.left_context,
                "gloss": line.gloss,
                "right_context": line.right_context,
                "start_time": line.start_time,
                "end_time": line.end_time,
                "duration": line.duration,
                "kwic_string": line.to_kwic_string(),
            }
            for line in concordance
        ],
    }


@router.post("/api/linguistic/cooccurrence")
async def analyze_cooccurrence_endpoint(request: CooccurrenceRequest):
    """
    Analyze co-occurrence patterns (collocations) for a target gloss.

    Returns glosses that frequently appear near the target.
    """
    from sltk.analysis.linguistic import analyze_cooccurrence
    from sltk.io.elan import read_eaf

    segments_by_file = {}
    for path in request.elan_paths:
        try:
            segments = read_eaf(path)
            segments_by_file[Path(path).stem] = list(segments)
        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {path}: {e}")
            continue

    if not segments_by_file:
        raise HTTPException(400, "No valid ELAN files found")

    result = analyze_cooccurrence(
        segments_by_file=segments_by_file,
        target_gloss=request.target_gloss,
        window_size=request.window_size,
    )

    return {
        "target_gloss": result.target_gloss,
        "total_occurrences": result.total_occurrences,
        "window_size": result.window_size,
        "top_collocates": result.top_collocates(30),
        "left_collocates": sorted(result.left_collocates.items(), key=lambda x: -x[1])[:20],
        "right_collocates": sorted(result.right_collocates.items(), key=lambda x: -x[1])[:20],
    }


@router.post("/api/linguistic/ngrams")
async def extract_ngrams_endpoint(request: NGramRequest):
    """
    Extract frequent n-grams (sign sequences) from corpus.

    Useful for identifying common sign combinations and phrases.
    """
    from sltk.analysis.linguistic import extract_ngrams
    from sltk.io.elan import read_eaf

    segments_by_file = {}
    for path in request.elan_paths:
        try:
            segments = read_eaf(path)
            segments_by_file[Path(path).stem] = list(segments)
        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {path}: {e}")
            continue

    if not segments_by_file:
        raise HTTPException(400, "No valid ELAN files found")

    ngrams = extract_ngrams(
        segments_by_file=segments_by_file,
        n=request.n,
        min_frequency=request.min_frequency,
        max_ngrams=request.max_ngrams,
    )

    return {
        "n": request.n,
        "total_ngrams": len(ngrams),
        "ngrams": [
            {
                "sequence": list(ng.glosses),
                "sequence_str": str(ng),
                "count": ng.count,
                "examples": ng.examples[:3],
            }
            for ng in ngrams
        ],
    }


@router.post("/api/linguistic/duration-analysis")
async def analyze_duration_endpoint(request: DurationAnalysisRequest):
    """
    Analyze sign duration patterns across the corpus.

    Returns statistics about how long signs are held, with per-signer breakdowns.
    """
    from sltk.analysis.linguistic import analyze_duration_patterns
    from sltk.io.elan import read_eaf

    segments_by_file = {}
    for path in request.elan_paths:
        try:
            segments = read_eaf(path)
            segments_by_file[Path(path).stem] = list(segments)
        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {path}: {e}")
            continue

    if not segments_by_file:
        raise HTTPException(400, "No valid ELAN files found")

    results = analyze_duration_patterns(
        segments_by_file=segments_by_file,
        gloss=request.gloss,
        min_occurrences=request.min_occurrences,
    )

    # Format results
    formatted = {}
    for gloss, analysis in results.items():
        formatted[gloss] = {
            "count": analysis.count,
            "mean_duration": round(analysis.mean_duration, 3),
            "std_duration": round(analysis.std_duration, 3),
            "min_duration": round(analysis.min_duration, 3),
            "max_duration": round(analysis.max_duration, 3),
            "median_duration": round(analysis.median_duration, 3),
            "quartiles": [round(q, 3) for q in analysis.quartiles],
        }

    return {
        "analyzed_glosses": len(formatted),
        "results": formatted,
    }


@router.post("/api/linguistic/minimal-pairs")
async def find_minimal_pairs_endpoint(request: MinimalPairsRequest):
    """
    Find potential minimal pairs (signs with similar glosses).

    Uses string similarity as a proxy for phonological similarity.
    """
    from sltk.analysis.linguistic import find_minimal_pairs
    from sltk.io.elan import read_eaf

    # Collect vocabulary
    gloss_counter: Counter[str] = Counter()
    for path in request.elan_paths:
        try:
            segments = read_eaf(path)
            for seg in segments:
                if seg.label:
                    gloss_counter[seg.label] += 1
        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {path}: {e}")
            continue

    if not gloss_counter:
        raise HTTPException(400, "No glosses found")

    vocabulary = list(gloss_counter.keys())
    pairs = find_minimal_pairs(
        vocabulary=vocabulary,
        legacy_threshold=request.similarity_threshold,
        max_pairs=request.max_pairs,
    )

    return {
        "vocabulary_size": len(vocabulary),
        "pairs_found": len(pairs),
        "pairs": [
            {
                "gloss1": pair.sign1,
                "gloss2": pair.sign2,
                "similarity": (
                    round(pair.distance, 3) if isinstance(pair.distance, float) else pair.distance
                ),
                "freq1": gloss_counter[pair.sign1],
                "freq2": gloss_counter[pair.sign2],
            }
            for pair in pairs
        ],
    }


@router.post("/api/linguistic/cooccurrence-matrix")
async def compute_cooccurrence_matrix_endpoint(request: CooccurrenceMatrixRequest):
    """
    Compute co-occurrence matrix for top glosses.

    Returns a matrix showing which signs commonly appear together.
    """
    from sltk.analysis.linguistic import compute_cooccurrence_matrix
    from sltk.io.elan import read_eaf

    segments_by_file = {}
    for path in request.elan_paths:
        try:
            segments = read_eaf(path)
            segments_by_file[Path(path).stem] = list(segments)
        except (OSError, ValueError, RuntimeError) as e:
            logger.debug(f"Failed to read ELAN file {path}: {e}")
            continue

    if not segments_by_file:
        raise HTTPException(400, "No valid ELAN files found")

    result = compute_cooccurrence_matrix(
        segments_by_file=segments_by_file,
        top_n_glosses=request.top_n_glosses,
        window_size=request.window_size,
    )

    return result


# ============================================================================
# Cross-Dataset Similarity Search
# ============================================================================

# Global similarity search instance (lazy loaded)
_similarity_search = None
_embedding_store_path = Path("/vol/research/SignFeaturePool/embeddings/sltk_embeddings.h5")


def get_similarity_search():
    """Get or create similarity search instance."""
    global _similarity_search
    if _similarity_search is None:
        try:
            from sltk.embedding import EmbeddingStore, SignRepEmbedder, SimilaritySearch

            embedder = SignRepEmbedder(lazy_load=True)
            store = EmbeddingStore(
                _embedding_store_path if _embedding_store_path.exists() else None
            )
            _similarity_search = SimilaritySearch(embedder, store=store)
        except ImportError as e:
            raise HTTPException(
                503, f"Similarity search not available: {e}. " "Make sure SignRep is installed."
            )
    return _similarity_search


@router.post("/api/similarity/search")
async def similarity_search(
    video_path: str,
    top_k: int = 10,
    target_datasets: list[str] | None = None,
    exclude_source: bool = True,
):
    """
    Find videos similar to a query video across datasets.

    Returns the top K most similar videos from the embedding store,
    optionally filtered by target datasets.
    """
    search = get_similarity_search()

    video_path_validated = validate_path(video_path, must_exist=True)

    try:
        matches = search.find_similar(
            query=video_path_validated,
            top_k=top_k,
            datasets=target_datasets,
            exclude_self=exclude_source,
        )

        return {
            "query": str(video_path_validated),
            "num_results": len(matches),
            "matches": [m.to_dict() for m in matches],
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Similarity search failed", e))


@router.post("/api/similarity/cross-dataset")
async def cross_dataset_similarity(
    video_path: str,
    source_dataset: str,
    target_datasets: list[str] = Query(default=[]),
    top_k: int = 10,
):
    """
    Find similar videos across multiple datasets.

    Given a video from one dataset, find the most similar videos
    in each of the target datasets.
    """
    search = get_similarity_search()

    video_path_validated = validate_path(video_path, must_exist=True)

    if not target_datasets:
        target_datasets = search.store.get_datasets()

    try:
        results = search.cross_dataset_search(
            query=video_path_validated,
            source_dataset=source_dataset,
            target_datasets=target_datasets,
            top_k=top_k,
        )

        return {
            "query": str(video_path_validated),
            "source_dataset": source_dataset,
            "results": {ds: [m.to_dict() for m in matches] for ds, matches in results.items()},
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Cross-dataset search failed", e))


@router.post("/api/similarity/index-video")
@rate_limit("5/minute")
async def index_video_for_similarity(
    request: Request,
    video_path: str,
    video_id: str | None = None,
    dataset: str = "",
    gloss: str | None = None,
):
    """
    Index a single video for similarity search.

    Computes the video embedding and adds it to the store.

    Rate limited to 5 requests per minute (embedding computation is expensive).
    """
    search = get_similarity_search()

    video_path_validated = validate_path(video_path, must_exist=True)

    try:
        vid_id = search.index_video(
            video_path=video_path_validated,
            video_id=video_id,
            dataset=dataset,
            gloss=gloss,
        )

        return {
            "status": "indexed",
            "video_id": vid_id,
            "video_path": str(video_path_validated),
            "dataset": dataset,
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Indexing failed", e))


@router.post("/api/similarity/index-dataset")
@rate_limit("5/minute")
async def index_dataset_for_similarity(
    request: Request,
    dataset_name: str,
    dataset_root: str,
    max_samples: int | None = None,
    background_tasks: BackgroundTasks = None,  # type: ignore[assignment]
):
    """
    Index an entire dataset for similarity search (background task).

    This can take a while for large datasets, so it runs in the background.

    Rate limited to 5 requests per minute (very expensive operation).
    """
    import uuid

    from sltk.api.routers.extraction import add_job_log
    from sltk.data.datasets import get_dataset

    dataset_root_validated = validate_path(dataset_root, must_exist=True)

    job_id = str(uuid.uuid4())[:8]

    def index_task():
        try:
            search = get_similarity_search()
            dataset = get_dataset(dataset_name, root=dataset_root_validated)
            count = search.index_dataset(dataset, max_samples=max_samples)
            search.save()
            add_job_log(job_id, f"Indexed {count} videos from {dataset_name}")
        except Exception as e:
            add_job_log(job_id, f"Indexing failed: {e}")

    if background_tasks:
        background_tasks.add_task(index_task)

    return {
        "status": "started",
        "job_id": job_id,
        "dataset": dataset_name,
        "message": "Indexing started in background. Check /api/extraction/logs for progress.",
    }


@router.get("/api/similarity/stats")
async def get_similarity_stats():
    """Get statistics about the similarity search index."""
    search = get_similarity_search()
    return search.get_statistics()


@router.get("/api/similarity/datasets")
async def get_indexed_datasets():
    """Get list of datasets in the similarity index."""
    search = get_similarity_search()
    datasets = search.store.get_datasets()
    stats = search.store.get_statistics()

    return {
        "datasets": datasets,
        "videos_per_dataset": stats.get("datasets", {}),
        "total_videos": stats.get("total_videos", 0),
    }


@router.post("/api/similarity/save")
async def save_similarity_index(path: str | None = None):
    """Save the similarity index to disk."""
    search = get_similarity_search()

    save_path = validate_path(path, must_exist=False) if path else _embedding_store_path

    try:
        search.save(save_path)
        return {
            "status": "saved",
            "path": str(save_path),
            "num_embeddings": len(search.store),
        }
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to save index", e))


# ============================================================================
# Cross-Dataset Gloss Comparison
# ============================================================================

# Cache for vocabulary comparisons (expensive to compute)
MAX_VOCAB_CACHE_SIZE = 20
_vocab_comparison_cache: dict[str, tuple[dict, float]] = {}
_vocab_cache_lock = threading.Lock()
VOCAB_COMPARISON_CACHE_TTL = 600  # 10 minutes


def _get_available_datasets() -> list[str]:
    """Get list of datasets available for cross-dataset comparison."""
    from sltk.data.datasets import DATASETS

    return list(DATASETS.keys())


def _load_dataset_samples(
    dataset_name: str, max_samples: int = 1000
) -> tuple[dict[str, list[Any]], dict[str, str], dict[str, str | None]]:
    """
    Load samples from a dataset for cross-dataset analysis.

    Returns:
        Tuple of (segments_by_sample, signer_ids, video_paths)
    """
    from sltk.api.dependencies import get_cached_dataset

    dataset = get_cached_dataset(dataset_name, split="all")

    segments_by_sample: dict[str, list[Any]] = {}
    signer_ids: dict[str, str] = {}
    video_paths: dict[str, str | None] = {}

    sample_ids = dataset._index[:max_samples] if hasattr(dataset, "_index") else []

    for sample_id in sample_ids:
        try:
            sample = dataset._load_sample(sample_id)
            if sample.segments:
                segments_by_sample[sample_id] = sample.segments
                if sample.signer_id:
                    signer_ids[sample_id] = sample.signer_id
                if hasattr(sample, "video_path") and sample.video_path:
                    video_paths[sample_id] = str(sample.video_path)
                else:
                    video_paths[sample_id] = None
        except Exception as e:
            logger.debug(f"Failed to load sample {sample_id} from {dataset_name}: {e}")
            continue

    return segments_by_sample, signer_ids, video_paths


@router.get("/api/analysis/cross-dataset/datasets")
async def list_available_datasets():
    """
    List datasets available for cross-dataset comparison.

    Returns information about each dataset including vocabulary size,
    sample count, and available tiers.
    """
    from sltk.api.dependencies import get_cached_dataset
    from sltk.data.datasets import DATASETS

    datasets = []

    for name in DATASETS.keys():
        try:
            dataset = get_cached_dataset(name, split="all")

            # Get basic info
            total_samples = len(dataset._index) if hasattr(dataset, "_index") else 0

            # Try to get vocabulary size from a few samples
            vocab = set()
            signers = set()
            tiers = set()

            sample_limit = min(100, total_samples)
            sample_ids = dataset._index[:sample_limit] if hasattr(dataset, "_index") else []

            for sample_id in sample_ids:
                try:
                    sample = dataset._load_sample(sample_id)
                    if sample.segments:
                        for seg in sample.segments:
                            if seg.label:
                                vocab.add(seg.label)
                            if seg.tier:
                                tiers.add(seg.tier)
                    if sample.signer_id:
                        signers.add(sample.signer_id)
                except Exception:
                    continue

            datasets.append(
                CrossDatasetInfo(
                    name=name,
                    total_samples=total_samples,
                    vocabulary_size=len(vocab),
                    unique_signers=len(signers),
                    language=getattr(dataset, "language", "unknown"),
                    tiers=sorted(list(tiers)),
                    has_video=True,  # Assume videos are available
                    has_poses=True,  # Assume poses can be extracted
                )
            )
        except Exception as e:
            logger.warning(f"Failed to get info for dataset {name}: {e}")
            # Still include dataset with minimal info
            datasets.append(
                CrossDatasetInfo(
                    name=name,
                    total_samples=0,
                    vocabulary_size=0,
                )
            )

    return datasets


@router.get("/api/analysis/cross-dataset/search/{gloss}")
@rate_limit("60/minute")
async def search_gloss_across_datasets(
    request: Request,
    gloss: str,
    datasets: list[str] | None = Query(None),
    case_sensitive: bool = False,
    context_window: int = Query(default=3, ge=0, le=10),
    tier: str = "gloss",
    max_results_per_dataset: int = Query(default=100, ge=1, le=500),
):
    """
    Find a gloss across multiple datasets.

    Searches for occurrences of a gloss in the specified datasets
    (or all available datasets if none specified) and returns
    matches with their context.

    Rate limited to 60 requests per minute.
    """
    from sltk.api.models import GlossMatchResponse

    # Determine which datasets to search
    target_datasets = datasets if datasets else _get_available_datasets()

    if not target_datasets:
        raise HTTPException(400, "No datasets available for search")

    matches_by_dataset: dict[str, list[GlossMatchResponse]] = {}
    dataset_frequencies: dict[str, int] = {}
    total_matches = 0

    target = gloss if case_sensitive else gloss.lower()

    def search_dataset(ds_name: str) -> tuple[str, list[GlossMatchResponse], int]:
        """Search a single dataset."""
        try:
            segments_by_sample, signer_ids, video_paths = _load_dataset_samples(
                ds_name, max_samples=1000
            )
        except HTTPException:
            return ds_name, [], 0

        matches: list[GlossMatchResponse] = []

        for sample_id, segments in segments_by_sample.items():
            # Filter by tier and extract labels
            filtered_segments = []
            for i, seg in enumerate(segments):
                seg_tier = getattr(seg, "tier", "gloss")
                if seg_tier == tier:
                    label = getattr(seg, "label", None)
                    if label:
                        filtered_segments.append((i, seg, label))

            # Find matches
            for idx, (seg_idx, seg, label) in enumerate(filtered_segments):
                label_cmp = label if case_sensitive else label.lower()
                if label_cmp == target:
                    # Get context
                    context_before = [
                        filtered_segments[j][2] for j in range(max(0, idx - context_window), idx)
                    ]
                    context_after = [
                        filtered_segments[j][2]
                        for j in range(
                            idx + 1, min(len(filtered_segments), idx + context_window + 1)
                        )
                    ]

                    matches.append(
                        GlossMatchResponse(
                            dataset=ds_name,
                            sample_id=sample_id,
                            segment_idx=seg_idx,
                            start=getattr(seg, "start", 0),
                            end=getattr(seg, "end", 0),
                            duration=getattr(seg, "end", 0) - getattr(seg, "start", 0),
                            context_before=context_before,
                            context_after=context_after,
                            signer_id=signer_ids.get(sample_id),
                            video_path=video_paths.get(sample_id),
                        )
                    )

                    if len(matches) >= max_results_per_dataset:
                        break

            if len(matches) >= max_results_per_dataset:
                break

        return ds_name, matches, len(matches)

    # Search each dataset
    for ds_name in target_datasets:
        ds_name, ds_matches, count = await asyncio.to_thread(search_dataset, ds_name)
        if ds_matches:
            matches_by_dataset[ds_name] = ds_matches
            dataset_frequencies[ds_name] = count
            total_matches += count

    return CrossDatasetSearchResponse(
        gloss=gloss,
        total_matches=total_matches,
        matches_by_dataset=matches_by_dataset,
        dataset_frequencies=dataset_frequencies,
    )


@router.post("/api/analysis/cross-dataset/vocabulary-comparison")
@rate_limit("10/minute")
async def compare_vocabularies(
    request: Request,
    body: VocabularyComparisonRequest,
):
    """
    Compare vocabularies across datasets.

    Computes shared vocabulary, unique terms per dataset, overlap matrix,
    and Jaccard similarity between dataset pairs.

    Rate limited to 10 requests per minute (expensive operation).
    Results are cached for 10 minutes.
    """

    # Check cache
    cache_key = f"{','.join(sorted(body.datasets))}:{body.tier}:{body.min_frequency}"
    current_time = time.time()

    with _vocab_cache_lock:
        if cache_key in _vocab_comparison_cache:
            cached, cached_at = _vocab_comparison_cache[cache_key]
            if current_time - cached_at < VOCAB_COMPARISON_CACHE_TTL:
                return VocabularyComparisonResponse(**cached)

    # Validate datasets exist
    available = set(_get_available_datasets())
    for ds in body.datasets:
        if ds not in available:
            raise HTTPException(404, f"Dataset not found: {ds}")

    def compute_comparison():
        vocabularies: dict[str, Counter] = {}

        for ds_name in body.datasets:
            try:
                segments_by_sample, _, _ = _load_dataset_samples(ds_name, max_samples=1000)
            except Exception:
                vocabularies[ds_name] = Counter()
                continue

            vocab_counter = Counter()
            for segments in segments_by_sample.values():
                for seg in segments:
                    seg_tier = getattr(seg, "tier", "gloss")
                    if seg_tier == body.tier:
                        label = getattr(seg, "label", None)
                        if label:
                            vocab_counter[label] += 1

            # Apply min frequency filter
            vocabularies[ds_name] = Counter(
                {k: v for k, v in vocab_counter.items() if v >= body.min_frequency}
            )

        # Compute shared vocabulary (present in ALL datasets)
        all_vocabs = [set(v.keys()) for v in vocabularies.values()]
        if all_vocabs:
            shared = set.intersection(*all_vocabs)
        else:
            shared = set()

        # Compute unique per dataset
        unique_by_dataset = {}
        for ds_name, vocab in vocabularies.items():
            other_vocabs = (
                set.union(*[set(v.keys()) for n, v in vocabularies.items() if n != ds_name])
                if len(vocabularies) > 1
                else set()
            )
            unique_by_dataset[ds_name] = sorted(set(vocab.keys()) - other_vocabs)[:100]

        # Compute overlap matrix and Jaccard similarity
        overlap_matrix: dict[str, dict[str, int]] = {}
        jaccard_matrix: dict[str, dict[str, float]] = {}

        for ds1 in body.datasets:
            overlap_matrix[ds1] = {}
            jaccard_matrix[ds1] = {}
            vocab1 = set(vocabularies[ds1].keys())

            for ds2 in body.datasets:
                vocab2 = set(vocabularies[ds2].keys())
                intersection = vocab1 & vocab2
                union = vocab1 | vocab2

                overlap_matrix[ds1][ds2] = len(intersection)
                jaccard_matrix[ds1][ds2] = (
                    round(len(intersection) / len(union), 4) if union else 0.0
                )

        return {
            "datasets": body.datasets,
            "shared_vocabulary": sorted(list(shared))[:500],  # Limit to 500
            "unique_by_dataset": unique_by_dataset,
            "vocabulary_sizes": {ds: len(v) for ds, v in vocabularies.items()},
            "overlap_matrix": overlap_matrix,
            "jaccard_similarity": jaccard_matrix,
        }

    result = await asyncio.to_thread(compute_comparison)

    # Cache result (with size limit)
    with _vocab_cache_lock:
        if len(_vocab_comparison_cache) >= MAX_VOCAB_CACHE_SIZE:
            oldest_key = min(_vocab_comparison_cache, key=lambda k: _vocab_comparison_cache[k][1])
            del _vocab_comparison_cache[oldest_key]
        _vocab_comparison_cache[cache_key] = (result, current_time)

    return VocabularyComparisonResponse(**result)


@router.post("/api/analysis/cross-dataset/parallel-examples")
@rate_limit("30/minute")
async def find_parallel_examples(
    request: Request,
    body: ParallelExamplesRequest,
):
    """
    Find parallel examples of glosses across datasets.

    For each specified gloss, finds example occurrences in each dataset.
    Useful for comparing how the same sign is used across different corpora.

    Rate limited to 30 requests per minute.
    """
    from sltk.api.models import ParallelExampleMatch

    # Validate datasets
    available = set(_get_available_datasets())
    for ds in body.datasets:
        if ds not in available:
            raise HTTPException(404, f"Dataset not found: {ds}")

    def find_examples():
        examples_by_dataset: dict[str, dict[str, list[ParallelExampleMatch]]] = {}
        coverage: dict[str, list[str]] = {ds: [] for ds in body.datasets}

        for ds_name in body.datasets:
            examples_by_dataset[ds_name] = {g: [] for g in body.glosses}

            try:
                segments_by_sample, signer_ids, video_paths = _load_dataset_samples(
                    ds_name, max_samples=1000
                )
            except Exception:
                continue

            # Search for each gloss
            glosses_lower = {g.lower(): g for g in body.glosses}

            for sample_id, segments in segments_by_sample.items():
                # Filter by tier
                filtered_segments = []
                for seg in segments:
                    seg_tier = getattr(seg, "tier", "gloss")
                    if seg_tier == body.tier:
                        label = getattr(seg, "label", None)
                        if label:
                            filtered_segments.append((seg, label))

                for idx, (seg, label) in enumerate(filtered_segments):
                    label_lower = label.lower()
                    if label_lower in glosses_lower:
                        original_gloss = glosses_lower[label_lower]

                        if (
                            len(examples_by_dataset[ds_name][original_gloss])
                            >= body.max_examples_per_dataset
                        ):
                            continue

                        # Get context
                        context_before = [
                            filtered_segments[j][1] for j in range(max(0, idx - 3), idx)
                        ]
                        context_after = [
                            filtered_segments[j][1]
                            for j in range(idx + 1, min(len(filtered_segments), idx + 4))
                        ]

                        examples_by_dataset[ds_name][original_gloss].append(
                            ParallelExampleMatch(
                                sample_id=sample_id,
                                start=getattr(seg, "start", 0),
                                end=getattr(seg, "end", 0),
                                duration=getattr(seg, "end", 0) - getattr(seg, "start", 0),
                                context_before=context_before,
                                context_after=context_after,
                                signer_id=signer_ids.get(sample_id),
                                video_path=video_paths.get(sample_id),
                            )
                        )

                        if original_gloss not in coverage[ds_name]:
                            coverage[ds_name].append(original_gloss)

        return examples_by_dataset, coverage

    examples_by_dataset, coverage = await asyncio.to_thread(find_examples)

    return ParallelExamplesResponse(
        glosses=body.glosses,
        examples_by_dataset=examples_by_dataset,
        coverage=coverage,
    )


@router.get("/api/analysis/cross-dataset/duration/{gloss}")
@rate_limit("60/minute")
async def compare_durations(
    request: Request,
    gloss: str,
    datasets: list[str] | None = Query(None),
    tier: str = "gloss",
):
    """
    Compare gloss duration across datasets.

    Returns duration statistics (mean, std, min, max, median, quartiles)
    for a gloss in each specified dataset.

    Rate limited to 60 requests per minute.
    """
    from sltk.api.models import DurationStatistics

    target_datasets = datasets if datasets else _get_available_datasets()

    if not target_datasets:
        raise HTTPException(400, "No datasets available")

    def compute_durations():
        statistics_by_dataset: dict[str, DurationStatistics] = {}
        all_durations: list[float] = []
        target = gloss.lower()

        for ds_name in target_datasets:
            try:
                segments_by_sample, _, _ = _load_dataset_samples(ds_name, max_samples=1000)
            except Exception:
                continue

            durations: list[float] = []

            for segments in segments_by_sample.values():
                for seg in segments:
                    seg_tier = getattr(seg, "tier", "gloss")
                    if seg_tier != tier:
                        continue

                    label = getattr(seg, "label", None)
                    if not label or label.lower() != target:
                        continue

                    start = getattr(seg, "start", 0)
                    end = getattr(seg, "end", 0)
                    duration = end - start
                    if duration > 0:
                        durations.append(duration)

            if durations:
                all_durations.extend(durations)
                arr = np.array(durations)
                statistics_by_dataset[ds_name] = DurationStatistics(
                    count=len(durations),
                    mean=round(float(arr.mean()), 4),
                    std=round(float(arr.std()), 4),
                    min=round(float(arr.min()), 4),
                    max=round(float(arr.max()), 4),
                    median=round(float(np.median(arr)), 4),
                    quartiles=(
                        round(float(np.percentile(arr, 25)), 4),
                        round(float(np.percentile(arr, 50)), 4),
                        round(float(np.percentile(arr, 75)), 4),
                    ),
                )

        # Compute cross-dataset stats
        cross_stats = None
        if all_durations:
            arr = np.array(all_durations)
            cross_stats = DurationStatistics(
                count=len(all_durations),
                mean=round(float(arr.mean()), 4),
                std=round(float(arr.std()), 4),
                min=round(float(arr.min()), 4),
                max=round(float(arr.max()), 4),
                median=round(float(np.median(arr)), 4),
                quartiles=(
                    round(float(np.percentile(arr, 25)), 4),
                    round(float(np.percentile(arr, 50)), 4),
                    round(float(np.percentile(arr, 75)), 4),
                ),
            )

        return statistics_by_dataset, cross_stats

    statistics_by_dataset, cross_stats = await asyncio.to_thread(compute_durations)

    return DurationComparisonResponse(
        gloss=gloss,
        statistics_by_dataset=statistics_by_dataset,
        cross_dataset_stats=cross_stats,
    )


@router.get("/api/analysis/cross-dataset/contexts/{gloss}")
@rate_limit("30/minute")
async def analyze_contexts(
    request: Request,
    gloss: str,
    datasets: list[str] | None = Query(None),
    context_size: int = Query(default=1, ge=1, le=5),
    tier: str = "gloss",
    min_frequency: int = Query(default=2, ge=1),
):
    """
    Analyze gloss context patterns across datasets.

    Returns the most common left contexts, right contexts, and bigrams
    containing the target gloss across the specified datasets.

    Rate limited to 30 requests per minute.
    """

    target_datasets = datasets if datasets else _get_available_datasets()

    if not target_datasets:
        raise HTTPException(400, "No datasets available")

    def analyze():
        left_contexts: Counter = Counter()
        right_contexts: Counter = Counter()
        bigrams: Counter = Counter()
        left_context_datasets: dict[str, set[str]] = {}
        right_context_datasets: dict[str, set[str]] = {}
        bigram_datasets: dict[str, set[str]] = {}

        target = gloss.lower()

        for ds_name in target_datasets:
            try:
                segments_by_sample, _, _ = _load_dataset_samples(ds_name, max_samples=1000)
            except Exception:
                continue

            for segments in segments_by_sample.values():
                # Filter by tier
                filtered_labels = []
                for seg in segments:
                    seg_tier = getattr(seg, "tier", "gloss")
                    if seg_tier == tier:
                        label = getattr(seg, "label", None)
                        if label:
                            filtered_labels.append(label)

                for idx, label in enumerate(filtered_labels):
                    if label.lower() != target:
                        continue

                    # Left context
                    if idx > 0:
                        left_ctx = " ".join(filtered_labels[max(0, idx - context_size) : idx])
                        left_contexts[left_ctx] += 1
                        if left_ctx not in left_context_datasets:
                            left_context_datasets[left_ctx] = set()
                        left_context_datasets[left_ctx].add(ds_name)

                    # Right context
                    if idx < len(filtered_labels) - 1:
                        right_ctx = " ".join(filtered_labels[idx + 1 : idx + 1 + context_size])
                        right_contexts[right_ctx] += 1
                        if right_ctx not in right_context_datasets:
                            right_context_datasets[right_ctx] = set()
                        right_context_datasets[right_ctx].add(ds_name)

                    # Bigrams
                    if idx > 0:
                        bigram = f"{filtered_labels[idx - 1]} {label}"
                        bigrams[bigram] += 1
                        if bigram not in bigram_datasets:
                            bigram_datasets[bigram] = set()
                        bigram_datasets[bigram].add(ds_name)

                    if idx < len(filtered_labels) - 1:
                        bigram = f"{label} {filtered_labels[idx + 1]}"
                        bigrams[bigram] += 1
                        if bigram not in bigram_datasets:
                            bigram_datasets[bigram] = set()
                        bigram_datasets[bigram].add(ds_name)

        # Filter by min frequency and create response
        left_patterns = [
            ContextPattern(
                pattern=p,
                frequency=c,
                datasets=sorted(list(left_context_datasets.get(p, set()))),
            )
            for p, c in left_contexts.most_common(50)
            if c >= min_frequency
        ]

        right_patterns = [
            ContextPattern(
                pattern=p,
                frequency=c,
                datasets=sorted(list(right_context_datasets.get(p, set()))),
            )
            for p, c in right_contexts.most_common(50)
            if c >= min_frequency
        ]

        bigram_patterns = [
            ContextPattern(
                pattern=p,
                frequency=c,
                datasets=sorted(list(bigram_datasets.get(p, set()))),
            )
            for p, c in bigrams.most_common(50)
            if c >= min_frequency
        ]

        return left_patterns, right_patterns, bigram_patterns

    left_patterns, right_patterns, bigram_patterns = await asyncio.to_thread(analyze)

    return ContextPatternsResponse(
        gloss=gloss,
        left_contexts=left_patterns,
        right_contexts=right_patterns,
        bigrams_with_target=bigram_patterns,
        datasets_analyzed=target_datasets,
    )


@router.post("/api/analysis/cross-dataset/batch-search")
@rate_limit("10/minute")
async def batch_search_glosses(
    request: Request,
    body: BatchSearchRequest,
):
    """
    Search for multiple glosses across datasets.

    Returns frequency information for each gloss across all specified datasets.
    More efficient than making individual search requests for each gloss.

    Rate limited to 10 requests per minute (expensive operation).
    """

    target_datasets = body.datasets if body.datasets else _get_available_datasets()

    if not target_datasets:
        raise HTTPException(400, "No datasets available")

    def batch_search():
        # Build lookup for case-insensitive matching
        glosses_lookup = {g.lower(): g for g in body.glosses}

        results: dict[str, dict[str, int]] = {g: {} for g in body.glosses}
        found_glosses: set[str] = set()

        for ds_name in target_datasets:
            try:
                segments_by_sample, _, _ = _load_dataset_samples(ds_name, max_samples=1000)
            except Exception:
                continue

            ds_counts: Counter = Counter()

            for segments in segments_by_sample.values():
                for seg in segments:
                    seg_tier = getattr(seg, "tier", "gloss")
                    if seg_tier != body.tier:
                        continue

                    label = getattr(seg, "label", None)
                    if not label:
                        continue

                    label_cmp = label if body.case_sensitive else label.lower()
                    if label_cmp in glosses_lookup:
                        original_gloss = label if body.case_sensitive else glosses_lookup[label_cmp]
                        ds_counts[original_gloss] += 1
                        found_glosses.add(original_gloss)

            # Update results
            for gloss, count in ds_counts.items():
                if count > 0:
                    results[gloss][ds_name] = count

        return results, found_glosses

    results, found_glosses = await asyncio.to_thread(batch_search)

    # Build response
    result_items = []
    for gloss in body.glosses:
        freqs = results.get(gloss, {})
        result_items.append(
            BatchSearchResultItem(
                gloss=gloss,
                total_matches=sum(freqs.values()),
                dataset_frequencies=freqs,
            )
        )

    not_found = [g for g in body.glosses if g not in found_glosses]

    return BatchSearchResponse(
        results=result_items,
        total_glosses_searched=len(body.glosses),
        glosses_found=len(found_glosses),
        glosses_not_found=not_found,
    )
