"""
Dataset endpoints for SLTK API.

Handles dataset listing, sample browsing, vocabulary analysis, and user connections.
"""

from __future__ import annotations

import asyncio
import json
import logging
import time
from collections import Counter
from pathlib import Path

from fastapi import APIRouter, HTTPException, Query, Request

from sltk.api.dependencies import (
    MAX_VOCABULARY_CACHE_SIZE,
    _evict_oldest_cache_entry,
    check_video_path,
    get_cached_dataset,
    get_segments_cache,
    get_vocabulary_cache,
    load_dataset_segments,
    rate_limit,
    safe_error_detail,
    scan_videos_bounded,
)
from sltk.api.models import (
    ConnectDatasetRequest,
    ConnectDatasetResponse,
    DatasetConnection,
    DatasetInfo,
    GlossExample,
    SampleInfo,
    SegmentData,
)

router = APIRouter(prefix="/api/datasets", tags=["datasets"])
logger = logging.getLogger(__name__)

# ============================================================================
# User Connections Storage
# ============================================================================

# User settings directory
SETTINGS_DIR = Path.home() / ".sltk"
CONNECTIONS_FILE = SETTINGS_DIR / "connections.json"

# Connections cache (mtime-based to avoid disk reads on every request)
_connections_cache: dict[str, DatasetConnection] | None = None
_connections_mtime: float = 0.0


def _load_connections() -> dict[str, DatasetConnection]:
    """
    Load user dataset connections from disk with mtime-based caching.

    Only re-reads the file if it has been modified since last load.
    This saves 5-20ms per request that needs connection info.
    """
    global _connections_cache, _connections_mtime

    try:
        if not CONNECTIONS_FILE.exists():
            _connections_cache = {}
            _connections_mtime = 0.0
            return {}

        current_mtime = CONNECTIONS_FILE.stat().st_mtime

        # Return cached data if file hasn't been modified
        if _connections_cache is not None and current_mtime <= _connections_mtime:
            return _connections_cache

        # File was modified or cache is empty - reload
        with open(CONNECTIONS_FILE) as f:
            data = json.load(f)
            _connections_cache = {k: DatasetConnection(**v) for k, v in data.items()}
            _connections_mtime = current_mtime
            return _connections_cache

    except (OSError, json.JSONDecodeError, TypeError, ValueError) as e:
        logger.debug(f"Failed to load connections file: {e}")
        return {}


def _invalidate_connections_cache() -> None:
    """Invalidate the connections cache (call after modifications)."""
    global _connections_cache
    _connections_cache = None


def _save_connections(connections: dict[str, DatasetConnection]) -> None:
    """Save user dataset connections to disk and invalidate cache."""
    SETTINGS_DIR.mkdir(parents=True, exist_ok=True)
    with open(CONNECTIONS_FILE, "w") as f:
        json.dump({k: v.model_dump() for k, v in connections.items()}, f, indent=2)
    # Invalidate cache so next load reads fresh data
    _invalidate_connections_cache()


def _scan_directory_counts(
    video_root: str,
    annotation_root: str | None,
    feature_root: str | None,
) -> tuple[int, int, int]:
    """Scan directories and return file counts."""
    video_exts = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
    annotation_exts = {".eaf", ".json", ".csv", ".txt"}
    feature_exts = {".h5", ".hdf5", ".pt", ".npy", ".npz"}

    video_count = 0
    annotation_count = 0
    feature_count = 0

    # Count videos
    video_path = Path(video_root)
    if video_path.exists():
        for ext in video_exts:
            video_count += len(list(video_path.rglob(f"*{ext}")))

    # Count annotations
    if annotation_root:
        ann_path = Path(annotation_root)
        if ann_path.exists():
            for ext in annotation_exts:
                annotation_count += len(list(ann_path.rglob(f"*{ext}")))

    # Count features
    if feature_root:
        feat_path = Path(feature_root)
        if feat_path.exists():
            for ext in feature_exts:
                feature_count += len(list(feat_path.rglob(f"*{ext}")))

    return video_count, annotation_count, feature_count


# Server log function (imported from settings router when available)
def _add_server_log(level: str, message: str, source: str = "datasets"):
    """Add a log entry. Will be replaced by actual implementation."""
    logger.log(
        (
            logging.INFO
            if level == "INFO"
            else logging.WARNING if level == "WARNING" else logging.ERROR
        ),
        f"[{source}] {message}",
    )


# ============================================================================
# User-Connected Datasets
# ============================================================================


@router.post("/connect", response_model=ConnectDatasetResponse)
async def connect_dataset(request: ConnectDatasetRequest):
    """
    Connect a new dataset by specifying its root directories.

    Scans the directories to discover available files and saves
    the connection for future use.
    """
    # Validate video root exists
    video_path = Path(request.video_root)
    if not video_path.exists():
        raise HTTPException(404, f"Video root not found: {request.video_root}")
    if not video_path.is_dir():
        raise HTTPException(400, f"Video root is not a directory: {request.video_root}")

    # Scan directories
    video_count, annotation_count, feature_count = _scan_directory_counts(
        request.video_root,
        request.annotation_root,
        request.feature_root,
    )

    if video_count == 0:
        raise HTTPException(400, f"No video files found in: {request.video_root}")

    # Save connection
    connections = _load_connections()
    connections[request.name] = DatasetConnection(
        name=request.name,
        language=request.language,
        video_root=request.video_root,
        annotation_root=request.annotation_root,
        feature_root=request.feature_root,
        video_count=video_count,
        annotation_count=annotation_count,
        feature_count=feature_count,
    )
    _save_connections(connections)

    _add_server_log(
        "INFO",
        f"Connected dataset '{request.name}' with {video_count} videos",
        "datasets",
    )

    return ConnectDatasetResponse(
        name=request.name,
        status="connected",
        video_count=video_count,
        annotation_count=annotation_count,
        feature_count=feature_count,
    )


@router.get("/connections")
async def list_dataset_connections() -> dict[str, DatasetConnection]:
    """List all user-connected datasets."""
    return _load_connections()


def _check_builtin_dataset(name: str, dataset_info: dict) -> tuple[str, dict]:
    """Check a single built-in dataset accessibility (blocking, run in thread)."""
    try:
        root_path = dataset_info.get("root")

        if root_path is None:
            return name, {
                "status": "unconfigured",
                "accessible": False,
                "path": None,
                "error": "No root path configured",
            }

        path = Path(root_path)
        accessible = path.exists() and path.is_dir()

        # Check for content (quick scan)
        sample_count = 0
        if accessible:
            try:
                # Count a few files to verify content
                files = list(path.glob("*"))[:10]
                sample_count = len(files)
            except (OSError, PermissionError):
                pass

        return name, {
            "status": (
                "connected"
                if accessible and sample_count > 0
                else "empty" if accessible else "inaccessible"
            ),
            "accessible": accessible,
            "path": str(root_path),
            "has_content": sample_count > 0,
        }
    except Exception as e:
        return name, {
            "status": "error",
            "accessible": False,
            "error": str(e),
        }


def _check_user_connection(name: str, conn: DatasetConnection) -> tuple[str, dict]:
    """Check a single user connection accessibility (blocking, run in thread)."""
    try:
        video_path = Path(conn.video_root)
        accessible = video_path.exists() and video_path.is_dir()

        return name, {
            "status": "connected" if accessible else "inaccessible",
            "accessible": accessible,
            "video_root": conn.video_root,
            "annotation_root": conn.annotation_root,
            "video_count": conn.video_count,
        }
    except Exception as e:
        return name, {
            "status": "error",
            "accessible": False,
            "error": str(e),
        }


@router.get("/validate")
async def validate_dataset_connections():
    """
    Validate all dataset connections and check accessibility.

    Returns status for each built-in dataset and user connection,
    including whether the paths are accessible.

    Uses asyncio.to_thread to avoid blocking the event loop on filesystem I/O.
    """
    from sltk.config import DATASET_PATHS
    from sltk.data.datasets import DATASETS

    results = {
        "builtin": {},
        "user": {},
        "timestamp": time.time(),
    }

    # Check built-in datasets in parallel using thread pool
    builtin_tasks = [
        asyncio.to_thread(_check_builtin_dataset, name, DATASET_PATHS.get(name, {}))
        for name in DATASETS.keys()
    ]

    # Check user connections
    connections = _load_connections()
    user_tasks = [
        asyncio.to_thread(_check_user_connection, name, conn) for name, conn in connections.items()
    ]

    # Run all checks concurrently
    all_results = await asyncio.gather(*builtin_tasks, *user_tasks, return_exceptions=True)

    # Separate results
    num_builtin = len(builtin_tasks)
    builtin_results = all_results[:num_builtin]
    user_results = all_results[num_builtin:]

    # Process builtin results
    for result in builtin_results:
        if isinstance(result, Exception):
            continue
        name, status = result
        results["builtin"][name] = status

    # Process user results
    for result in user_results:
        if isinstance(result, Exception):
            continue
        name, status = result
        results["user"][name] = status

    return results


@router.delete("/connections/{name}")
async def delete_dataset_connection(name: str) -> dict[str, str]:
    """Remove a dataset connection."""
    connections = _load_connections()
    if name not in connections:
        raise HTTPException(404, f"Connection not found: {name}")

    del connections[name]
    _save_connections(connections)

    _add_server_log("INFO", f"Removed dataset connection: {name}", "datasets")
    return {"status": "deleted", "name": name}


@router.delete("/connections")
async def delete_all_connections() -> dict[str, str | int]:
    """Remove all dataset connections."""
    connections = _load_connections()
    count = len(connections)

    if count > 0:
        _save_connections({})
        _add_server_log("INFO", f"Removed all {count} dataset connections", "datasets")

    return {"status": "cleared", "deleted_count": count}


@router.get("/connections/{name}/videos")
async def get_connected_dataset_videos(
    name: str,
    limit: int = Query(1000, ge=1, le=10000),
):
    """
    Get video files from a user-connected dataset.

    Returns video paths for extraction.
    Uses bounded directory scanning with max depth of 3 levels.
    """
    connections = _load_connections()
    if name not in connections:
        raise HTTPException(404, f"Connection not found: {name}")

    conn = connections[name]
    video_path = Path(conn.video_root)
    video_exts = {".mp4", ".mov", ".avi", ".mkv", ".webm"}

    # Use bounded scanning with asyncio.to_thread to avoid blocking
    videos = await asyncio.to_thread(
        scan_videos_bounded,
        video_path,
        video_exts,
        max_depth=3,
        limit=limit,
    )

    return {
        "dataset": name,
        "videos": videos,
        "total_count": conn.video_count,
    }


# ============================================================================
# Dataset Listing and Browsing
# ============================================================================


@router.get("/list", response_model=list[DatasetInfo])
async def list_datasets():
    """
    List available datasets (both built-in and user-connected).

    Uses lazy loading - only returns sample counts if dataset is already cached.
    This avoids initializing all datasets just for a listing (5-20s → instant).
    """
    from sltk.api.dependencies import DATASET_CACHE_TTL, _dataset_cache, _dataset_cache_lock
    from sltk.data.datasets import DATASETS

    result = []
    current_time = time.time()

    # Built-in datasets
    for name, cls in DATASETS.items():
        # Handle both class attributes and properties for DEFAULT_VIDEO_ROOT
        # Some datasets define it as a property that requires instantiation
        root_val = getattr(cls, "DEFAULT_VIDEO_ROOT", None)
        if root_val is not None and not isinstance(root_val, (str, Path)):
            # It's a property object, try to get value from config
            try:
                from sltk.config import DATASET_PATHS

                dataset_info = DATASET_PATHS.get(name, {})
                root_val = dataset_info.get("root", "")
            except (ImportError, KeyError, AttributeError) as e:
                logger.debug(f"Could not get config for dataset {name}: {e}")
                root_val = ""

        # LAZY LOADING: Only get sample count if dataset is already cached
        # Don't initialize datasets just for listing (avoids 5-20s delay)
        total_samples = 0
        cache_key = (name, "all")
        with _dataset_cache_lock:
            if cache_key in _dataset_cache:
                ds, ts = _dataset_cache[cache_key]
                if current_time - ts < DATASET_CACHE_TTL:
                    # Dataset is cached and valid, use its sample count
                    if hasattr(ds, "_index"):
                        total_samples = len(ds._index)

        result.append(
            DatasetInfo(
                name=name,
                root=str(root_val or ""),
                splits=["train", "val", "test"],
                total_samples=total_samples,
                languages=cls.LANGUAGES,
                tasks=cls.TASKS,
            )
        )

    # User-connected datasets
    connections = _load_connections()
    for conn in connections.values():
        result.append(
            DatasetInfo(
                name=f"user:{conn.name}",  # Prefix to distinguish
                root=conn.video_root,
                splits=["all"],
                total_samples=conn.video_count,
                languages=[conn.language] if conn.language != "unknown" else [],
                tasks=["exploration"],
            )
        )

    return result


@router.get("/{name}/videos")
async def get_dataset_videos(name: str, limit: int = Query(1000, ge=1, le=10000)):
    """
    Get video paths from a dataset for extraction.

    Returns a list of videos that can be used for feature extraction.
    Uses cached dataset instances and async I/O.
    """
    _add_server_log("INFO", f"Loading videos from dataset: {name}", "datasets")

    try:
        try:
            dataset = get_cached_dataset(name, split="all")
        except HTTPException:
            raise
        except Exception as e:
            # Dataset root not available - return empty list
            _add_server_log("WARNING", f"Dataset {name} not available: {e}", "datasets")
            return {
                "dataset": name,
                "videos": [],
                "error": f"Dataset not available: {e}",
            }

        def _collect_videos():
            """Sync helper to collect video paths with caching."""
            videos = []
            seen_paths: set[str] = set()
            path_cache: dict[str, tuple[bool, float | None]] = {}

            for sample_id in dataset._index[:limit]:
                try:
                    sample = dataset._load_sample(sample_id)
                    if sample.video_path:
                        path_str = str(sample.video_path)
                        if path_str not in seen_paths:
                            exists, size_mb = check_video_path(sample.video_path, path_cache)
                            if exists:
                                seen_paths.add(path_str)
                                videos.append(
                                    {
                                        "path": path_str,
                                        "name": sample.video_path.name,
                                        "size_mb": size_mb,
                                    }
                                )
                except (OSError, ValueError, KeyError) as e:
                    logger.debug(f"Failed to load sample {sample_id}: {e}")
                    continue
            return videos

        videos = await asyncio.to_thread(_collect_videos)

        _add_server_log("INFO", f"Found {len(videos)} videos in dataset {name}", "datasets")

        return {
            "dataset": name,
            "videos": videos,
            "total": len(videos),
        }

    except HTTPException:
        raise
    except Exception as e:
        _add_server_log("ERROR", f"Failed to load dataset videos: {e}", "datasets")
        raise HTTPException(500, safe_error_detail("Failed to load dataset", e))


@router.get("/{name}/samples")
@rate_limit("10/minute")
async def get_dataset_samples(
    request: Request,
    name: str,
    split: str = "all",
    limit: int = 50,
    offset: int = 0,
):
    """Get samples from a dataset. Uses cached dataset instances and async I/O."""
    try:
        try:
            dataset = get_cached_dataset(name, split=split)
        except HTTPException:
            raise
        except (OSError, ValueError, RuntimeError) as e:
            # Dataset root not available
            logger.debug(f"Dataset {name} not available: {e}")
            return {
                "dataset": name,
                "split": split,
                "samples": [],
                "total": 0,
                "error": f"Dataset not available: {e}",
            }

        # Get sample IDs
        total = len(dataset)
        sample_ids = dataset._index[offset : offset + limit]

        def _load_samples():
            """Sync helper to load samples with path caching."""
            samples = []
            path_cache: dict[str, tuple[bool, float | None]] = {}

            for sid in sample_ids:
                try:
                    sample = dataset._load_sample(sid)
                    has_video, _ = check_video_path(sample.video_path, path_cache)
                    samples.append(
                        SampleInfo(
                            id=sample.id,
                            video_path=str(sample.video_path) if sample.video_path else None,
                            has_video=has_video,
                            has_poses=sample.poses is not None,
                            num_segments=len(sample.segments) if sample.segments else 0,
                            num_glosses=len(sample.glosses) if sample.glosses else 0,
                            signer_id=sample.signer_id,
                            metadata=sample.metadata or {},
                        )
                    )
                except (OSError, ValueError, KeyError, RuntimeError) as e:
                    # Skip samples that fail to load
                    logger.debug(f"Failed to load sample {sid}: {e}")
                    samples.append(
                        SampleInfo(
                            id=sid,
                            metadata={"error": str(e)},
                        )
                    )
            return samples

        samples = await asyncio.to_thread(_load_samples)

        return {
            "dataset": name,
            "split": split,
            "samples": samples,
            "total": total,
            "offset": offset,
            "limit": limit,
        }
    except HTTPException:
        raise
    except (OSError, ValueError, RuntimeError) as e:
        logger.debug(f"Failed to load dataset {name}: {e}")
        raise HTTPException(500, safe_error_detail("Failed to load dataset", e))


@router.get("/{name}/sample/{sample_id}")
async def get_dataset_sample(name: str, sample_id: str):
    """Get detailed information about a single sample. Uses cached dataset and async I/O."""
    try:
        dataset = get_cached_dataset(name, split="all")

        if sample_id not in dataset._index:
            raise HTTPException(404, f"Sample not found: {sample_id}")

        def _load_sample_sync():
            sample = dataset._load_sample(sample_id)
            has_video, _ = check_video_path(sample.video_path)
            return {
                "id": sample.id,
                "dataset": name,
                "video_path": str(sample.video_path) if sample.video_path else None,
                "has_video": has_video,
                "signer_id": sample.signer_id,
                "glosses": sample.glosses[:100] if sample.glosses else [],  # Limit glosses
                "segments": [
                    SegmentData(
                        start=s.start,
                        end=s.end,
                        label=s.label,
                        confidence=s.confidence,
                        tier=s.tier,
                    )
                    for s in (sample.segments[:200] if sample.segments else [])  # Limit segments
                ],
                "num_total_segments": len(sample.segments) if sample.segments else 0,
                "num_total_glosses": len(sample.glosses) if sample.glosses else 0,
                "metadata": sample.metadata or {},
            }

        return await asyncio.to_thread(_load_sample_sync)
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to load sample", e))


# ============================================================================
# Vocabulary and Gloss Analysis
# ============================================================================


@router.get("/{name}/vocabulary")
@rate_limit("10/minute")
async def get_dataset_vocabulary(
    request: Request,
    name: str,
    limit: int = 100,
    max_samples: int = 500,
):
    """
    Get vocabulary statistics for a dataset.

    Collects glosses from sample.glosses if available, otherwise falls back
    to extracting labels from segments. Returns empty vocabulary for datasets
    that only have captions/translations without gloss annotations.
    """
    try:
        # Check vocabulary cache first
        _vocabulary_cache, _vocabulary_cache_lock, VOCABULARY_CACHE_TTL = get_vocabulary_cache()
        current_time = time.time()
        cache_key = f"{name}:{max_samples}"

        with _vocabulary_cache_lock:
            if cache_key in _vocabulary_cache:
                cached_data, ts = _vocabulary_cache[cache_key]
                if current_time - ts < VOCABULARY_CACHE_TTL:
                    logger.debug(f"Using cached vocabulary for {name}")
                    return {
                        "dataset": name,
                        "total_tokens": cached_data["total_tokens"],
                        "vocabulary_size": cached_data["vocabulary_size"],
                        "source": cached_data.get("source", "glosses"),
                        "samples_analyzed": cached_data.get("samples_analyzed", 0),
                        "frequencies": dict(list(cached_data["frequencies"].items())[:limit]),
                    }
                # Expired, remove from cache
                del _vocabulary_cache[cache_key]

        dataset = get_cached_dataset(name, split="all")

        def _collect_glosses():
            """Sync helper to collect glosses from samples."""
            all_glosses: list[str] = []
            source = "glosses"
            samples_loaded = 0

            for sample_id in dataset._index[:max_samples]:
                try:
                    sample = dataset._load_sample(sample_id)
                    samples_loaded += 1

                    # Try glosses first
                    if sample.glosses:
                        all_glosses.extend(sample.glosses)
                    # Fall back to segment labels if no glosses
                    elif sample.segments:
                        labels = [s.label for s in sample.segments if s.label]
                        if labels:
                            all_glosses.extend(labels)
                            if source == "glosses":
                                source = "segments"  # Mark that we're using segments
                except Exception as e:
                    logger.debug(f"Failed to load sample {sample_id}: {e}")
                    continue

            return all_glosses, source, samples_loaded

        all_glosses, source, samples_loaded = await asyncio.to_thread(_collect_glosses)
        counter = Counter(all_glosses)

        # Handle empty vocabulary gracefully
        if not all_glosses:
            logger.warning(f"No glosses found in dataset {name} (checked {samples_loaded} samples)")
            return {
                "dataset": name,
                "total_tokens": 0,
                "vocabulary_size": 0,
                "source": "none",
                "samples_analyzed": samples_loaded,
                "frequencies": {},
                "message": (
                    "This dataset does not have gloss annotations."
                    " It may only contain captions or translations."
                ),
            }

        # Cache the full vocabulary data
        full_data = {
            "total_tokens": len(all_glosses),
            "vocabulary_size": len(counter),
            "source": source,
            "samples_analyzed": samples_loaded,
            "frequencies": dict(counter.most_common()),
        }
        with _vocabulary_cache_lock:
            _vocabulary_cache[cache_key] = (full_data, current_time)
            logger.debug(
                f"Cached vocabulary for {name} ({len(counter)} unique glosses from {source})"
            )

        # Evict oldest entry if cache is too large
        _evict_oldest_cache_entry(
            _vocabulary_cache, _vocabulary_cache_lock, MAX_VOCABULARY_CACHE_SIZE
        )

        return {
            "dataset": name,
            "total_tokens": len(all_glosses),
            "vocabulary_size": len(counter),
            "source": source,
            "samples_analyzed": samples_loaded,
            "frequencies": dict(counter.most_common(limit)),
        }
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Failed to get vocabulary for {name}: {e}")
        raise HTTPException(500, safe_error_detail("Failed to get vocabulary", e))


@router.get("/{name}/gloss/{gloss}/examples")
@rate_limit("10/minute")
async def get_gloss_examples(
    request: Request,
    name: str,
    gloss: str,
    limit: int = 20,
):
    """Find examples of a specific gloss in the dataset. Uses cached segments when available."""
    try:
        dataset = get_cached_dataset(name, split="all")
        gloss_lower = gloss.lower()

        def _find_examples():
            """Sync helper to find gloss examples."""
            examples = []

            # First, try to use cached segments (much faster)
            _segments_cache, _segments_cache_lock, SEGMENTS_CACHE_TTL = get_segments_cache()
            current_time = time.time()
            cached_segments = None

            with _segments_cache_lock:
                # Check for any cached segments for this dataset
                for (cached_name, max_samples), (segs, signers, ts) in list(
                    _segments_cache.items()
                ):
                    if cached_name == name and current_time - ts < SEGMENTS_CACHE_TTL:
                        cached_segments = segs
                        break

            if cached_segments:
                logger.debug(f"Using cached segments for gloss examples: {name}")
                for sample_id, segments in cached_segments.items():
                    if len(examples) >= limit:
                        break
                    # Segments are always Segment objects from SegmentList
                    for i, seg in enumerate(segments):
                        label = getattr(seg, "label", None)
                        if label and label.lower() == gloss_lower:
                            # Get context from surrounding segments
                            context = []
                            for j in range(max(0, i - 2), min(len(segments), i + 3)):
                                if j != i:
                                    ctx_label = getattr(segments[j], "label", None)
                                    if ctx_label:
                                        context.append(ctx_label)

                            examples.append(
                                GlossExample(
                                    sample_id=sample_id,
                                    video_path=None,  # Not available from cache
                                    start=getattr(seg, "start", 0.0),
                                    end=getattr(seg, "end", 0.0),
                                    label=label,
                                    tier=getattr(seg, "tier", ""),
                                    context=context[:4],
                                )
                            )
                            if len(examples) >= limit:
                                break

                if examples:
                    return examples  # Found enough in cache

            # Fall back to scanning dataset (slower but more complete)
            for sid in dataset._index:
                if len(examples) >= limit:
                    break

                try:
                    sample = dataset._load_sample(sid)
                    if not sample.segments:
                        continue

                    # Find matching segments
                    for i, seg in enumerate(sample.segments):
                        if seg.label and seg.label.lower() == gloss_lower:
                            # Get context (surrounding glosses)
                            context = []
                            for j in range(max(0, i - 2), min(len(sample.segments), i + 3)):
                                if j != i and sample.segments[j].label:
                                    context.append(sample.segments[j].label)

                            examples.append(
                                GlossExample(
                                    sample_id=sample.id,
                                    video_path=(
                                        str(sample.video_path) if sample.video_path else None
                                    ),
                                    start=seg.start,
                                    end=seg.end,
                                    label=seg.label,
                                    tier=seg.tier,
                                    context=context[:4],  # Limit context
                                )
                            )

                            if len(examples) >= limit:
                                break
                except Exception as e:
                    logger.debug(f"Failed to load sample {sid}: {e}")
                    continue
            return examples

        examples = await asyncio.to_thread(_find_examples)

        return {
            "dataset": name,
            "gloss": gloss,
            "count": len(examples),
            "examples": examples,
        }
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(500, safe_error_detail("Failed to find examples", e))


# ============================================================================
# Dataset-based Linguistic Analysis Endpoints
# ============================================================================


@router.get("/{name}/analysis/concordance")
@rate_limit("10/minute")
async def get_dataset_concordance(
    request: Request,
    name: str,
    gloss: str = Query(..., description="Target gloss to search for"),
    context_window: int = Query(5, ge=1, le=20, description="Context window size"),
    max_samples: int = Query(500, ge=1, le=2000, description="Max samples to analyze"),
):
    """
    Build a concordance (KWIC) view for a target gloss in a dataset.

    Shows each occurrence with surrounding context for linguistic analysis.
    """
    from sltk.analysis.linguistic import build_concordance

    try:
        segments_by_file, signer_ids = await asyncio.to_thread(
            load_dataset_segments, name, max_samples
        )

        concordance_lines = await asyncio.to_thread(
            build_concordance,
            segments_by_file=segments_by_file,
            target_gloss=gloss,
            context_window=context_window,
            case_sensitive=False,
            signer_ids=signer_ids,
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Concordance analysis failed for {name}: {e}")
        raise HTTPException(500, safe_error_detail("Concordance analysis failed", e))

    return {
        "dataset": name,
        "gloss": gloss,
        "context_window": context_window,
        "total_occurrences": len(concordance_lines),
        "entries": [
            {
                "sample_id": line.file_id,
                "left_context": line.left_context,
                "target": line.gloss,
                "right_context": line.right_context,
                "start_time": line.start_time,
                "end_time": line.end_time,
                "duration": line.duration,
                "tier": line.tier,
                "signer_id": line.signer_id,
            }
            for line in concordance_lines[:200]  # Limit results
        ],
    }


@router.get("/{name}/analysis/ngrams")
@rate_limit("10/minute")
async def get_dataset_ngrams(
    request: Request,
    name: str,
    n: int = Query(2, ge=2, le=6, description="N-gram size"),
    min_freq: int = Query(2, ge=1, description="Minimum frequency"),
    max_ngrams: int = Query(100, ge=1, le=500, description="Max n-grams to return"),
    max_samples: int = Query(500, ge=1, le=2000, description="Max samples to analyze"),
):
    """
    Extract frequent n-grams (sign sequences) from a dataset.

    Useful for identifying common sign combinations and phrases.
    """
    from sltk.analysis.linguistic import extract_ngrams

    try:
        segments_by_file, _ = await asyncio.to_thread(load_dataset_segments, name, max_samples)

        ngrams = await asyncio.to_thread(
            extract_ngrams,
            segments_by_file=segments_by_file,
            n=n,
            min_frequency=min_freq,
            max_ngrams=max_ngrams,
            include_examples=True,
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"N-gram analysis failed for {name}: {e}")
        raise HTTPException(500, safe_error_detail("N-gram analysis failed", e))

    # Count total n-grams before filtering
    total_ngrams = sum(ng.count for ng in ngrams)

    return {
        "dataset": name,
        "n": n,
        "min_freq": min_freq,
        "total_ngrams": total_ngrams,
        "ngrams": [
            {
                "ngram": list(ng.glosses),
                "frequency": ng.count,
                "examples": ng.examples[:3],  # Limit examples
            }
            for ng in ngrams
        ],
    }


@router.get("/{name}/analysis/cooccurrence")
@rate_limit("10/minute")
async def get_dataset_cooccurrence(
    request: Request,
    name: str,
    gloss: str = Query(..., description="Target gloss to analyze"),
    window_size: int = Query(3, ge=1, le=10, description="Context window size"),
    min_freq: int = Query(2, ge=1, description="Minimum co-occurrence frequency"),
    max_samples: int = Query(500, ge=1, le=2000, description="Max samples to analyze"),
):
    """
    Analyze co-occurrence patterns for a target gloss in a dataset.

    Returns glosses that frequently appear near the target.
    """
    from sltk.analysis.linguistic import analyze_cooccurrence

    try:
        segments_by_file, _ = await asyncio.to_thread(load_dataset_segments, name, max_samples)

        result = await asyncio.to_thread(
            analyze_cooccurrence,
            segments_by_file=segments_by_file,
            target_gloss=gloss,
            window_size=window_size,
            case_sensitive=False,
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Co-occurrence analysis failed for {name}: {e}")
        raise HTTPException(500, safe_error_detail("Co-occurrence analysis failed", e))

    # Filter by min frequency and sort
    cooccurrences = [
        {"gloss1": gloss, "gloss2": collocate, "count": count}
        for collocate, count in result.top_collocates(100)
        if count >= min_freq
    ]

    return {
        "dataset": name,
        "target_gloss": gloss,
        "window_size": window_size,
        "min_freq": min_freq,
        "total_occurrences": result.total_occurrences,
        "total_pairs": len(cooccurrences),
        "cooccurrences": cooccurrences,
        "left_collocates": dict(Counter(result.left_collocates).most_common(20)),
        "right_collocates": dict(Counter(result.right_collocates).most_common(20)),
    }


@router.get("/{name}/analysis/duration")
@rate_limit("10/minute")
async def get_dataset_duration_analysis(
    request: Request,
    name: str,
    gloss: str | None = Query(None, description="Specific gloss (None = all)"),
    min_occurrences: int = Query(5, ge=1, description="Min occurrences for analysis"),
    max_samples: int = Query(500, ge=1, le=2000, description="Max samples to analyze"),
):
    """
    Analyze sign duration patterns in a dataset.

    Returns statistics about how long signs are held.
    """
    from sltk.analysis.linguistic import analyze_duration_patterns

    try:
        # Run blocking I/O in thread pool to avoid blocking event loop
        segments_by_file, signer_ids = await asyncio.to_thread(
            load_dataset_segments, name, max_samples
        )

        results = await asyncio.to_thread(
            analyze_duration_patterns,
            segments_by_file=segments_by_file,
            gloss=gloss,
            min_occurrences=min_occurrences,
            signer_ids=signer_ids,
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Duration analysis failed for {name}: {e}")
        raise HTTPException(500, safe_error_detail("Duration analysis failed", e))

    if not results:
        return {
            "dataset": name,
            "gloss_filter": gloss,
            "total_glosses": 0,
            "stats": [],
            "message": "No glosses met the minimum occurrence threshold",
        }

    stats = [
        {
            "gloss": analysis.gloss,
            "count": analysis.count,
            "mean_duration": round(analysis.mean_duration, 4),
            "std_duration": round(analysis.std_duration, 4),
            "min_duration": round(analysis.min_duration, 4),
            "max_duration": round(analysis.max_duration, 4),
            "median_duration": round(analysis.median_duration, 4),
            "by_signer": analysis.by_signer,
        }
        for analysis in results.values()
    ]

    # Sort by count
    stats.sort(key=lambda x: -x["count"])  # type: ignore[operator]

    return {
        "dataset": name,
        "gloss_filter": gloss,
        "total_glosses": len(stats),
        "stats": stats[:100],  # Limit to top 100
    }


@router.get("/{name}/analysis/minimal-pairs")
async def get_dataset_minimal_pairs(
    name: str,
    similarity_threshold: float = Query(0.7, ge=0.5, le=1.0, description="Min similarity"),
    max_pairs: int = Query(50, ge=1, le=200, description="Max pairs to return"),
    max_samples: int = Query(500, ge=1, le=2000, description="Max samples for vocabulary"),
):
    """
    Find potential minimal pairs (signs with similar glosses) in a dataset.

    Uses string similarity as a proxy for phonological similarity.
    """
    from sltk.analysis.linguistic import find_minimal_pairs

    try:
        segments_by_file, _ = await asyncio.to_thread(load_dataset_segments, name, max_samples)

        # Build vocabulary
        vocab = []
        for segments in segments_by_file.values():
            for seg in segments:
                label = getattr(seg, "label", None) or (
                    seg.get("label") if isinstance(seg, dict) else None
                )
                if label:
                    vocab.append(label)

        # Get unique glosses that appear at least twice
        gloss_counts = Counter(vocab)
        frequent_glosses = [g for g, c in gloss_counts.items() if c >= 2]

        pairs = await asyncio.to_thread(
            find_minimal_pairs,
            vocabulary=frequent_glosses,
            legacy_threshold=similarity_threshold,
            max_pairs=max_pairs,
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Minimal pairs analysis failed for {name}: {e}")
        raise HTTPException(500, safe_error_detail("Minimal pairs analysis failed", e))

    return {
        "dataset": name,
        "similarity_threshold": similarity_threshold,
        "vocabulary_size": len(frequent_glosses),
        "total_pairs": len(pairs),
        "pairs": [
            {
                "gloss1": p.sign1,
                "gloss2": p.sign2,
                "similarity": round(1 - p.distance, 3) if isinstance(p.distance, float) else 1.0,
                "distance": round(p.distance, 3) if isinstance(p.distance, float) else p.distance,
                "difference_type": p.differing_parameter or p.method,
            }
            for p in pairs
        ],
    }


@router.get("/{name}/analysis/cooccurrence-matrix")
async def get_dataset_cooccurrence_matrix(
    name: str,
    top_n: int = Query(30, ge=5, le=100, description="Number of top glosses"),
    window_size: int = Query(2, ge=1, le=5, description="Context window"),
    max_samples: int = Query(500, ge=1, le=2000, description="Max samples to analyze"),
):
    """
    Compute co-occurrence matrix for top glosses in a dataset.

    Returns a matrix showing which signs commonly appear together.
    """
    from sltk.analysis.linguistic import compute_cooccurrence_matrix

    try:
        segments_by_file, _ = await asyncio.to_thread(load_dataset_segments, name, max_samples)

        result = await asyncio.to_thread(
            compute_cooccurrence_matrix,
            segments_by_file=segments_by_file,
            top_n_glosses=top_n,
            window_size=window_size,
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Co-occurrence matrix analysis failed for {name}: {e}")
        raise HTTPException(500, safe_error_detail("Co-occurrence matrix analysis failed", e))

    return {
        "dataset": name,
        "top_n": top_n,
        "window_size": window_size,
        "glosses": result["glosses"],
        "matrix": result["matrix"],
        "frequencies": result["frequencies"],
    }
