"""CorpusDB — per-workspace SQLite database for ELAN annotation data."""

from __future__ import annotations

import functools
import json as _json_module
import logging
import math
import re
import sqlite3
import threading
import time
from collections import Counter, defaultdict
from collections.abc import Sequence
from datetime import datetime, timezone
from difflib import SequenceMatcher
from pathlib import Path
from typing import Any

from sltk.db import queries as Q
from sltk.db.ingest import parse_and_store_eaf
from sltk.db.schema import CREATE_TABLES, FTS_TRIGGERS, MIGRATE_V1_TO_V2, PRAGMA, SCHEMA_VERSION

_logger = logging.getLogger(__name__)


# ============================================================================
# Query caching infrastructure
# ============================================================================

_CACHE_MISS = object()


def _make_cache_key(name: str, *args: Any, **kwargs: Any) -> str:
    """Build a deterministic cache key from method name + arguments."""
    parts = [name]
    for a in args:
        if isinstance(a, dict):
            parts.append(_json_module.dumps(a, sort_keys=True))
        elif isinstance(a, (list, tuple)):
            parts.append(_json_module.dumps(a))
        else:
            parts.append(str(a))
    for k, v in sorted(kwargs.items()):
        if isinstance(v, dict):
            parts.append(f"{k}={_json_module.dumps(v, sort_keys=True)}")
        elif isinstance(v, (list, tuple)):
            parts.append(f"{k}={_json_module.dumps(v)}")
        else:
            parts.append(f"{k}={v}")
    return "|".join(parts)


def _cached_read(method):
    """Decorator: cache read-method results in ``self._query_cache``.

    Cache is invalidated by ``_clear_caches()`` on any write operation.
    """

    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        key = _make_cache_key(method.__name__, *args, **kwargs)
        hit = self._query_cache.get(key, _CACHE_MISS)
        if hit is not _CACHE_MISS:
            return hit
        result = method(self, *args, **kwargs)
        self._query_cache[key] = result
        return result

    return wrapper


def _cache_clearing(method):
    """Decorator: clear ``self._query_cache`` after write operations."""

    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        result = method(self, *args, **kwargs)
        self._clear_caches()
        return result

    return wrapper


class _RWLock:
    """Simple read-write lock: multiple concurrent readers, exclusive writer."""

    def __init__(self) -> None:
        self._read_ready = threading.Condition(threading.Lock())
        self._readers = 0

    def acquire_read(self) -> None:
        with self._read_ready:
            self._readers += 1

    def release_read(self) -> None:
        with self._read_ready:
            self._readers -= 1
            if self._readers == 0:
                self._read_ready.notify_all()

    def acquire_write(self) -> None:
        self._read_ready.acquire()
        while self._readers > 0:
            self._read_ready.wait()

    def release_write(self) -> None:
        self._read_ready.release()

    def __enter__(self) -> _RWLock:
        """Context manager for read access."""
        self.acquire_read()
        return self

    def __exit__(self, *exc: object) -> None:
        self.release_read()


class _WriteLock:
    """Context manager for write access on an _RWLock."""

    def __init__(self, rwlock: _RWLock) -> None:
        self._rwlock = rwlock

    def __enter__(self) -> _WriteLock:
        self._rwlock.acquire_write()
        return self

    def __exit__(self, *exc: object) -> None:
        self._rwlock.release_write()


class CorpusDB:
    """Thread-safe SQLite corpus database.

    Each instance manages a single ``.db`` file at *db_path*.
    Connections use WAL journal mode for concurrent read access.
    A read-write lock allows multiple concurrent readers while
    serializing writes.
    """

    def __init__(self, db_path: str | Path) -> None:
        self._db_path = Path(db_path)
        self._db_path.parent.mkdir(parents=True, exist_ok=True)
        self._rwlock = _RWLock()
        self._rlock = self._rwlock  # context manager for read lock
        self._wlock = _WriteLock(self._rwlock)  # context manager for write lock
        self._local = threading.local()  # per-thread read connections
        self._write_conn: sqlite3.Connection | None = None
        self._tier_cache: list[dict[str, Any]] | None = None
        self._tier_cache_time: float = 0
        self._query_cache: dict[str, Any] = {}
        self._init_db()

    # ------------------------------------------------------------------
    # Connection management
    # ------------------------------------------------------------------

    def _get_conn(self) -> sqlite3.Connection:
        """Get a per-thread read connection. Uses WAL for concurrent reads."""
        conn = getattr(self._local, "conn", None)
        if conn is None:
            conn = sqlite3.connect(
                str(self._db_path),
                check_same_thread=False,
            )
            conn.row_factory = sqlite3.Row
            conn.executescript(PRAGMA)
            self._local.conn = conn
        return conn

    def _get_write_conn(self) -> sqlite3.Connection:
        """Get the single shared write connection. Must hold _wlock."""
        if self._write_conn is None:
            self._write_conn = sqlite3.connect(
                str(self._db_path),
                check_same_thread=False,
            )
            self._write_conn.row_factory = sqlite3.Row
            self._write_conn.executescript(PRAGMA)
        return self._write_conn

    def _init_db(self) -> None:
        with self._wlock:
            conn = self._get_write_conn()
            conn.executescript(CREATE_TABLES)
            conn.executescript(FTS_TRIGGERS)
            # Check for migration
            try:
                row = conn.execute("SELECT value FROM meta WHERE key = 'schema_version'").fetchone()
                current_version = row["value"] if row else "0"
            except Exception:
                current_version = "0"
            if current_version < "2":
                _logger.info("Migrating corpus DB from v%s to v2", current_version)
                conn.executescript(MIGRATE_V1_TO_V2)
            conn.execute(
                "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
                ("schema_version", SCHEMA_VERSION),
            )
            conn.commit()

    def close(self) -> None:
        self._rwlock.acquire_write()
        try:
            if self._write_conn is not None:
                self._write_conn.close()
                self._write_conn = None
            # Close any per-thread read connection on this thread
            conn = getattr(self._local, "conn", None)
            if conn is not None:
                conn.close()
                self._local.conn = None
        finally:
            self._rwlock.release_write()

    def _clear_caches(self) -> None:
        """Clear all cached query results and the tier cache."""
        self._query_cache.clear()
        self._tier_cache = None

    def _cached_available_tiers(self) -> list[dict[str, Any]]:
        """Return available_tiers() with short-lived cache (5s TTL)."""
        now = time.monotonic()
        if self._tier_cache is not None and (now - self._tier_cache_time) < 5.0:
            return self._tier_cache
        result = self.available_tiers()
        self._tier_cache = result
        self._tier_cache_time = now
        return result

    # ------------------------------------------------------------------
    # Ingestion
    # ------------------------------------------------------------------

    def ingest_eaf(self, path: str | Path, source: str = "file") -> bool:
        """Parse and store an .eaf file. Skips if mtime unchanged.

        Returns ``True`` if ingested, ``False`` if skipped.
        """
        with self._wlock:
            conn = self._get_write_conn()
            result = parse_and_store_eaf(conn, path, source)
            if result:
                # Auto-populate file_metadata from ELAN participants
                file_row = conn.execute(
                    "SELECT id FROM files WHERE path = ?", (str(path),)
                ).fetchone()
                if file_row:
                    fid = file_row["id"]
                    participants = conn.execute(
                        "SELECT DISTINCT participant FROM tiers "
                        "WHERE file_id = ? AND participant IS NOT NULL AND participant != ''",
                        (fid,),
                    ).fetchall()
                    if participants:
                        # Store first participant as file-level metadata
                        conn.execute(
                            Q.INSERT_FILE_METADATA,
                            (fid, "participant", participants[0]["participant"], "elan"),
                        )
                conn.execute(
                    "INSERT OR REPLACE INTO meta (key, value) " "VALUES (?, ?)",
                    (
                        "last_updated",
                        datetime.now(timezone.utc).isoformat(),
                    ),
                )
                conn.commit()
                conn.execute("ANALYZE")
                self._clear_caches()
            return result

    def ingest_synthetic(
        self,
        video_path: str,
        gloss: str,
        tier_name: str = "Gloss",
    ) -> None:
        """Create a synthetic corpus entry from a video filename.

        Uses ``synthetic://{stem}`` as the file path to avoid UNIQUE
        conflicts with real ``.eaf`` files.
        """
        stem = Path(video_path).stem
        synthetic_path = f"synthetic://{stem}"
        with self._wlock:
            conn = self._get_write_conn()
            row = conn.execute(
                "SELECT id FROM files WHERE path = ?",
                (synthetic_path,),
            ).fetchone()
            if row:
                file_id = row[0]
                conn.execute(
                    "DELETE FROM segments WHERE file_id = ?",
                    (file_id,),
                )
                conn.execute(
                    "DELETE FROM tiers WHERE file_id = ?",
                    (file_id,),
                )
            else:
                cur = conn.execute(
                    Q.INSERT_FILE,
                    (synthetic_path, f"{stem}.synthetic", stem, None, None, 0.0, 1),
                )
                file_id = cur.lastrowid
            # Insert tier + segment
            cur = conn.execute(
                Q.INSERT_TIER,
                (file_id, tier_name, "default-lt", "", "", None),
            )
            tier_rowid = cur.lastrowid
            conn.execute(
                Q.INSERT_SEGMENT,
                (file_id, tier_rowid, f"syn_{stem}", 0, 0, gloss, None, "filename"),
            )
            conn.execute(
                "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
                ("last_updated", datetime.now(timezone.utc).isoformat()),
            )
            conn.commit()

    def remove_file(self, path: str | Path) -> None:
        """Delete a file and all its children (CASCADE)."""
        with self._wlock:
            conn = self._get_write_conn()
            conn.execute(Q.DELETE_FILE, (str(path),))
            conn.commit()

    def rebuild(self, directory: str | Path) -> int:
        """Drop all data and re-ingest every .eaf in *directory*.

        Returns the number of files ingested.
        """
        directory = Path(directory)
        with self._wlock:
            conn = self._get_write_conn()
            conn.executescript(
                "DELETE FROM segments;"
                "DELETE FROM tiers;"
                "DELETE FROM media_refs;"
                "DELETE FROM files;"
                "INSERT INTO segments_fts(segments_fts) "
                "VALUES ('rebuild');"
            )
            count = 0
            for eaf in sorted(directory.rglob("*.eaf")):
                if parse_and_store_eaf(conn, eaf, "file"):
                    count += 1
            conn.execute(
                "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
                (
                    "last_updated",
                    datetime.now(timezone.utc).isoformat(),
                ),
            )
            conn.commit()
            conn.execute("ANALYZE")
            self._clear_caches()
        return count

    # ------------------------------------------------------------------
    # Raw SQL execution (read-only)
    # ------------------------------------------------------------------

    _FORBIDDEN_SQL_RE = None  # lazily compiled

    def execute_readonly(self, sql: str, limit: int = 500) -> list[dict[str, Any]]:
        """Execute a read-only SQL query and return results as dicts.

        Only SELECT and WITH (CTE) statements are allowed.
        Results are capped at *limit* rows.
        """
        import re

        if self._FORBIDDEN_SQL_RE is None:
            CorpusDB._FORBIDDEN_SQL_RE = re.compile(
                r"\b(INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|ATTACH|DETACH|PRAGMA|REPLACE|REINDEX|VACUUM)\b",
                re.IGNORECASE,
            )

        # Strip SQL comments (-- line comments) before validation
        lines = sql.strip().splitlines()
        lines = [ln for ln in lines if not ln.strip().startswith("--")]
        stripped = "\n".join(lines).strip().rstrip(";")
        if not stripped:
            raise ValueError("Empty SQL query")

        # Must start with SELECT or WITH
        first_word = stripped.split()[0].upper()
        if first_word not in ("SELECT", "WITH"):
            raise ValueError(f"Only SELECT queries are allowed, got: {first_word}")

        # Reject any forbidden keywords anywhere in the query
        assert self._FORBIDDEN_SQL_RE is not None
        match = self._FORBIDDEN_SQL_RE.search(stripped)
        if match:
            raise ValueError(f"Forbidden SQL keyword: {match.group(0)}")

        # Wrap in a subquery to enforce LIMIT
        wrapped = f"SELECT * FROM ({stripped}) LIMIT {int(limit)}"

        with self._rlock:
            conn = self._get_conn()
            try:
                cursor = conn.execute(wrapped)
                columns = [desc[0] for desc in cursor.description]
                rows = cursor.fetchall()
            except sqlite3.Error as e:
                raise ValueError(f"SQL error: {e}") from e

        return [{col: row[i] for i, col in enumerate(columns)} for row in rows]

    # ------------------------------------------------------------------
    # Queries
    # ------------------------------------------------------------------

    def file_list(self) -> list[dict[str, Any]]:
        """List all files with segment counts."""
        with self._rlock:
            rows = self._get_conn().execute(Q.FILE_LIST).fetchall()
        return [
            {
                "path": r["path"],
                "name": r["name"],
                "stem": r["stem"],
                "segment_count": r["segment_count"],
            }
            for r in rows
        ]

    def corpus_summary(self, source: str | None = None) -> dict[str, Any]:
        """Total files, segments, vocabulary size, signer count.

        ``vocabulary_size`` counts distinct labels on **gloss tiers only**.
        ``total_segments`` counts all segments (all tiers).
        """
        gloss_sub = self._GLOSS_TIER_SUBQUERY
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            conn = self._get_conn()
            if source:
                # Filtered summary: count only files/segments matching source
                row = conn.execute(
                    "SELECT COUNT(DISTINCT s.file_id) AS total_files, "
                    "COUNT(s.id) AS total_segments, "
                    "COUNT(DISTINCT t.participant) AS signer_count, "
                    "COUNT(DISTINCT t.id) AS total_tiers "
                    "FROM segments s "
                    "JOIN tiers t ON s.tier_rowid = t.id "
                    f"WHERE 1=1 {src_clause}",
                    src_params,
                ).fetchone()
            else:
                row = conn.execute(Q.CORPUS_SUMMARY).fetchone()
            vocab_row = conn.execute(
                "SELECT COUNT(DISTINCT s.label_lower) AS vocab "
                "FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' "
                f"AND s.tier_rowid IN ({gloss_sub}) {src_clause}",
                src_params,
            ).fetchone()
        return {
            "total_files": row["total_files"],
            "total_segments": row["total_segments"],
            "vocabulary_size": vocab_row["vocab"],
            "signer_count": row["signer_count"],
            "total_tiers": row["total_tiers"],
        }

    def status(self) -> dict[str, Any]:
        """DB stats: file count, segment count, last updated."""
        with self._rlock:
            row = self._get_conn().execute(Q.DB_STATUS).fetchone()
        return {
            "file_count": row["file_count"],
            "segment_count": row["segment_count"],
            "last_updated": row["last_updated"],
        }

    def top_glosses(
        self,
        top_n: int = 100,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """Return the *top_n* most frequent glosses."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        src_clause, src_params = self._build_source_filter_sql(source)
        query = (
            "SELECT s.label_lower AS gloss, count(*) AS freq "
            "FROM segments s "
            f"WHERE s.label IS NOT NULL AND s.label != '' {file_clause} {tier_clause} {src_clause} "
            "GROUP BY s.label_lower ORDER BY freq DESC LIMIT ?"
        )
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(query, file_params + tier_params + src_params + [top_n])
                .fetchall()
            )
        return [{"gloss": r["gloss"], "frequency": r["freq"]} for r in rows]

    def find_gloss(
        self,
        gloss: str,
        signer_id: str | None = None,
        tier_id: str | None = None,
    ) -> list[dict[str, Any]]:
        """All occurrences of *gloss* with file/tier info."""
        with self._rlock:
            conn = self._get_conn()
            where = "s.label_lower = lower(?)"
            params: list[Any] = [gloss]
            if signer_id:
                where += " AND t.participant = ?"
                params.append(signer_id)
            if tier_id:
                where += " AND t.tier_id = ?"
                params.append(tier_id)
            sql = (
                "SELECT s.label, s.start_ms, s.end_ms, s.duration_ms, "
                "s.annotation_id, s.confidence, s.source, "
                "t.tier_id, t.participant, "
                "f.path, f.name, f.stem "
                "FROM segments s "
                "JOIN tiers t ON s.tier_rowid = t.id "
                "JOIN files f ON s.file_id = f.id "
                f"WHERE {where} "
                "ORDER BY f.name, s.start_ms"
            )
            rows = conn.execute(sql, params).fetchall()
        return [
            {
                "label": r["label"],
                "start_ms": r["start_ms"],
                "end_ms": r["end_ms"],
                "duration_ms": r["duration_ms"],
                "annotation_id": r["annotation_id"],
                "confidence": r["confidence"],
                "source": r["source"],
                "tier": r["tier_id"],
                "participant": r["participant"],
                "file_path": r["path"],
                "file_name": r["name"],
                "file_stem": r["stem"],
            }
            for r in rows
        ]

    def gloss_duration_stats(
        self,
        gloss: str,
        signer_id: str | None = None,
        tier_id: str | None = None,
    ) -> dict[str, Any]:
        """Duration stats (avg/std/min/max/median) for *gloss*."""
        with self._rlock:
            conn = self._get_conn()
            where = "s.label_lower = lower(?)"
            params: list[Any] = [gloss]
            needs_tier_join = bool(signer_id or tier_id)
            if signer_id:
                where += " AND t.participant = ?"
                params.append(signer_id)
            if tier_id:
                where += " AND t.tier_id = ?"
                params.append(tier_id)
            tier_join = (
                "JOIN tiers t ON s.tier_rowid = t.id "
                if needs_tier_join
                else "JOIN tiers t ON s.tier_rowid = t.id "
            )
            row = conn.execute(
                f"SELECT count(*) AS cnt, avg(s.duration_ms) AS mean_ms, "
                f"min(s.duration_ms) AS min_ms, max(s.duration_ms) AS max_ms "
                f"FROM segments s {tier_join} WHERE {where}",
                params,
            ).fetchone()
            dur_rows = conn.execute(
                f"SELECT s.duration_ms FROM segments s {tier_join} "
                f"WHERE {where} ORDER BY s.duration_ms",
                params,
            ).fetchall()

        count = row["cnt"]
        if count == 0:
            return {
                "count": 0,
                "mean_ms": 0,
                "std_ms": 0,
                "min_ms": 0,
                "max_ms": 0,
                "median_ms": 0,
            }

        durations = [r["duration_ms"] for r in dur_rows]
        mean = row["mean_ms"]
        variance = sum((d - mean) ** 2 for d in durations) / count if count > 1 else 0
        std = math.sqrt(variance)

        mid = count // 2
        if count % 2 == 0:
            median = (durations[mid - 1] + durations[mid]) / 2.0
        else:
            median = durations[mid]

        return {
            "count": count,
            "mean_ms": round(mean, 1),
            "std_ms": round(std, 1),
            "min_ms": row["min_ms"],
            "max_ms": row["max_ms"],
            "median_ms": round(median, 1),
        }

    def compare_gloss_duration_by_signer(
        self,
        gloss: str,
        tier_id: str | None = None,
    ) -> list[dict[str, Any]]:
        """Per-signer duration stats for *gloss*."""
        with self._rlock:
            where = "s.label_lower = lower(?) AND t.participant != ''"
            params: list[Any] = [gloss]
            if tier_id:
                where += " AND t.tier_id = ?"
                params.append(tier_id)
            sql = (
                "SELECT t.participant, count(*) AS cnt, "
                "avg(s.duration_ms) AS mean_ms, "
                "min(s.duration_ms) AS min_ms, max(s.duration_ms) AS max_ms "
                "FROM segments s "
                "JOIN tiers t ON s.tier_rowid = t.id "
                f"WHERE {where} "
                "GROUP BY t.participant ORDER BY cnt DESC"
            )
            rows = self._get_conn().execute(sql, params).fetchall()
        return [
            {
                "participant": r["participant"],
                "count": r["cnt"],
                "mean_ms": round(r["mean_ms"], 1) if r["mean_ms"] else 0,
                "min_ms": r["min_ms"],
                "max_ms": r["max_ms"],
            }
            for r in rows
        ]

    def search_glosses(self, query: str, limit: int = 100) -> list[dict[str, Any]]:
        """Full-text search over segment labels."""
        with self._rlock:
            rows = self._get_conn().execute(Q.FTS_SEARCH, (query, limit)).fetchall()
        return [
            {
                "label": r["label"],
                "start_ms": r["start_ms"],
                "end_ms": r["end_ms"],
                "duration_ms": r["duration_ms"],
                "annotation_id": r["annotation_id"],
                "source": r["source"],
                "tier": r["tier_id"],
                "participant": r["participant"],
                "file_path": r["path"],
                "file_name": r["name"],
            }
            for r in rows
        ]

    def concordance(
        self,
        target: str,
        context_window: int = 5,
        limit: int = 0,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """KWIC concordance view for *target*.

        *limit* caps the total number of results (0 = unlimited).
        """
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        src_clause, src_params = self._build_source_filter_sql(source)

        # Bulk query: fetch all segments from files containing the target,
        # filtered by tier roles and metadata, ordered for sequential scan.
        sql = (
            "SELECT s.label, s.start_ms, s.end_ms, s.file_id, "
            "t.tier_id, f.path, f.name "
            "FROM segments s "
            "JOIN tiers t ON s.tier_rowid = t.id "
            "JOIN files f ON s.file_id = f.id "
            "WHERE s.file_id IN ("
            "  SELECT DISTINCT s2.file_id FROM segments s2 "
            "  WHERE s2.label_lower = lower(?)"
            ") "
            f"AND s.label IS NOT NULL AND s.label != '' "
            f"{file_clause} {tier_clause} {src_clause} "
            "ORDER BY s.file_id, t.tier_id, s.start_ms"
        )
        params = [target] + file_params + tier_params + src_params

        with self._rlock:
            rows = self._get_conn().execute(sql, params).fetchall()

        # Group by file_id, then build KWIC context
        results: list[dict[str, Any]] = []
        target_lower = target.lower()

        # Single pass: group segments by file
        current_file: int | None = None
        file_segs: list[Any] = []

        def _process_file(segs: list) -> None:
            labels = [s["label"] for s in segs]
            for i, seg in enumerate(segs):
                if seg["label"].lower() == target_lower:
                    left = labels[max(0, i - context_window) : i]
                    right = labels[i + 1 : i + 1 + context_window]
                    results.append(
                        {
                            "left": left,
                            "target": seg["label"],
                            "right": right,
                            "start_ms": seg["start_ms"],
                            "end_ms": seg["end_ms"],
                            "tier": seg["tier_id"],
                            "file_path": seg["path"],
                            "file_name": seg["name"],
                        }
                    )

        for row in rows:
            if row["file_id"] != current_file:
                if file_segs:
                    _process_file(file_segs)
                    if limit > 0 and len(results) >= limit:
                        return results[:limit]
                current_file = row["file_id"]
                file_segs = []
            file_segs.append(row)

        if file_segs:
            _process_file(file_segs)

        return results[:limit] if limit > 0 else results

    def ngram_frequency(
        self,
        n: int = 2,
        limit: int = 50,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """N-gram frequency across all files."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        src_clause, src_params = self._build_source_filter_sql(source)

        sql = (
            "SELECT s.label, s.file_id "
            "FROM segments s "
            "JOIN tiers t ON s.tier_rowid = t.id "
            f"WHERE s.label IS NOT NULL AND s.label != '' "
            f"{file_clause} {tier_clause} {src_clause} "
            "ORDER BY s.file_id, t.tier_id, s.start_ms"
        )
        with self._rlock:
            rows = self._get_conn().execute(sql, file_params + tier_params + src_params).fetchall()

        # Single pass: build n-grams per file boundary
        counter: Counter[tuple[str, ...]] = Counter()
        current_file: int | None = None
        labels: list[str] = []

        for row in rows:
            if row["file_id"] != current_file:
                # Process previous file's labels
                for i in range(len(labels) - n + 1):
                    counter[tuple(labels[i : i + n])] += 1
                current_file = row["file_id"]
                labels = []
            labels.append(row["label"])

        # Final file
        for i in range(len(labels) - n + 1):
            counter[tuple(labels[i : i + n])] += 1

        return [
            {"ngram": list(gram), "frequency": freq} for gram, freq in counter.most_common(limit)
        ]

    def cooccurrence(
        self,
        target: str,
        window: int = 3,
        limit: int = 50,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """Co-occurrence of *target* with other labels."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        src_clause, src_params = self._build_source_filter_sql(source)

        # Bulk fetch: only files that contain the target
        sql = (
            "SELECT s.label, s.file_id "
            "FROM segments s "
            f"WHERE s.file_id IN ("
            f"  SELECT DISTINCT s2.file_id FROM segments s2 "
            f"  WHERE s2.label_lower = lower(?)"
            f") "
            f"AND s.label IS NOT NULL AND s.label != '' "
            f"{file_clause} {tier_clause} {src_clause} "
            "ORDER BY s.file_id, s.start_ms"
        )
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(sql, [target] + file_params + tier_params + src_params)
                .fetchall()
            )

        counter: Counter[str] = Counter()
        target_lower = target.lower()
        current_file: int | None = None
        labels: list[str] = []

        def _process_file(labels: list[str]) -> None:
            for i, lab in enumerate(labels):
                if lab.lower() != target_lower:
                    continue
                start = max(0, i - window)
                end = min(len(labels), i + window + 1)
                for j in range(start, end):
                    if j != i:
                        counter[labels[j]] += 1

        for row in rows:
            if row["file_id"] != current_file:
                if labels:
                    _process_file(labels)
                current_file = row["file_id"]
                labels = []
            labels.append(row["label"])

        if labels:
            _process_file(labels)

        return [{"label": lab, "count": cnt} for lab, cnt in counter.most_common(limit)]

    def vocabulary_stats(
        self,
        limit: int | None = None,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
        source: str | None = None,
    ) -> dict[str, Any]:
        """Vocabulary frequencies and aggregate stats."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        src_clause, src_params = self._build_source_filter_sql(source)
        base_where = (
            f"WHERE s.label IS NOT NULL AND s.label != '' {file_clause} {tier_clause} {src_clause}"
        )
        limit_clause = "LIMIT ?" if limit else ""
        limit_params = [limit] if limit else []
        with self._rlock:
            conn = self._get_conn()
            rows = conn.execute(
                f"SELECT s.label_lower AS gloss, count(*) AS freq FROM segments s "
                f"{base_where} GROUP BY s.label_lower ORDER BY freq DESC {limit_clause}",
                file_params + tier_params + src_params + limit_params,
            ).fetchall()
            total_row = conn.execute(
                f"SELECT count(*) AS total FROM segments s {base_where}",
                file_params + tier_params + src_params,
            ).fetchone()
        total_tokens = total_row["total"]
        frequencies: dict[str, int] = {r["gloss"]: r["freq"] for r in rows}
        vocabulary_size = len(frequencies)
        return {
            "vocabulary_size": vocabulary_size,
            "total_tokens": total_tokens,
            "frequencies": frequencies,
        }

    def cooccurrence_matrix(
        self,
        top_n: int = 30,
        window: int = 2,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
        source: str | None = None,
    ) -> dict[str, Any]:
        """Co-occurrence matrix for top N glosses."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        src_clause, src_params = self._build_source_filter_sql(source)

        with self._rlock:
            conn = self._get_conn()
            top_rows = conn.execute(
                f"SELECT s.label_lower AS gloss, count(*) AS freq "
                f"FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' {file_clause} {tier_clause} {src_clause} "
                f"GROUP BY s.label_lower ORDER BY freq DESC LIMIT ?",
                file_params + tier_params + src_params + [top_n],
            ).fetchall()
            glosses = [r["gloss"] for r in top_rows]
            freq_map = {r["gloss"]: r["freq"] for r in top_rows}

            if not glosses:
                return {"glosses": [], "matrix": [], "frequencies": {}}

            # Bulk fetch all labels ordered by file + time
            rows = conn.execute(
                f"SELECT s.label, s.file_id "
                f"FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' "
                f"{file_clause} {tier_clause} {src_clause} "
                f"ORDER BY s.file_id, s.start_ms",
                file_params + tier_params + src_params,
            ).fetchall()

        gloss_set = set(glosses)
        idx = {g: i for i, g in enumerate(glosses)}
        matrix = [[0] * len(glosses) for _ in range(len(glosses))]

        # Single pass: build co-occurrence per file boundary
        current_file: int | None = None
        labels: list[str] = []

        def _process_labels(labels: list[str]) -> None:
            for i, lab in enumerate(labels):
                if lab not in gloss_set:
                    continue
                start = max(0, i - window)
                end = min(len(labels), i + window + 1)
                for j in range(start, end):
                    if j != i and labels[j] in gloss_set:
                        matrix[idx[lab]][idx[labels[j]]] += 1

        for row in rows:
            if row["file_id"] != current_file:
                if labels:
                    _process_labels(labels)
                current_file = row["file_id"]
                labels = []
            labels.append(row["label"].lower() if row["label"] else "")

        if labels:
            _process_labels(labels)

        return {
            "glosses": glosses,
            "matrix": matrix,
            "frequencies": freq_map,
        }

    def all_duration_stats(
        self,
        limit: int = 50,
        min_count: int = 5,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """Per-gloss duration aggregates for top glosses."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    f"SELECT s.label_lower AS gloss, s.duration_ms "
                    f"FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' AND s.duration_ms > 0 "
                    f"{file_clause} {tier_clause} {src_clause} "
                    f"ORDER BY s.label_lower",
                    file_params + tier_params + src_params,
                )
                .fetchall()
            )

        # Group durations by gloss
        by_gloss: dict[str, list[int]] = defaultdict(list)
        for r in rows:
            by_gloss[r["gloss"]].append(r["duration_ms"])

        results: list[dict[str, Any]] = []
        for gloss, durations in by_gloss.items():
            count = len(durations)
            if count < min_count:
                continue
            mean = sum(durations) / count
            variance = sum((d - mean) ** 2 for d in durations) / count if count > 1 else 0
            std = math.sqrt(variance)
            durations_sorted = sorted(durations)
            mid = count // 2
            if count % 2 == 0:
                median = (durations_sorted[mid - 1] + durations_sorted[mid]) / 2.0
            else:
                median = durations_sorted[mid]
            results.append(
                {
                    "gloss": gloss,
                    "count": count,
                    "mean_ms": round(mean, 1),
                    "std_ms": round(std, 1),
                    "min_ms": min(durations),
                    "max_ms": max(durations),
                    "median_ms": round(median, 1),
                }
            )

        # Sort by count desc, take top `limit`
        results.sort(key=lambda x: x["count"], reverse=True)
        return results[:limit]

    def minimal_pairs(
        self,
        threshold: float = 0.7,
        max_pairs: int = 50,
        source: str | None = None,
    ) -> dict[str, Any]:
        """Find glosses that are similar in form (SequenceMatcher). Gloss tiers only."""
        gloss_filter = f"AND s.tier_rowid IN ({self._GLOSS_TIER_SUBQUERY})"
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    "SELECT s.label_lower AS gloss, COUNT(*) AS freq "
                    "FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' {gloss_filter} {src_clause} "
                    "GROUP BY s.label_lower ORDER BY freq DESC LIMIT ?",
                    src_params + [200],
                )
                .fetchall()
            )

        glosses = [r["gloss"] for r in rows if len(r["gloss"]) > 1]
        pairs: list[dict[str, Any]] = []

        for i in range(len(glosses)):
            for j in range(i + 1, len(glosses)):
                ratio = SequenceMatcher(None, glosses[i], glosses[j]).ratio()
                if ratio >= threshold and ratio < 1.0:
                    pairs.append(
                        {
                            "gloss1": glosses[i],
                            "gloss2": glosses[j],
                            "similarity": round(ratio, 3),
                        }
                    )
                    if len(pairs) >= max_pairs:
                        break
            if len(pairs) >= max_pairs:
                break

        pairs.sort(key=lambda x: x["similarity"], reverse=True)
        return {
            "vocabulary_size": len(glosses),
            "total_pairs": len(pairs),
            "similarity_threshold": threshold,
            "pairs": pairs,
        }

    # ------------------------------------------------------------------
    # Dashboard / frequency distribution
    # ------------------------------------------------------------------

    def dashboard_stats(self, source: str | None = None) -> dict[str, Any]:
        """Comprehensive dashboard metrics.

        Vocabulary metrics (vocabulary_size, total_tokens, hapax, dis_legomena)
        are computed from **gloss tiers only**, not all tiers.
        """
        gloss_filter = f"AND s.tier_rowid IN ({self._GLOSS_TIER_SUBQUERY})"
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            conn = self._get_conn()
            if source:
                summary = conn.execute(
                    "SELECT COUNT(DISTINCT s.file_id) AS total_files, "
                    "COUNT(s.id) AS total_segments, "
                    "COUNT(DISTINCT t.participant) AS signer_count "
                    "FROM segments s "
                    "JOIN tiers t ON s.tier_rowid = t.id "
                    f"WHERE 1=1 {src_clause}",
                    src_params,
                ).fetchone()
            else:
                summary = conn.execute(Q.CORPUS_SUMMARY).fetchone()

            # Gloss frequency distribution
            freq_rows = conn.execute(
                "SELECT s.label_lower AS gloss, COUNT(*) AS freq "
                "FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' {gloss_filter} {src_clause} "
                "GROUP BY s.label_lower ORDER BY freq DESC",
                src_params,
            ).fetchall()

            # Duration stats (gloss tiers only)
            dur_row = conn.execute(
                "SELECT AVG(s.duration_ms) AS avg_dur, "
                "       SUM(s.duration_ms) AS total_dur "
                "FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' "
                f"AND s.duration_ms > 0 {gloss_filter} {src_clause}",
                src_params,
            ).fetchone()

            # Median duration via OFFSET trick
            median_row = conn.execute(
                "SELECT s.duration_ms FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' "
                f"AND s.duration_ms > 0 {gloss_filter} {src_clause} "
                "ORDER BY s.duration_ms "
                "LIMIT 1 OFFSET ("
                "  SELECT COUNT(*) / 2 FROM segments s2 "
                f"  WHERE s2.label IS NOT NULL AND s2.label != '' "
                f"  AND s2.duration_ms > 0 "
                f"  AND s2.tier_rowid IN ({self._GLOSS_TIER_SUBQUERY})"
                f"  {src_clause}"
                ")",
                src_params + src_params,
            ).fetchone()

            # Unique tier names (not row count)
            if source:
                tier_names_row = conn.execute(
                    "SELECT COUNT(DISTINCT t.tier_id) AS cnt FROM tiers t "
                    "JOIN segments s ON s.tier_rowid = t.id "
                    f"WHERE 1=1 {src_clause}",
                    src_params,
                ).fetchone()
            else:
                tier_names_row = conn.execute(
                    "SELECT COUNT(DISTINCT t.tier_id) AS cnt FROM tiers t"
                ).fetchone()

            # Per-signer token counts (top 10)
            signer_rows = conn.execute(
                "SELECT t.participant AS signer, COUNT(*) AS tokens "
                "FROM segments s "
                "JOIN tiers t ON t.id = s.tier_rowid "
                f"WHERE s.label IS NOT NULL AND s.label != '' "
                f"AND t.participant != '' {gloss_filter} {src_clause} "
                "GROUP BY t.participant ORDER BY tokens DESC LIMIT 10",
                src_params,
            ).fetchall()

        frequencies = {r["gloss"]: r["freq"] for r in freq_rows}
        total_tokens = sum(frequencies.values())
        vocab_size = len(frequencies)
        hapax = sum(1 for f in frequencies.values() if f == 1)
        dis_legomena = sum(1 for f in frequencies.values() if f == 2)

        avg_dur = round(dur_row["avg_dur"]) if dur_row["avg_dur"] else 0
        total_dur_ms = dur_row["total_dur"] or 0
        median_dur = median_row["duration_ms"] if median_row else 0
        total_files = summary["total_files"]

        return {
            "total_files": total_files,
            "total_segments": summary["total_segments"],
            "vocabulary_size": vocab_size,
            "signer_count": summary["signer_count"],
            "unique_tier_names": tier_names_row["cnt"] if tier_names_row else 0,
            "total_tokens": total_tokens,
            "hapax_count": hapax,
            "dis_legomena_count": dis_legomena,
            "avg_signs_per_file": round(total_tokens / total_files, 1) if total_files else 0,
            "avg_duration_ms": avg_dur,
            "median_duration_ms": median_dur,
            "total_duration_sec": round(total_dur_ms / 1000, 1),
            "ttr": round(vocab_size / total_tokens, 4) if total_tokens else 0,
            "top_signers": [{"signer": r["signer"], "tokens": r["tokens"]} for r in signer_rows],
        }

    def frequency_distribution(
        self,
        limit: int = 500,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """Ranked frequency distribution (for Zipf curve). Gloss tiers only."""
        gloss_filter = f"AND s.tier_rowid IN ({self._GLOSS_TIER_SUBQUERY})"
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    "SELECT s.label_lower AS gloss, COUNT(*) AS freq "
                    "FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' {gloss_filter} {src_clause} "
                    "GROUP BY s.label_lower ORDER BY freq DESC",
                    src_params,
                )
                .fetchall()
            )
        result = []
        for rank, r in enumerate(rows[:limit], 1):
            result.append(
                {
                    "rank": rank,
                    "gloss": r["gloss"],
                    "frequency": r["freq"],
                }
            )
        return result

    def duration_histogram(
        self,
        bins: int = 30,
        max_ms: int = 3000,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """Histogram bins for sign durations. Gloss tiers only."""
        gloss_filter = f"AND s.tier_rowid IN ({self._GLOSS_TIER_SUBQUERY})"
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    "SELECT s.duration_ms FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' AND s.duration_ms > 0 {gloss_filter} {src_clause}",
                    src_params,
                )
                .fetchall()
            )

        durations = [r["duration_ms"] for r in rows if r["duration_ms"] <= max_ms]
        if not durations:
            return []

        bin_width = max_ms / bins
        histogram: list[dict[str, Any]] = []
        for i in range(bins):
            lo = int(i * bin_width)
            hi = int((i + 1) * bin_width)
            count = sum(1 for d in durations if lo <= d < hi)
            histogram.append(
                {
                    "bin_start": lo,
                    "bin_end": hi,
                    "count": count,
                }
            )
        return histogram

    # ------------------------------------------------------------------
    # Lexical richness
    # ------------------------------------------------------------------

    def lexical_richness(
        self,
        mattr_window: int = 50,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
    ) -> dict[str, Any]:
        """Compute lexical richness metrics.

        Metrics:
        - TTR (Type-Token Ratio)
        - Hapax legomena / dislegomena
        - Yule's K
        - Simpson's Diversity Index
        - Brunet's W
        - Honore's R
        - MATTR (Moving Average TTR)
        """
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    f"SELECT s.label_lower AS label FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' "
                    f"{file_clause} {tier_clause} "
                    f"ORDER BY s.file_id, s.start_ms",
                    file_params + tier_params,
                )
                .fetchall()
            )

        labels = [r["label"] for r in rows]
        n = len(labels)
        if n == 0:
            return {
                "total_tokens": 0,
                "vocabulary_size": 0,
                "ttr": 0,
                "hapax_count": 0,
                "hapax_pct": 0,
                "dis_legomena_count": 0,
                "dis_legomena_pct": 0,
                "yules_k": 0,
                "simpsons_d": 0,
                "brunets_w": 0,
                "honores_r": 0,
                "mattr": 0,
                "mattr_window": mattr_window,
            }

        freq = Counter(labels)
        v = len(freq)
        hapax = sum(1 for f in freq.values() if f == 1)
        dis = sum(1 for f in freq.values() if f == 2)

        ttr = v / n if n > 0 else 0

        # Yule's K
        freq_of_freq = Counter(freq.values())
        m2 = sum(i * i * vi for i, vi in freq_of_freq.items())
        yules_k = 10000 * (m2 - n) / (n * n) if n > 1 else 0

        # Simpson's D
        simpsons_d = 1 - sum(f * (f - 1) for f in freq.values()) / (n * (n - 1)) if n > 1 else 0

        # Brunet's W: N^(V^-0.172)
        brunets_w = n ** (v**-0.172) if v > 0 else 0

        # Honore's R: 100 * ln(N) / (1 - V1/V)
        if v > 0 and hapax < v:
            honores_r = 100 * math.log(n) / (1 - hapax / v)
        else:
            honores_r = 0

        # MATTR — O(n) sliding window counter
        if n <= mattr_window:
            mattr = ttr
        else:
            win_counter: Counter[str] = Counter(labels[:mattr_window])
            ttr_sum = len(win_counter) / mattr_window
            for i in range(1, n - mattr_window + 1):
                old = labels[i - 1]
                win_counter[old] -= 1
                if win_counter[old] == 0:
                    del win_counter[old]
                win_counter[labels[i + mattr_window - 1]] += 1
                ttr_sum += len(win_counter) / mattr_window
            mattr = ttr_sum / (n - mattr_window + 1)

        return {
            "total_tokens": n,
            "vocabulary_size": v,
            "ttr": round(ttr, 6),
            "hapax_count": hapax,
            "hapax_pct": round(hapax / v * 100, 2) if v > 0 else 0,
            "dis_legomena_count": dis,
            "dis_legomena_pct": round(dis / v * 100, 2) if v > 0 else 0,
            "yules_k": round(yules_k, 4),
            "simpsons_d": round(simpsons_d, 6),
            "brunets_w": round(brunets_w, 4),
            "honores_r": round(honores_r, 4),
            "mattr": round(mattr, 6),
            "mattr_window": mattr_window,
        }

    def per_signer_richness(
        self,
        mattr_window: int = 50,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
    ) -> list[dict[str, Any]]:
        """Lexical richness metrics per signer."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    f"SELECT t.participant, s.label_lower AS label "
                    f"FROM segments s "
                    f"JOIN tiers t ON s.tier_rowid = t.id "
                    f"WHERE s.label IS NOT NULL AND s.label != '' AND t.participant != '' "
                    f"{file_clause} {tier_clause} "
                    f"ORDER BY t.participant, s.file_id, s.start_ms",
                    file_params + tier_params,
                )
                .fetchall()
            )

        by_signer: dict[str, list[str]] = defaultdict(list)
        for r in rows:
            by_signer[r["participant"]].append(r["label"])

        results: list[dict[str, Any]] = []
        for participant, labels in sorted(by_signer.items()):
            n = len(labels)
            freq = Counter(labels)
            v = len(freq)
            hapax = sum(1 for f in freq.values() if f == 1)
            ttr = v / n if n > 0 else 0

            # MATTR — O(n) sliding window counter
            if n <= mattr_window:
                mattr = ttr
            else:
                wc: Counter[str] = Counter(labels[:mattr_window])
                ts = len(wc) / mattr_window
                for i in range(1, n - mattr_window + 1):
                    old = labels[i - 1]
                    wc[old] -= 1
                    if wc[old] == 0:
                        del wc[old]
                    wc[labels[i + mattr_window - 1]] += 1
                    ts += len(wc) / mattr_window
                mattr = ts / (n - mattr_window + 1)

            # Yule's K
            freq_of_freq = Counter(freq.values())
            m2 = sum(i * i * vi for i, vi in freq_of_freq.items())
            yules_k = 10000 * (m2 - n) / (n * n) if n > 1 else 0

            results.append(
                {
                    "participant": participant,
                    "total_tokens": n,
                    "vocabulary_size": v,
                    "ttr": round(ttr, 4),
                    "hapax_count": hapax,
                    "mattr": round(mattr, 4),
                    "yules_k": round(yules_k, 4),
                }
            )

        return results

    # ------------------------------------------------------------------
    # Sign transitions (bigram probabilities)
    # ------------------------------------------------------------------

    def sign_transitions(
        self,
        top_n: int = 30,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
    ) -> dict[str, Any]:
        """Bigram transition probabilities for top N signs.

        Returns a matrix of P(sign_j | sign_i) along with
        per-sign entropy and predictability scores.
        """
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    f"SELECT s.file_id, s.label_lower AS label "
                    f"FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' "
                    f"{file_clause} {tier_clause} "
                    f"ORDER BY s.file_id, s.start_ms",
                    file_params + tier_params,
                )
                .fetchall()
            )

        # Build file-grouped label sequences
        sequences: dict[int, list[str]] = defaultdict(list)
        for r in rows:
            sequences[r["file_id"]].append(r["label"])

        # Count bigrams
        bigram_counts: Counter[tuple[str, str]] = Counter()
        unigram_counts: Counter[str] = Counter()
        for labels in sequences.values():
            for i in range(len(labels)):
                unigram_counts[labels[i]] += 1
                if i < len(labels) - 1:
                    bigram_counts[(labels[i], labels[i + 1])] += 1

        # Top N by frequency
        top_signs = [s for s, _ in unigram_counts.most_common(top_n)]
        sign_set = set(top_signs)
        idx = {s: i for i, s in enumerate(top_signs)}

        # Build probability matrix
        n = len(top_signs)
        matrix = [[0.0] * n for _ in range(n)]
        for (a, b), count in bigram_counts.items():
            if a in sign_set and b in sign_set:
                total_a = unigram_counts[a]
                matrix[idx[a]][idx[b]] = round(count / total_a, 4) if total_a > 0 else 0

        # Per-sign entropy: H(X|sign_i) = -sum(p * log2(p))
        entropy_scores: list[dict[str, Any]] = []
        for sign in top_signs:
            probs = []
            total = unigram_counts[sign]
            if total == 0:
                entropy_scores.append({"sign": sign, "entropy": 0, "predictability": 1})
                continue
            for other in top_signs:
                p = bigram_counts.get((sign, other), 0) / total
                if p > 0:
                    probs.append(p)
            if probs:
                entropy = -sum(p * math.log2(p) for p in probs)
                max_entropy = math.log2(len(probs)) if len(probs) > 1 else 1
                predictability = 1 - (entropy / max_entropy) if max_entropy > 0 else 1
            else:
                entropy = 0
                predictability = 1
            entropy_scores.append(
                {
                    "sign": sign,
                    "entropy": round(entropy, 4),
                    "predictability": round(predictability, 4),
                }
            )

        return {
            "signs": top_signs,
            "matrix": matrix,
            "entropy_scores": entropy_scores,
            "frequencies": {s: unigram_counts[s] for s in top_signs},
        }

    # ------------------------------------------------------------------
    # Fluency / temporal analysis
    # ------------------------------------------------------------------

    def fluency_stats(
        self,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
    ) -> dict[str, Any]:
        """Per-file signing rate and pause analysis."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    f"SELECT s.file_id, f.name, f.stem, "
                    f"s.start_ms, s.end_ms, s.duration_ms, s.label "
                    f"FROM segments s "
                    f"JOIN files f ON s.file_id = f.id "
                    f"WHERE s.label IS NOT NULL AND s.label != '' AND s.duration_ms > 0 "
                    f"{file_clause} {tier_clause} "
                    f"ORDER BY s.file_id, s.start_ms",
                    file_params + tier_params,
                )
                .fetchall()
            )

        # Group by file
        by_file: dict[int, list[dict]] = defaultdict(list)
        file_meta: dict[int, dict] = {}
        for r in rows:
            fid = r["file_id"]
            by_file[fid].append(
                {
                    "start_ms": r["start_ms"],
                    "end_ms": r["end_ms"],
                    "duration_ms": r["duration_ms"],
                }
            )
            if fid not in file_meta:
                file_meta[fid] = {"name": r["name"], "stem": r["stem"]}

        file_stats: list[dict[str, Any]] = []
        all_pauses: list[float] = []
        all_durations: list[int] = []

        for fid, segs in by_file.items():
            if not segs:
                continue
            segs.sort(key=lambda x: x["start_ms"])
            n_signs = len(segs)
            durations = [s["duration_ms"] for s in segs]
            all_durations.extend(durations)
            avg_duration = sum(durations) / n_signs

            # Time span of the file
            file_start = segs[0]["start_ms"]
            file_end = segs[-1]["end_ms"]
            span_sec = (file_end - file_start) / 1000

            signs_per_min = (n_signs / span_sec * 60) if span_sec > 0 else 0

            # Pauses (gaps between consecutive signs)
            pauses: list[float] = []
            for i in range(1, len(segs)):
                gap = segs[i]["start_ms"] - segs[i - 1]["end_ms"]
                if gap > 0:
                    pauses.append(gap)
                    all_pauses.append(gap)

            avg_pause = sum(pauses) / len(pauses) if pauses else 0

            file_stats.append(
                {
                    "file_name": file_meta[fid]["name"],
                    "file_stem": file_meta[fid]["stem"],
                    "sign_count": n_signs,
                    "span_sec": round(span_sec, 2),
                    "signs_per_min": round(signs_per_min, 1),
                    "avg_sign_duration_ms": round(avg_duration, 1),
                    "pause_count": len(pauses),
                    "avg_pause_ms": round(avg_pause, 1),
                    "median_pause_ms": round(sorted(pauses)[len(pauses) // 2], 1) if pauses else 0,
                }
            )

        # Global pause distribution
        pause_histogram: list[dict[str, Any]] = []
        if all_pauses:
            max_pause = min(max(all_pauses), 5000)
            bin_count = 20
            bin_width = max_pause / bin_count
            for i in range(bin_count):
                lo = int(i * bin_width)
                hi = int((i + 1) * bin_width)
                count = sum(1 for p in all_pauses if lo <= p < hi)
                pause_histogram.append(
                    {
                        "bin_start": lo,
                        "bin_end": hi,
                        "count": count,
                    }
                )

        # Duration vs frequency (for scatter plot)
        duration_freq: list[dict[str, Any]] = []
        with self._rlock:
            conn = self._get_conn()
            freq_rows = conn.execute(
                f"SELECT s.label_lower AS gloss, count(*) AS freq "
                f"FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' "
                f"{file_clause} {tier_clause} "
                f"GROUP BY s.label_lower ORDER BY freq DESC",
                file_params + tier_params,
            ).fetchall()
            raw = conn.execute(
                f"SELECT s.label_lower AS gloss, s.duration_ms "
                f"FROM segments s "
                f"WHERE s.label IS NOT NULL AND s.label != '' AND s.duration_ms > 0 "
                f"{file_clause} {tier_clause} "
                f"ORDER BY s.label_lower",
                file_params + tier_params,
            ).fetchall()
        freq_map = {r["gloss"]: r["freq"] for r in freq_rows}
        by_gloss: dict[str, list[int]] = defaultdict(list)
        for r in raw:
            by_gloss[r["gloss"]].append(r["duration_ms"])
        for gloss, durs in by_gloss.items():
            if gloss in freq_map and len(durs) >= 3:
                duration_freq.append(
                    {
                        "gloss": gloss,
                        "frequency": freq_map[gloss],
                        "mean_duration_ms": round(sum(durs) / len(durs), 1),
                    }
                )

        return {
            "file_stats": sorted(file_stats, key=lambda x: x["signs_per_min"], reverse=True),
            "pause_histogram": pause_histogram,
            "duration_vs_frequency": duration_freq[:200],
            "global_avg_pause_ms": round(sum(all_pauses) / len(all_pauses), 1) if all_pauses else 0,
            "global_median_pause_ms": (
                round(sorted(all_pauses)[len(all_pauses) // 2], 1) if all_pauses else 0
            ),
        }

    def per_signer_fluency(
        self,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
    ) -> list[dict[str, Any]]:
        """Signing rate comparison across signers."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    f"SELECT s.file_id, f.name, f.stem, t.participant, "
                    f"s.start_ms, s.end_ms, s.duration_ms, s.label "
                    f"FROM segments s "
                    f"JOIN tiers t ON s.tier_rowid = t.id "
                    f"JOIN files f ON s.file_id = f.id "
                    f"WHERE s.label IS NOT NULL AND s.label != '' AND s.duration_ms > 0 "
                    f"{file_clause} {tier_clause} "
                    f"ORDER BY t.participant, s.file_id, s.start_ms",
                    file_params + tier_params,
                )
                .fetchall()
            )

        by_signer: dict[str, list[dict]] = defaultdict(list)
        for r in rows:
            if r["participant"]:
                by_signer[r["participant"]].append(
                    {
                        "start_ms": r["start_ms"],
                        "end_ms": r["end_ms"],
                        "duration_ms": r["duration_ms"],
                    }
                )

        results: list[dict[str, Any]] = []
        for participant, segs in sorted(by_signer.items()):
            if not segs:
                continue
            segs.sort(key=lambda x: x["start_ms"])
            n = len(segs)
            durations = [s["duration_ms"] for s in segs]
            avg_dur = sum(durations) / n
            span_sec = (segs[-1]["end_ms"] - segs[0]["start_ms"]) / 1000
            spm = (n / span_sec * 60) if span_sec > 0 else 0

            pauses = []
            for i in range(1, len(segs)):
                gap = segs[i]["start_ms"] - segs[i - 1]["end_ms"]
                if gap > 0:
                    pauses.append(gap)

            results.append(
                {
                    "participant": participant,
                    "sign_count": n,
                    "signs_per_min": round(spm, 1),
                    "avg_sign_duration_ms": round(avg_dur, 1),
                    "avg_pause_ms": round(sum(pauses) / len(pauses), 1) if pauses else 0,
                }
            )

        return results

    # ------------------------------------------------------------------
    # Path metadata extraction
    # ------------------------------------------------------------------

    @staticmethod
    def _extract_path_segment(path: str, index_from_right: int) -> str | None:
        """Extract a segment from *path* by index from the right.

        index 0 = filename stem, 1 = parent directory, 2 = grandparent, etc.
        Returns ``None`` if the path doesn't have enough components.
        """
        parts = Path(path).parts
        if index_from_right == 0:
            return Path(path).stem
        # parts[-1] is filename, parts[-2] is parent dir, etc.
        idx = len(parts) - 1 - index_from_right
        if idx < 0:
            return None
        return parts[idx]

    def preview_path_extraction(self, segment_index: int) -> list[dict[str, Any]]:
        """Preview what value would be extracted from each file path."""
        with self._rlock:
            rows = self._get_conn().execute(Q.FILE_PATHS_WITH_IDS).fetchall()

        results: list[dict[str, Any]] = []
        for r in rows:
            path = r["path"]
            if path.startswith("synthetic://"):
                continue
            extracted = self._extract_path_segment(path, segment_index)
            results.append(
                {
                    "file_id": r["id"],
                    "path": path,
                    "extracted_value": extracted,
                }
            )
        return results

    def apply_path_as_participant(
        self, segment_index: int, only_empty: bool = True
    ) -> dict[str, int]:
        """Update tiers.participant from a path segment for all files."""
        with self._wlock:
            conn = self._get_write_conn()
            rows = conn.execute(Q.FILE_PATHS_WITH_IDS).fetchall()

            updated_files = 0
            updated_tiers = 0
            query = Q.UPDATE_TIER_PARTICIPANT_EMPTY if only_empty else Q.UPDATE_TIER_PARTICIPANT_ALL

            for r in rows:
                path = r["path"]
                if path.startswith("synthetic://"):
                    continue
                value = self._extract_path_segment(path, segment_index)
                if value is None:
                    continue
                cur = conn.execute(query, (value, r["id"]))
                if cur.rowcount > 0:
                    updated_files += 1
                    updated_tiers += cur.rowcount

            conn.commit()
        return {"updated_files": updated_files, "updated_tiers": updated_tiers}

    def apply_path_as_tier(self, segment_index: int, tier_name: str = "PathMeta") -> dict[str, int]:
        """Create a new tier per file with the extracted path value as label."""
        with self._wlock:
            conn = self._get_write_conn()
            rows = conn.execute(Q.FILE_PATHS_WITH_IDS).fetchall()

            created_tiers = 0
            created_segments = 0

            for r in rows:
                path = r["path"]
                file_id = r["id"]
                if path.startswith("synthetic://"):
                    continue
                value = self._extract_path_segment(path, segment_index)
                if value is None:
                    continue

                # Skip if tier already exists for this file
                existing = conn.execute(
                    "SELECT id FROM tiers WHERE file_id = ? AND tier_id = ?",
                    (file_id, tier_name),
                ).fetchone()
                if existing:
                    continue

                # Get max end_ms for file span
                max_row = conn.execute(Q.FILE_MAX_END_MS, (file_id,)).fetchone()
                max_end = max_row["max_end_ms"] if max_row else 0

                # Create tier
                cur = conn.execute(
                    Q.INSERT_TIER,
                    (file_id, tier_name, "default-lt", value, "", None),
                )
                tier_rowid = cur.lastrowid
                created_tiers += 1

                # Create single segment spanning the file
                conn.execute(
                    Q.INSERT_SEGMENT,
                    (
                        file_id,
                        tier_rowid,
                        f"path_{file_id}",
                        0,
                        max_end,
                        value,
                        None,
                        "path_extraction",
                    ),
                )
                created_segments += 1

            conn.commit()
        return {"created_tiers": created_tiers, "created_segments": created_segments}

    def signers_per_file(self) -> dict[str, list[str]]:
        """Return a mapping of file path -> list of distinct signer/participant IDs."""
        with self._rlock:
            rows = self._get_conn().execute(Q.SIGNERS_PER_FILE).fetchall()
        result: dict[str, list[str]] = {}
        for r in rows:
            signers_str = r["signers"]
            if signers_str:
                result[r["path"]] = [s.strip() for s in signers_str.split(",") if s.strip()]
        return result

    # ------------------------------------------------------------------
    # File metadata
    # ------------------------------------------------------------------

    def set_file_metadata(self, file_id: int, key: str, value: str, source: str = "manual") -> None:
        """Insert or replace a metadata key/value for a file."""
        with self._wlock:
            conn = self._get_write_conn()
            conn.execute(Q.INSERT_FILE_METADATA, (file_id, key, value, source))
            conn.commit()

    def get_file_metadata(self, file_id: int | None = None) -> list[dict[str, Any]]:
        """Return all file_metadata rows, optionally filtered to one file."""
        with self._rlock:
            conn = self._get_conn()
            if file_id is not None:
                rows = conn.execute(
                    "SELECT fm.file_id, fm.key, fm.value, fm.source, f.path "
                    "FROM file_metadata fm JOIN files f ON fm.file_id = f.id "
                    "WHERE fm.file_id = ? ORDER BY fm.key",
                    (file_id,),
                ).fetchall()
            else:
                rows = conn.execute(
                    "SELECT fm.file_id, fm.key, fm.value, fm.source, f.path "
                    "FROM file_metadata fm JOIN files f ON fm.file_id = f.id "
                    "ORDER BY fm.key, f.path"
                ).fetchall()
        return [dict(r) for r in rows]

    def list_metadata_keys(self) -> list[dict[str, Any]]:
        """Return metadata key summary: unique values, file count, sources."""
        with self._rlock:
            rows = self._get_conn().execute(Q.LIST_METADATA_KEYS).fetchall()
        return [
            {
                "key": r["key"],
                "unique_values": r["unique_values"],
                "file_count": r["file_count"],
                "sources": [s.strip() for s in r["sources"].split(",")],
            }
            for r in rows
        ]

    def get_metadata_values(self, key: str) -> list[dict[str, Any]]:
        """Return distinct values for a metadata key with file counts."""
        with self._rlock:
            rows = self._get_conn().execute(Q.METADATA_VALUES_FOR_KEY, (key,)).fetchall()
        return [{"value": r["value"], "file_count": r["file_count"]} for r in rows]

    def metadata_status(self) -> dict[str, Any]:
        """Overall metadata summary."""
        with self._rlock:
            conn = self._get_conn()
            keys = conn.execute(Q.LIST_METADATA_KEYS).fetchall()
            total_files = conn.execute("SELECT count(*) AS c FROM files").fetchone()["c"]
            files_with = conn.execute(
                "SELECT COUNT(DISTINCT file_id) AS c FROM file_metadata"
            ).fetchone()["c"]
            elan_participants = conn.execute(
                "SELECT COUNT(DISTINCT t.participant) AS c FROM tiers t "
                "WHERE t.participant IS NOT NULL AND t.participant != ''"
            ).fetchone()["c"]
        return {
            "keys": [
                {
                    "key": r["key"],
                    "unique_values": r["unique_values"],
                    "file_count": r["file_count"],
                    "sources": [s.strip() for s in r["sources"].split(",")],
                }
                for r in keys
            ],
            "elan_participants": elan_participants,
            "total_files": total_files,
            "files_with_metadata": files_with,
        }

    # ------------------------------------------------------------------
    # Tier roles
    # ------------------------------------------------------------------

    def list_unique_tier_ids(self) -> list[dict[str, Any]]:
        """All unique tier IDs with segment counts and assigned roles."""
        with self._rlock:
            rows = self._get_conn().execute(Q.LIST_UNIQUE_TIER_IDS).fetchall()
        return [
            {
                "tier_id": r["tier_id"],
                "linguistic_type": r["linguistic_type"],
                "segment_count": r["segment_count"],
                "file_count": r["file_count"],
                "role": r["role"],
                "is_explicit": bool(r["is_explicit"]),
            }
            for r in rows
        ]

    def set_tier_role(self, tier_id: str, role: str) -> None:
        """Set/update the role for a tier ID pattern."""
        with self._wlock:
            conn = self._get_write_conn()
            conn.execute(Q.SET_TIER_ROLE, (tier_id, role))
            conn.commit()

    def set_tier_roles_bulk(self, assignments: dict[str, str]) -> int:
        """Batch update tier roles. Returns number of rows affected."""
        with self._wlock:
            conn = self._get_write_conn()
            count = 0
            for tier_id, role in assignments.items():
                conn.execute(Q.SET_TIER_ROLE, (tier_id, role))
                count += 1
            conn.commit()
        return count

    def get_tier_roles(self) -> dict[str, str]:
        """Return {tier_id_pattern: role} mapping."""
        with self._rlock:
            rows = self._get_conn().execute(Q.GET_TIER_ROLES).fetchall()
        return {r["tier_id_pattern"]: r["role"] for r in rows}

    def delete_tier_role(self, pattern: str) -> bool:
        """Delete a single tier role assignment. Returns True if deleted."""
        with self._wlock:
            conn = self._get_write_conn()
            cur = conn.execute(Q.DELETE_TIER_ROLE, (pattern,))
            conn.commit()
        return cur.rowcount > 0

    def delete_all_tier_roles(self) -> int:
        """Delete all tier role assignments. Returns count deleted."""
        with self._wlock:
            conn = self._get_write_conn()
            cur = conn.execute(Q.DELETE_ALL_TIER_ROLES)
            conn.commit()
        return cur.rowcount

    # ------------------------------------------------------------------
    # Group comparison
    # ------------------------------------------------------------------

    def grouped_comparison(
        self,
        group_by: str = "participant",
        metadata_key: str | None = None,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """Compare linguistic metrics grouped by signer or metadata key.

        *group_by*: ``"participant"`` or ``"metadata"``.
        When ``"metadata"``, *metadata_key* must be provided.
        Gloss tiers only.
        """
        gloss_sub = self._GLOSS_TIER_SUBQUERY
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            conn = self._get_conn()
            if group_by == "metadata" and metadata_key:
                rows = conn.execute(
                    "SELECT fm.value AS group_value, "
                    "COUNT(DISTINCT s.label_lower) AS vocabulary_size, "
                    "COUNT(s.id) AS total_tokens, "
                    "AVG(s.duration_ms) AS avg_duration_ms, "
                    "COUNT(DISTINCT s.file_id) AS file_count "
                    "FROM segments s "
                    "JOIN files f ON s.file_id = f.id "
                    "JOIN file_metadata fm ON fm.file_id = f.id AND fm.key = ? "
                    "WHERE s.label IS NOT NULL AND s.label != '' "
                    f"AND s.tier_rowid IN ({gloss_sub}) {src_clause} "
                    "GROUP BY fm.value ORDER BY total_tokens DESC",
                    [metadata_key] + src_params,
                ).fetchall()
            else:
                rows = conn.execute(
                    "SELECT t.participant AS group_value, "
                    "COUNT(DISTINCT s.label_lower) AS vocabulary_size, "
                    "COUNT(s.id) AS total_tokens, "
                    "AVG(s.duration_ms) AS avg_duration_ms, "
                    "COUNT(DISTINCT s.file_id) AS file_count "
                    "FROM segments s "
                    "JOIN tiers t ON s.tier_rowid = t.id "
                    "WHERE s.label IS NOT NULL AND s.label != '' AND t.participant != '' "
                    f"AND s.tier_rowid IN ({gloss_sub}) {src_clause} "
                    "GROUP BY t.participant ORDER BY total_tokens DESC",
                    src_params,
                ).fetchall()
        return [
            {
                "group_value": r["group_value"],
                "vocabulary_size": r["vocabulary_size"],
                "total_tokens": r["total_tokens"],
                "avg_duration_ms": round(r["avg_duration_ms"] or 0, 1),
                "file_count": r["file_count"],
            }
            for r in rows
        ]

    def top_glosses_for_group(
        self,
        group_by: str,
        group_value: str,
        metadata_key: str | None = None,
        limit: int = 30,
        source: str | None = None,
    ) -> list[dict[str, Any]]:
        """Top glosses for a specific group value. Gloss tiers only."""
        gloss_sub = self._GLOSS_TIER_SUBQUERY
        src_clause, src_params = self._build_source_filter_sql(source)
        with self._rlock:
            conn = self._get_conn()
            if group_by == "metadata" and metadata_key:
                rows = conn.execute(
                    "SELECT s.label_lower AS gloss, COUNT(*) AS freq "
                    "FROM segments s "
                    "JOIN files f ON s.file_id = f.id "
                    "JOIN file_metadata fm ON fm.file_id = f.id AND fm.key = ? AND fm.value = ? "
                    "WHERE s.label IS NOT NULL AND s.label != '' "
                    f"AND s.tier_rowid IN ({gloss_sub}) {src_clause} "
                    "GROUP BY s.label_lower ORDER BY freq DESC LIMIT ?",
                    [metadata_key, group_value] + src_params + [limit],
                ).fetchall()
            else:
                rows = conn.execute(
                    "SELECT s.label_lower AS gloss, COUNT(*) AS freq "
                    "FROM segments s "
                    "JOIN tiers t ON s.tier_rowid = t.id "
                    "WHERE s.label IS NOT NULL AND s.label != '' AND t.participant = ? "
                    f"AND s.tier_rowid IN ({gloss_sub}) {src_clause} "
                    "GROUP BY s.label_lower ORDER BY freq DESC LIMIT ?",
                    [group_value] + src_params + [limit],
                ).fetchall()
        return [{"gloss": r["gloss"], "freq": r["freq"]} for r in rows]

    # ------------------------------------------------------------------
    # Hand comparison
    # ------------------------------------------------------------------

    def hand_comparison(
        self,
        top_n: int = 50,
        left_tier_ids: list[str] | None = None,
        right_tier_ids: list[str] | None = None,
    ) -> dict[str, Any]:
        """Compare left vs right hand tiers.

        When *left_tier_ids* / *right_tier_ids* are given, uses those
        explicit tier IDs.  Otherwise falls back to tier_roles table
        entries with role ``left_hand`` / ``right_hand``.
        """
        with self._rlock:
            conn = self._get_conn()
            # Use explicit-tier path when EITHER side has tier IDs provided
            if left_tier_ids is not None or right_tier_ids is not None:
                lt = left_tier_ids or []
                rt = right_tier_ids or []
                left_stats = self._hand_stats_by_tier_ids(conn, lt)
                right_stats = self._hand_stats_by_tier_ids(conn, rt)
                left_vocab = self._hand_vocab_by_tier_ids(conn, lt)
                right_vocab = self._hand_vocab_by_tier_ids(conn, rt)
                simultaneous = self._simultaneous_by_tier_ids(conn, lt, rt, top_n)
            else:
                left_stats = conn.execute(Q.HAND_STATS_BY_ROLE, ("left_hand",)).fetchone()
                right_stats = conn.execute(Q.HAND_STATS_BY_ROLE, ("right_hand",)).fetchone()
                left_vocab = conn.execute(Q.HAND_VOCAB_BY_ROLE, ("left_hand",)).fetchall()
                right_vocab = conn.execute(Q.HAND_VOCAB_BY_ROLE, ("right_hand",)).fetchall()
                simultaneous = conn.execute(Q.SIMULTANEOUS_SIGNS, (top_n,)).fetchall()

        left_set = {r["gloss"] for r in left_vocab}
        right_set = {r["gloss"] for r in right_vocab}
        shared = left_set & right_set
        left_only = left_set - right_set
        right_only = right_set - left_set

        def _stats_dict(row: Any) -> dict:
            if row is None or row["total_tokens"] == 0:
                return {
                    "total_tokens": 0,
                    "vocabulary_size": 0,
                    "avg_duration_ms": 0,
                    "file_count": 0,
                }
            return {
                "total_tokens": row["total_tokens"],
                "vocabulary_size": row["vocabulary_size"],
                "avg_duration_ms": round(row["avg_duration_ms"] or 0, 1),
                "file_count": row["file_count"],
            }

        return {
            "left_hand": _stats_dict(left_stats),
            "right_hand": _stats_dict(right_stats),
            "left_vocab": [{"gloss": r["gloss"], "freq": r["freq"]} for r in left_vocab[:top_n]],
            "right_vocab": [{"gloss": r["gloss"], "freq": r["freq"]} for r in right_vocab[:top_n]],
            "shared_count": len(shared),
            "left_only_count": len(left_only),
            "right_only_count": len(right_only),
            "simultaneous": [
                {"left_gloss": r["left_gloss"], "right_gloss": r["right_gloss"], "freq": r["freq"]}
                for r in simultaneous
            ],
        }

    # ------------------------------------------------------------------
    # Tier-ID-based hand helpers (for explicit tier selection)
    # ------------------------------------------------------------------

    @staticmethod
    def _hand_stats_by_tier_ids(
        conn: sqlite3.Connection, tier_ids: list[str]
    ) -> sqlite3.Row | None:
        if not tier_ids:
            return None
        ph = ", ".join("?" for _ in tier_ids)
        return conn.execute(
            f"SELECT COUNT(s.id) AS total_tokens, "
            f"COUNT(DISTINCT s.label_lower) AS vocabulary_size, "
            f"AVG(s.duration_ms) AS avg_duration_ms, "
            f"COUNT(DISTINCT s.file_id) AS file_count "
            f"FROM segments s "
            f"JOIN tiers t ON s.tier_rowid = t.id "
            f"WHERE t.tier_id IN ({ph}) "
            f"AND s.label IS NOT NULL AND s.label != ''",
            tier_ids,
        ).fetchone()

    @staticmethod
    def _hand_vocab_by_tier_ids(conn: sqlite3.Connection, tier_ids: list[str]) -> list[sqlite3.Row]:
        if not tier_ids:
            return []
        ph = ", ".join("?" for _ in tier_ids)
        return conn.execute(
            f"SELECT s.label_lower AS gloss, COUNT(*) AS freq "
            f"FROM segments s "
            f"JOIN tiers t ON s.tier_rowid = t.id "
            f"WHERE t.tier_id IN ({ph}) "
            f"AND s.label IS NOT NULL AND s.label != '' "
            f"GROUP BY s.label_lower ORDER BY freq DESC",
            tier_ids,
        ).fetchall()

    @staticmethod
    def _simultaneous_by_tier_ids(
        conn: sqlite3.Connection,
        left_tier_ids: list[str],
        right_tier_ids: list[str],
        top_n: int,
    ) -> list[sqlite3.Row]:
        if not left_tier_ids or not right_tier_ids:
            return []
        lph = ", ".join("?" for _ in left_tier_ids)
        rph = ", ".join("?" for _ in right_tier_ids)
        return conn.execute(
            f"SELECT lh.label_lower AS left_gloss, rh.label_lower AS right_gloss, "
            f"COUNT(*) AS freq "
            f"FROM segments lh "
            f"JOIN tiers lt ON lh.tier_rowid = lt.id "
            f"JOIN segments rh ON rh.file_id = lh.file_id "
            f"JOIN tiers rt ON rh.tier_rowid = rt.id "
            f"WHERE lt.tier_id IN ({lph}) AND rt.tier_id IN ({rph}) "
            f"AND lh.label IS NOT NULL AND lh.label != '' "
            f"AND rh.label IS NOT NULL AND rh.label != '' "
            f"AND lh.start_ms < rh.end_ms AND rh.start_ms < lh.end_ms "
            f"GROUP BY lh.label_lower, rh.label_lower "
            f"ORDER BY freq DESC LIMIT ?",
            left_tier_ids + right_tier_ids + [top_n],
        ).fetchall()

    # ------------------------------------------------------------------
    # Filter helpers
    # ------------------------------------------------------------------

    @staticmethod
    def _build_file_filter_sql(
        metadata_filter: dict[str, str] | None,
    ) -> tuple[str, list[Any]]:
        """Build SQL WHERE clause fragment for file metadata filtering.

        Returns (clause_str, params) where clause starts with AND.
        """
        if not metadata_filter:
            return ("", [])
        clauses: list[str] = []
        params: list[Any] = []
        for key, value in metadata_filter.items():
            clauses.append(
                "AND s.file_id IN "
                "(SELECT file_id FROM file_metadata WHERE key = ? AND value = ?)"
            )
            params.extend([key, value])
        return (" ".join(clauses), params)

    # Patterns used to identify gloss tiers by name when no explicit
    # tier_roles are provided.  Matches tier_id case-insensitively.
    _GLOSS_TIER_PATTERNS: list[str] = ["gloss", "idgloss", "id-gloss"]

    # Reusable SQL subquery that selects tier rowids for gloss tiers.
    # Matches by name patterns OR explicit 'gloss' role in tier_roles table.
    _GLOSS_TIER_SUBQUERY = (
        "SELECT t.id FROM tiers t "
        "LEFT JOIN tier_roles tr ON t.tier_id = tr.tier_id_pattern "
        "WHERE tr.role = 'gloss' "
        "OR (tr.role IS NULL AND ("
        "LOWER(t.tier_id) LIKE '%gloss%' "
        "OR LOWER(t.tier_id) LIKE '%idgloss%' "
        "OR LOWER(t.tier_id) LIKE '%id-gloss%'"
        "))"
    )

    @staticmethod
    def _build_source_filter_sql(
        source: str | None,
    ) -> tuple[str, list[Any]]:
        """Build SQL WHERE clause fragment for annotation source filtering.

        Filters segments to only those belonging to files tagged as *source*
        (``'human'`` or ``'model'``).  Uses the same heuristic fallback as
        :meth:`get_file_sources`: stems ending with ``_segments`` or
        ``_spotted`` default to ``'model'``; everything else to ``'human'``.

        When *source* is ``None``, no filtering is applied.
        """
        if not source:
            return ("", [])
        clause = (
            "AND s.file_id IN ("
            "SELECT f.id FROM files f "
            "LEFT JOIN file_metadata fm "
            "  ON fm.file_id = f.id AND fm.key = 'annotation_source' "
            "WHERE COALESCE(fm.value, "
            "  CASE WHEN f.stem LIKE '%_segments' "
            "       OR f.stem LIKE '%_spotted' "
            "  THEN 'model' ELSE 'human' END"
            ") = ?)"
        )
        return (clause, [source])

    @classmethod
    def _build_tier_filter_sql(
        cls,
        tier_roles: list[str] | None,
    ) -> tuple[str, list[Any]]:
        """Build SQL WHERE clause fragment for tier role filtering.

        Default (None): restricts to gloss tiers only, identified by
        name-pattern matching (tier_id containing 'gloss', 'idgloss',
        or 'id-gloss') OR an explicit 'gloss' assignment in the
        tier_roles table.

        When *tier_roles* is provided explicitly, uses the tier_roles
        table to match tiers by their assigned role.
        """
        if tier_roles is None:
            return (f"AND s.tier_rowid IN ({cls._GLOSS_TIER_SUBQUERY})", [])
        placeholders = ", ".join("?" for _ in tier_roles)
        clause = (
            f"AND s.tier_rowid IN ("
            f"SELECT t.id FROM tiers t "
            f"LEFT JOIN tier_roles tr ON t.tier_id = tr.tier_id_pattern "
            f"WHERE COALESCE(tr.role, 'other') IN ({placeholders})"
            f")"
        )
        return (clause, list(tier_roles))

    def _filtered_file_ids(
        self, conn: sqlite3.Connection, metadata_filter: dict[str, str] | None
    ) -> list[int] | None:
        """Return list of file IDs matching metadata filter, or None if no filter."""
        if not metadata_filter:
            return None
        # Start with all files, intersect for each key/value
        result_set: set[int] | None = None
        for key, value in metadata_filter.items():
            rows = conn.execute(
                "SELECT file_id FROM file_metadata WHERE key = ? AND value = ?",
                (key, value),
            ).fetchall()
            ids = {r["file_id"] for r in rows}
            result_set = ids if result_set is None else result_set & ids
        return list(result_set) if result_set is not None else []

    def _tier_role_clause_for_iter(
        self, conn: sqlite3.Connection | None, tier_roles: list[str] | None
    ) -> str:
        """Return a SQL fragment for filtering segments by tier role, for use in iteration queries."""
        if tier_roles is None:
            return f"AND s.tier_rowid IN ({self._GLOSS_TIER_SUBQUERY})"
        placeholders = ", ".join(f"'{r}'" for r in tier_roles)
        return (
            f"AND s.tier_rowid IN ("
            f"SELECT t.id FROM tiers t "
            f"LEFT JOIN tier_roles tr ON t.tier_id = tr.tier_id_pattern "
            f"WHERE COALESCE(tr.role, 'other') IN ({placeholders})"
            f")"
        )

    # ------------------------------------------------------------------
    # Substring extraction
    # ------------------------------------------------------------------

    def preview_substring_extraction(
        self,
        segment_index: int,
        definitions: list[dict[str, Any]],
    ) -> dict[str, Any]:
        """Preview character-level substring extraction from path segments.

        definitions: [{key: str, start: int, end: int}]
        """
        with self._rlock:
            rows = self._get_conn().execute(Q.FILE_PATHS_WITH_IDS).fetchall()

        files: list[dict[str, Any]] = []
        unique_per_key: dict[str, set[str]] = {d["key"]: set() for d in definitions}
        empty_count = 0

        for r in rows:
            path = r["path"]
            if path.startswith("synthetic://"):
                continue
            segment_val = self._extract_path_segment(path, segment_index)
            fields: dict[str, str | None] = {}
            any_empty = False
            for d in definitions:
                if segment_val and d["start"] < len(segment_val) and d["end"] <= len(segment_val):
                    extracted = segment_val[d["start"] : d["end"]]
                    fields[d["key"]] = extracted
                    unique_per_key[d["key"]].add(extracted)
                else:
                    fields[d["key"]] = None
                    any_empty = True
            if any_empty:
                empty_count += 1
            files.append(
                {
                    "file_id": r["id"],
                    "path": path,
                    "segment_value": segment_val,
                    "fields": fields,
                }
            )

        return {
            "files": files,
            "unique_per_key": {k: sorted(v) for k, v in unique_per_key.items()},
            "total_files": len(files),
            "empty_count": empty_count,
        }

    def apply_substring_extraction(
        self,
        segment_index: int,
        definitions: list[dict[str, Any]],
        set_participant_key: str | None = None,
    ) -> dict[str, Any]:
        """Apply character-level substring extraction and store as file_metadata."""
        with self._wlock:
            conn = self._get_write_conn()
            rows = conn.execute(Q.FILE_PATHS_WITH_IDS).fetchall()

            applied_files = 0
            fields_created = 0
            participant_updated = 0

            for r in rows:
                path = r["path"]
                file_id = r["id"]
                if path.startswith("synthetic://"):
                    continue
                segment_val = self._extract_path_segment(path, segment_index)
                if not segment_val:
                    continue

                file_had_data = False
                for d in definitions:
                    if d["start"] < len(segment_val) and d["end"] <= len(segment_val):
                        extracted = segment_val[d["start"] : d["end"]]
                        conn.execute(
                            Q.INSERT_FILE_METADATA,
                            (file_id, d["key"], extracted, "path_extraction"),
                        )
                        fields_created += 1
                        file_had_data = True

                        # Optionally set participant from a specific field
                        if set_participant_key and d["key"] == set_participant_key:
                            cur = conn.execute(
                                Q.UPDATE_TIER_PARTICIPANT_EMPTY,
                                (extracted, file_id),
                            )
                            participant_updated += cur.rowcount

                if file_had_data:
                    applied_files += 1

            conn.execute(
                "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
                ("last_updated", datetime.now(timezone.utc).isoformat()),
            )
            conn.commit()

        return {
            "applied_files": applied_files,
            "fields_created": fields_created,
            "participant_updated": participant_updated,
        }

    def apply_segment_as_metadata(
        self,
        segment_index: int,
        field_name: str,
        set_participant_key: bool = False,
    ) -> dict[str, Any]:
        """Store the whole path segment value as file metadata.

        Unlike substring extraction which slices characters, this uses the
        entire segment value at the given path position.  Ideal for directory
        segments where the value varies across files (e.g. region names).
        """
        with self._wlock:
            conn = self._get_write_conn()
            rows = conn.execute(Q.FILE_PATHS_WITH_IDS).fetchall()

            applied_files = 0
            participant_updated = 0

            for r in rows:
                path = r["path"]
                file_id = r["id"]
                if path.startswith("synthetic://"):
                    continue
                segment_val = self._extract_path_segment(path, segment_index)
                if not segment_val:
                    continue

                conn.execute(
                    Q.INSERT_FILE_METADATA,
                    (file_id, field_name, segment_val, "path_extraction"),
                )
                applied_files += 1

                if set_participant_key:
                    cur = conn.execute(
                        Q.UPDATE_TIER_PARTICIPANT_EMPTY,
                        (segment_val, file_id),
                    )
                    participant_updated += cur.rowcount

            conn.execute(
                "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
                ("last_updated", datetime.now(timezone.utc).isoformat()),
            )
            conn.commit()

        return {
            "applied_files": applied_files,
            "fields_created": applied_files,  # one field per file
            "participant_updated": participant_updated,
        }

    def preview_delimiter_extraction(
        self,
        segment_index: int,
        delimiter: str,
        assignments: list[dict[str, Any]],
    ) -> dict[str, Any]:
        """Preview delimiter-based token extraction from path segments.

        assignments: [{token_index: int, field_name: str}]
        """
        with self._rlock:
            rows = self._get_conn().execute(Q.FILE_PATHS_WITH_IDS).fetchall()

        files: list[dict[str, Any]] = []
        unique_per_key: dict[str, set[str]] = {a["field_name"]: set() for a in assignments}
        empty_count = 0

        for r in rows:
            path = r["path"]
            if path.startswith("synthetic://"):
                continue
            segment_val = self._extract_path_segment(path, segment_index)
            tokens = segment_val.split(delimiter) if segment_val else []
            fields: dict[str, str | None] = {}
            any_empty = False
            for a in assignments:
                idx = a["token_index"]
                if idx < len(tokens):
                    val = tokens[idx]
                    fields[a["field_name"]] = val
                    unique_per_key[a["field_name"]].add(val)
                else:
                    fields[a["field_name"]] = None
                    any_empty = True
            if any_empty:
                empty_count += 1
            files.append(
                {
                    "file_id": r["id"],
                    "path": path,
                    "segment_value": segment_val,
                    "fields": fields,
                }
            )

        return {
            "files": files,
            "unique_per_key": {k: sorted(v) for k, v in unique_per_key.items()},
            "total_files": len(files),
            "empty_count": empty_count,
        }

    def apply_delimiter_extraction(
        self,
        segment_index: int,
        delimiter: str,
        assignments: list[dict[str, Any]],
        set_participant_key: str | None = None,
    ) -> dict[str, Any]:
        """Apply delimiter-based token extraction and store as file_metadata."""
        with self._wlock:
            conn = self._get_write_conn()
            rows = conn.execute(Q.FILE_PATHS_WITH_IDS).fetchall()

            applied_files = 0
            fields_created = 0
            participant_updated = 0

            for r in rows:
                path = r["path"]
                file_id = r["id"]
                if path.startswith("synthetic://"):
                    continue
                segment_val = self._extract_path_segment(path, segment_index)
                if not segment_val:
                    continue

                tokens = segment_val.split(delimiter)
                file_had_data = False
                for a in assignments:
                    idx = a["token_index"]
                    if idx < len(tokens):
                        extracted = tokens[idx]
                        conn.execute(
                            Q.INSERT_FILE_METADATA,
                            (file_id, a["field_name"], extracted, "delimiter_extraction"),
                        )
                        fields_created += 1
                        file_had_data = True

                        if set_participant_key and a["field_name"] == set_participant_key:
                            cur = conn.execute(
                                Q.UPDATE_TIER_PARTICIPANT_EMPTY,
                                (extracted, file_id),
                            )
                            participant_updated += cur.rowcount

                if file_had_data:
                    applied_files += 1

            conn.execute(
                "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
                ("last_updated", datetime.now(timezone.utc).isoformat()),
            )
            conn.commit()

        return {
            "applied_files": applied_files,
            "fields_created": fields_created,
            "participant_updated": participant_updated,
        }

    # ------------------------------------------------------------------
    # Universal search
    # ------------------------------------------------------------------

    def universal_search(
        self,
        query: str,
        limit: int = 20,
        tier_id: str | None = None,
    ) -> dict[str, Any]:
        """Search files, glosses, signers, and metadata in one call.

        When *tier_id* is given, gloss results are restricted to that tier.

        Returns ``{"files": [...], "glosses": [...], "signers": [...], "metadata": [...]}``.
        """
        like_pat = f"%{query}%"
        with self._rlock:
            conn = self._get_conn()

            # 1) Files
            file_rows = conn.execute(
                Q.SEARCH_FILES, (like_pat, like_pat, like_pat, limit)
            ).fetchall()
            files = []
            for r in file_rows:
                signers_str = r["signers"]
                signers_list = (
                    [s.strip() for s in signers_str.split(",") if s.strip()] if signers_str else []
                )
                files.append(
                    {
                        "file_id": r["id"],
                        "path": r["path"],
                        "name": r["name"],
                        "stem": r["stem"],
                        "segment_count": r["segment_count"],
                        "signers": signers_list,
                    }
                )

            # Build optional tier filter clause for gloss queries
            tier_join = ""
            tier_where = ""
            tier_params: list[Any] = []
            if tier_id:
                tier_join = "JOIN tiers t ON s.tier_rowid = t.id "
                tier_where = "AND t.tier_id = ? "
                tier_params = [tier_id]

            # 2) Glosses -- FTS5 first, then LIKE prefix fallback, deduplicated
            gloss_map: dict[str, dict[str, Any]] = {}
            try:
                fts_query = query.strip().replace('"', '""')
                fts_rows = conn.execute(
                    "SELECT s.label_lower AS gloss, COUNT(*) AS cnt, "
                    "COUNT(DISTINCT s.file_id) AS file_count, "
                    "MIN(f.path) AS sample_path, MIN(f.name) AS sample_name "
                    "FROM segments_fts fts "
                    "JOIN segments s ON fts.rowid = s.id "
                    "JOIN files f ON s.file_id = f.id "
                    f"{tier_join}"
                    f"WHERE segments_fts MATCH ? {tier_where}"
                    "GROUP BY s.label_lower "
                    "ORDER BY cnt DESC LIMIT ?",
                    [f'"{fts_query}"*'] + tier_params + [limit],
                ).fetchall()
                for r in fts_rows:
                    gloss_map[r["gloss"]] = {
                        "gloss": r["gloss"],
                        "count": r["cnt"],
                        "file_count": r["file_count"],
                        "sample_file_path": r["sample_path"],
                        "sample_file_name": r["sample_name"],
                    }
            except Exception:
                pass  # FTS can fail on special characters

            # LIKE fallback for glosses not found by FTS
            if len(gloss_map) < limit:
                like_rows = conn.execute(
                    "SELECT s.label_lower AS gloss, COUNT(*) AS cnt, "
                    "COUNT(DISTINCT s.file_id) AS file_count, "
                    "MIN(f.path) AS sample_path, MIN(f.name) AS sample_name "
                    "FROM segments s "
                    "JOIN files f ON s.file_id = f.id "
                    f"{tier_join}"
                    "WHERE s.label_lower LIKE ? AND s.label IS NOT NULL AND s.label != '' "
                    f"{tier_where}"
                    "GROUP BY s.label_lower "
                    "ORDER BY cnt DESC LIMIT ?",
                    [like_pat.lower()] + tier_params + [limit],
                ).fetchall()
                for r in like_rows:
                    if r["gloss"] not in gloss_map:
                        gloss_map[r["gloss"]] = {
                            "gloss": r["gloss"],
                            "count": r["cnt"],
                            "file_count": r["file_count"],
                            "sample_file_path": r["sample_path"],
                            "sample_file_name": r["sample_name"],
                        }

            glosses = sorted(gloss_map.values(), key=lambda x: x["count"], reverse=True)[:limit]

            # 3) Signers
            signer_rows = conn.execute(Q.SEARCH_SIGNERS, (like_pat, limit)).fetchall()
            signers = [
                {
                    "participant": r["participant"],
                    "file_count": r["file_count"],
                    "segment_count": r["segment_count"],
                }
                for r in signer_rows
            ]

            # 4) Metadata
            meta_rows = conn.execute(Q.SEARCH_METADATA, (like_pat, limit)).fetchall()
            metadata = [
                {
                    "key": r["key"],
                    "value": r["value"],
                    "file_count": r["file_count"],
                    "source": r["source"],
                }
                for r in meta_rows
            ]

        return {
            "files": files,
            "glosses": glosses,
            "signers": signers,
            "metadata": metadata,
        }

    # ------------------------------------------------------------------
    # Available tiers (for cross-tier tools)
    # ------------------------------------------------------------------

    # Priority-ordered auto-detection rules for tier roles.
    # Each entry: (role, patterns, match_mode)
    # match_mode: "substring" for safe patterns, "word" for short ambiguous ones
    # Order matters — first match wins. Gloss must come before hand roles
    # so that "RH-IDgloss" → gloss (not right_hand).
    _AUTO_DETECT_RULES: list[tuple[str, list[str], str]] = [
        ("gloss", ["gloss", "idgloss", "id-gloss"], "substring"),
        ("translation", ["transl", "free translation"], "substring"),
        (
            "nonmanual",
            [
                "mouth",
                "mouthgest",
                "mouthing",
                "head",
                "eye",
                "face",
                "body",
                "gaze",
                "brow",
                "cheek",
                "n-manual",
                "n-nonmanual",
            ],
            "substring",
        ),
        ("constructed_action", ["constructed action"], "substring"),
        ("constructed_action", ["ca"], "word"),
        ("clause", ["clause"], "substring"),
        ("clause", ["clu"], "word"),
        (
            "discourse",
            [
                "turn",
                "question",
                "backchannel",
                "back channel",
                "discourse",
            ],
            "substring",
        ),
        ("left_hand", ["lh-", "left hand"], "substring"),
        ("left_hand", ["lh"], "word"),
        ("right_hand", ["rh-", "right hand"], "substring"),
        ("right_hand", ["rh"], "word"),
        ("boundary", ["bound"], "substring"),
        ("metadata", ["comment", "interview", "stimulus", "theme", "note"], "substring"),
    ]

    @classmethod
    def _detect_tier_role(cls, tier_id: str) -> str:
        """Detect the most likely role for a tier based on its ID.

        Uses priority-ordered pattern matching. First match wins.
        Returns ``"other"`` if no pattern matches.
        """
        lower = tier_id.lower()
        for role, patterns, mode in cls._AUTO_DETECT_RULES:
            if mode == "substring":
                if any(p in lower for p in patterns):
                    return role
            else:  # "word" — word-boundary match
                for p in patterns:
                    if re.search(rf"(?:^|[^a-z]){re.escape(p)}(?:$|[^a-z])", lower):
                        return role
        return "other"

    def auto_detect_tier_roles(self) -> list[dict[str, Any]]:
        """Run auto-detection on all unique tier IDs.

        Returns a list of dicts with tier info, ``suggested_role``,
        and ``current_role`` (from the tier_roles table, or ``None``).
        """
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    "SELECT t.tier_id, t.linguistic_type, "
                    "COUNT(s.id) AS segment_count, "
                    "COUNT(DISTINCT t.file_id) AS file_count, "
                    "tr.role AS current_role "
                    "FROM tiers t "
                    "LEFT JOIN tier_roles tr ON t.tier_id = tr.tier_id_pattern "
                    "LEFT JOIN segments s ON s.tier_rowid = t.id "
                    "GROUP BY t.tier_id "
                    "ORDER BY segment_count DESC"
                )
                .fetchall()
            )

        return [
            {
                "tier_id": r["tier_id"],
                "linguistic_type": r["linguistic_type"],
                "segment_count": r["segment_count"],
                "file_count": r["file_count"],
                "suggested_role": self._detect_tier_role(r["tier_id"]),
                "current_role": r["current_role"],
            }
            for r in rows
        ]

    def available_tiers(self) -> list[dict[str, Any]]:
        """Return non-empty tiers with suggested categories based on name patterns."""
        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    "SELECT t.tier_id, t.linguistic_type, "
                    "COUNT(s.id) AS segment_count, "
                    "COUNT(DISTINCT t.file_id) AS file_count "
                    "FROM tiers t "
                    "LEFT JOIN segments s ON s.tier_rowid = t.id "
                    "GROUP BY t.tier_id "
                    "HAVING segment_count > 0 "
                    "ORDER BY segment_count DESC"
                )
                .fetchall()
            )

        return [
            {
                "tier_id": r["tier_id"],
                "linguistic_type": r["linguistic_type"],
                "segment_count": r["segment_count"],
                "file_count": r["file_count"],
                "suggested_category": self._detect_tier_role(r["tier_id"]),
            }
            for r in rows
        ]

    # ------------------------------------------------------------------
    # Temporal overlap primitive (for cross-tier tools)
    # ------------------------------------------------------------------

    def tier_overlap(
        self,
        anchor_tier_ids: list[str],
        target_tier_ids: list[str],
        anchor_label: str | None = None,
        min_overlap_ms: int = 0,
        metadata_filter: dict[str, str] | None = None,
        limit: int = 500,
    ) -> list[dict[str, Any]]:
        """Find temporally overlapping segments across tiers.

        For each segment on *anchor_tier_ids*, find all overlapping
        segments on *target_tier_ids* within the same file.
        """
        if not anchor_tier_ids or not target_tier_ids:
            return []

        a_ph = ", ".join("?" for _ in anchor_tier_ids)
        t_ph = ", ".join("?" for _ in target_tier_ids)

        label_clause = ""
        label_params: list[Any] = []
        if anchor_label:
            label_clause = "AND a.label_lower = lower(?)"
            label_params = [anchor_label]

        file_clause = ""
        file_params: list[Any] = []
        if metadata_filter:
            for key, value in metadata_filter.items():
                file_clause += (
                    " AND a.file_id IN "
                    "(SELECT file_id FROM file_metadata WHERE key = ? AND value = ?)"
                )
                file_params.extend([key, value])

        overlap_clause = ""
        overlap_params: list[Any] = []
        if min_overlap_ms > 0:
            overlap_clause = "AND (MIN(a.end_ms, tgt.end_ms) - MAX(a.start_ms, tgt.start_ms)) >= ?"
            overlap_params = [min_overlap_ms]

        sql = (
            "SELECT a.id AS anchor_id, a.label AS anchor_label, "
            "a.start_ms AS anchor_start, a.end_ms AS anchor_end, "
            "at.tier_id AS anchor_tier, at.participant AS anchor_participant, "
            "f.name AS file_name, f.path AS file_path, "
            "tgt.id AS target_id, tgt.label AS target_label, "
            "tgt.start_ms AS target_start, tgt.end_ms AS target_end, "
            "tt.tier_id AS target_tier, tt.participant AS target_participant "
            "FROM segments a "
            "JOIN tiers at ON a.tier_rowid = at.id "
            "JOIN files f ON a.file_id = f.id "
            "JOIN segments tgt ON tgt.file_id = a.file_id "
            "JOIN tiers tt ON tgt.tier_rowid = tt.id "
            f"WHERE at.tier_id IN ({a_ph}) "
            f"AND tt.tier_id IN ({t_ph}) "
            "AND a.label IS NOT NULL AND a.label != '' "
            "AND tgt.label IS NOT NULL AND tgt.label != '' "
            "AND a.start_ms < tgt.end_ms AND tgt.start_ms < a.end_ms "
            f"{label_clause} {file_clause} {overlap_clause} "
            "ORDER BY f.name, a.start_ms "
            f"LIMIT ?"
        )

        params = (
            anchor_tier_ids
            + target_tier_ids
            + label_params
            + file_params
            + overlap_params
            + [limit]
        )

        with self._rlock:
            rows = self._get_conn().execute(sql, params).fetchall()

        return [dict(r) for r in rows]

    # ------------------------------------------------------------------
    # Multi-tier concordance
    # ------------------------------------------------------------------

    def multi_tier_concordance(
        self,
        target: str,
        anchor_tier_ids: list[str] | None = None,
        display_tier_ids: list[str] | None = None,
        context_window_ms: int = 2000,
        limit: int = 200,
        metadata_filter: dict[str, str] | None = None,
    ) -> list[dict[str, Any]]:
        """Multi-tier KWIC: for each hit of *target*, show overlapping annotations on display tiers."""
        # Default anchor = gloss tiers, display = all non-empty tiers
        cached_tiers = self._cached_available_tiers()
        if anchor_tier_ids is None:
            anchor_tier_ids = [
                t["tier_id"] for t in cached_tiers if t["suggested_category"] == "gloss"
            ]
        if display_tier_ids is None:
            display_tier_ids = [t["tier_id"] for t in cached_tiers]

        if not anchor_tier_ids:
            return []

        a_ph = ", ".join("?" for _ in anchor_tier_ids)
        d_ph = ", ".join("?" for _ in display_tier_ids) if display_tier_ids else "''"

        file_clause = ""
        file_params: list[Any] = []
        if metadata_filter:
            for key, value in metadata_filter.items():
                file_clause += (
                    " AND a.file_id IN "
                    "(SELECT file_id FROM file_metadata WHERE key = ? AND value = ?)"
                )
                file_params.extend([key, value])

        # Single bulk query: anchors + their overlapping segments via JOIN
        sql = (
            "SELECT a.id AS anchor_id, a.label AS anchor_label, "
            "a.start_ms AS anchor_start, a.end_ms AS anchor_end, "
            "a.file_id, at.tier_id AS anchor_tier, at.participant, "
            "f.name AS file_name, f.path AS file_path, "
            "s.label AS ov_label, s.start_ms AS ov_start, s.end_ms AS ov_end, "
            "dt.tier_id AS ov_tier "
            "FROM segments a "
            "JOIN tiers at ON a.tier_rowid = at.id "
            "JOIN files f ON a.file_id = f.id "
            "LEFT JOIN segments s ON s.file_id = a.file_id "
            "  AND s.label IS NOT NULL AND s.label != '' "
            f"  AND s.start_ms < (a.end_ms + ?) AND s.end_ms > (a.start_ms - ?) "
            f"LEFT JOIN tiers dt ON s.tier_rowid = dt.id AND dt.tier_id IN ({d_ph}) "
            f"WHERE at.tier_id IN ({a_ph}) "
            "AND a.label_lower = lower(?) "
            f"{file_clause} "
            "ORDER BY f.name, a.start_ms, dt.tier_id, s.start_ms"
        )

        params = (
            [context_window_ms, context_window_ms]
            + display_tier_ids
            + anchor_tier_ids
            + [target]
            + file_params
        )

        with self._rlock:
            rows = self._get_conn().execute(sql, params).fetchall()

        # Group by anchor_id
        results: list[dict[str, Any]] = []
        current_anchor_id: int | None = None
        current_result: dict[str, Any] | None = None
        count = 0

        for row in rows:
            if row["anchor_id"] != current_anchor_id:
                if current_result is not None:
                    results.append(current_result)
                    count += 1
                    if count >= limit:
                        return results
                current_anchor_id = row["anchor_id"]
                current_result = {
                    "anchor_label": row["anchor_label"],
                    "anchor_tier": row["anchor_tier"],
                    "start_ms": row["anchor_start"],
                    "end_ms": row["anchor_end"],
                    "file_name": row["file_name"],
                    "file_path": row["file_path"],
                    "participant": row["participant"],
                    "overlapping": [],
                }

            # Add overlap if present (LEFT JOIN may produce NULLs)
            assert current_result is not None
            if row["ov_label"] and row["ov_tier"]:
                anchor_dur = row["anchor_end"] - row["anchor_start"]
                ov_start = max(row["anchor_start"], row["ov_start"])
                ov_end = min(row["anchor_end"], row["ov_end"])
                overlap_pct = (
                    round((ov_end - ov_start) / anchor_dur * 100, 1)
                    if anchor_dur > 0 and ov_end > ov_start
                    else 0
                )
                current_result["overlapping"].append(
                    {
                        "tier": row["ov_tier"],
                        "label": row["ov_label"],
                        "start_ms": row["ov_start"],
                        "end_ms": row["ov_end"],
                        "overlap_pct": overlap_pct,
                    }
                )

        if current_result is not None:
            results.append(current_result)

        return results

    # ------------------------------------------------------------------
    # Non-manual feature co-occurrence
    # ------------------------------------------------------------------

    def nonmanual_cooccurrence(
        self,
        manual_tier_ids: list[str] | None = None,
        nonmanual_tier_ids: list[str] | None = None,
        target_sign: str | None = None,
        min_count: int = 5,
        top_n: int = 100,
        metadata_filter: dict[str, str] | None = None,
    ) -> dict[str, Any]:
        """Count non-manual annotations co-occurring with manual signs."""
        tiers = self._cached_available_tiers()
        if manual_tier_ids is None:
            manual_tier_ids = [t["tier_id"] for t in tiers if t["suggested_category"] == "gloss"]
        if nonmanual_tier_ids is None:
            nonmanual_tier_ids = [
                t["tier_id"] for t in tiers if t["suggested_category"] == "nonmanual"
            ]

        if not manual_tier_ids or not nonmanual_tier_ids:
            return {"signs": [], "total_signs_analyzed": 0, "overall_nm_rate": 0}

        m_ph = ", ".join("?" for _ in manual_tier_ids)
        nm_ph = ", ".join("?" for _ in nonmanual_tier_ids)

        file_clause = ""
        file_params: list[Any] = []
        if metadata_filter:
            for key, value in metadata_filter.items():
                file_clause += (
                    " AND m.file_id IN "
                    "(SELECT file_id FROM file_metadata WHERE key = ? AND value = ?)"
                )
                file_params.extend([key, value])

        target_clause = ""
        target_params: list[Any] = []
        if target_sign:
            target_clause = "AND m.label_lower = lower(?)"
            target_params = [target_sign]

        with self._rlock:
            conn = self._get_conn()
            # Get all manual signs with their non-manual overlaps
            sql = (
                "SELECT m.id AS sign_id, m.label_lower AS sign, m.start_ms, m.end_ms, m.file_id, "
                "nm.label AS nm_label, nm.start_ms AS nm_start, nm.end_ms AS nm_end, "
                "nmt.tier_id AS nm_tier "
                "FROM segments m "
                "JOIN tiers mt ON m.tier_rowid = mt.id "
                f"LEFT JOIN segments nm ON nm.file_id = m.file_id "
                f"AND nm.start_ms < m.end_ms AND nm.start_ms >= m.start_ms - 200 "
                f"AND m.start_ms < nm.end_ms "
                f"AND nm.label IS NOT NULL AND nm.label != '' "
                f"LEFT JOIN tiers nmt ON nm.tier_rowid = nmt.id AND nmt.tier_id IN ({nm_ph}) "
                f"WHERE mt.tier_id IN ({m_ph}) "
                "AND m.label IS NOT NULL AND m.label != '' "
                f"{target_clause} {file_clause} "
                "ORDER BY m.label_lower, m.file_id, m.start_ms"
            )
            rows = conn.execute(
                sql,
                nonmanual_tier_ids + manual_tier_ids + target_params + file_params,
            ).fetchall()

        # Aggregate
        seen_sign_ids: dict[str, set[int]] = defaultdict(set)
        sign_with_nm: dict[str, set[int]] = defaultdict(set)
        nm_forms: dict[str, Counter[str]] = defaultdict(Counter)

        for r in rows:
            sign = r["sign"]
            sign_id = r["sign_id"]
            seen_sign_ids[sign].add(sign_id)
            if r["nm_label"] and r["nm_tier"]:
                sign_with_nm[sign].add(sign_id)
                nm_forms[sign][f"{r['nm_tier']}:{r['nm_label']}"] += 1

        results: list[dict[str, Any]] = []
        total_analyzed = 0
        total_with_nm = 0
        for sign, ids in seen_sign_ids.items():
            total = len(ids)
            if total < min_count:
                continue
            with_nm = len(sign_with_nm.get(sign, set()))
            nm_rate = with_nm / total if total > 0 else 0
            forms = [
                {"tier_label": k, "count": v}
                for k, v in nm_forms.get(sign, Counter()).most_common(10)
            ]
            results.append(
                {
                    "sign": sign,
                    "total_count": total,
                    "with_nonmanual": with_nm,
                    "nonmanual_rate": round(nm_rate, 4),
                    "forms": forms,
                }
            )
            total_analyzed += total
            total_with_nm += with_nm

        results.sort(key=lambda x: x["total_count"], reverse=True)
        overall_rate = total_with_nm / total_analyzed if total_analyzed > 0 else 0

        return {
            "signs": results[:top_n],
            "total_signs_analyzed": total_analyzed,
            "overall_nm_rate": round(overall_rate, 4),
        }

    # ------------------------------------------------------------------
    # Cross-tier co-occurrence matrix
    # ------------------------------------------------------------------

    def cross_tier_cooccurrence(
        self,
        row_tier_ids: list[str],
        col_tier_ids: list[str],
        top_n_rows: int = 30,
        top_n_cols: int = 30,
        min_overlap_ms: int = 0,
        metadata_filter: dict[str, str] | None = None,
    ) -> dict[str, Any]:
        """Build an asymmetric co-occurrence matrix from temporal overlap
        between segments on two different tier sets.

        Returns ``row_labels``, ``col_labels``, ``matrix`` (list-of-lists),
        ``row_frequencies``, ``col_frequencies``, ``total_overlaps``.
        """
        if not row_tier_ids or not col_tier_ids:
            return {
                "row_labels": [],
                "col_labels": [],
                "matrix": [],
                "row_frequencies": {},
                "col_frequencies": {},
                "total_overlaps": 0,
                "row_tier_ids": row_tier_ids,
                "col_tier_ids": col_tier_ids,
            }

        # Build optional metadata file filter (uses alias ``r`` for row segs)
        file_clause = ""
        file_params: list[Any] = []
        if metadata_filter:
            for key, value in metadata_filter.items():
                file_clause += (
                    " AND r.file_id IN "
                    "(SELECT file_id FROM file_metadata WHERE key = ? AND value = ?)"
                )
                file_params.extend([key, value])

        row_ph = ", ".join("?" for _ in row_tier_ids)
        col_ph = ", ".join("?" for _ in col_tier_ids)

        with self._rlock:
            conn = self._get_conn()

            # Phase 1: Top N labels per axis
            top_label_sql = (
                "SELECT s.label_lower AS label, COUNT(*) AS freq "
                "FROM segments s JOIN tiers t ON s.tier_rowid = t.id "
                "WHERE t.tier_id IN ({ph}) "
                "AND s.label IS NOT NULL AND s.label != '' "
                "GROUP BY s.label_lower ORDER BY freq DESC LIMIT ?"
            )
            top_rows = conn.execute(
                top_label_sql.format(ph=row_ph),
                row_tier_ids + [top_n_rows],
            ).fetchall()
            top_cols = conn.execute(
                top_label_sql.format(ph=col_ph),
                col_tier_ids + [top_n_cols],
            ).fetchall()

        if not top_rows or not top_cols:
            return {
                "row_labels": [],
                "col_labels": [],
                "matrix": [],
                "row_frequencies": {},
                "col_frequencies": {},
                "total_overlaps": 0,
                "row_tier_ids": row_tier_ids,
                "col_tier_ids": col_tier_ids,
            }

        row_labels = [r["label"] for r in top_rows]
        col_labels = [c["label"] for c in top_cols]
        row_freqs = {r["label"]: r["freq"] for r in top_rows}
        col_freqs = {c["label"]: c["freq"] for c in top_cols}

        rl_ph = ", ".join("?" for _ in row_labels)
        cl_ph = ", ".join("?" for _ in col_labels)

        # Minimum overlap constraint
        overlap_clause = ""
        if min_overlap_ms > 0:
            overlap_clause = f" AND (MIN(r.end_ms, c.end_ms) - MAX(r.start_ms, c.start_ms)) >= {int(min_overlap_ms)}"

        with self._rlock:
            conn = self._get_conn()
            # Phase 2: Temporal-overlap join
            sql = (
                "SELECT r.label_lower AS row_label, c.label_lower AS col_label, COUNT(*) AS cnt "
                "FROM segments r "
                "  JOIN tiers rt ON r.tier_rowid = rt.id "
                "  JOIN segments c ON c.file_id = r.file_id "
                "  JOIN tiers ct ON c.tier_rowid = ct.id "
                f"WHERE rt.tier_id IN ({row_ph}) AND ct.tier_id IN ({col_ph}) "
                f"  AND r.label_lower IN ({rl_ph}) AND c.label_lower IN ({cl_ph}) "
                "  AND r.start_ms < c.end_ms AND c.start_ms < r.end_ms "
                f"  {overlap_clause} {file_clause} "
                "GROUP BY r.label_lower, c.label_lower"
            )
            params = row_tier_ids + col_tier_ids + row_labels + col_labels + file_params
            pair_rows = conn.execute(sql, params).fetchall()

        # Build matrix
        row_idx = {lab: i for i, lab in enumerate(row_labels)}
        col_idx = {lab: i for i, lab in enumerate(col_labels)}
        matrix = [[0] * len(col_labels) for _ in range(len(row_labels))]
        total_overlaps = 0
        for pr in pair_rows:
            ri = row_idx.get(pr["row_label"])
            ci = col_idx.get(pr["col_label"])
            if ri is not None and ci is not None:
                matrix[ri][ci] = pr["cnt"]
                total_overlaps += pr["cnt"]

        return {
            "row_labels": row_labels,
            "col_labels": col_labels,
            "matrix": matrix,
            "row_frequencies": row_freqs,
            "col_frequencies": col_freqs,
            "total_overlaps": total_overlaps,
            "row_tier_ids": row_tier_ids,
            "col_tier_ids": col_tier_ids,
        }

    # ------------------------------------------------------------------
    # Constructed action analysis
    # ------------------------------------------------------------------

    def constructed_action_analysis(
        self,
        ca_tier_ids: list[str] | None = None,
        gloss_tier_ids: list[str] | None = None,
        top_n: int = 50,
        metadata_filter: dict[str, str] | None = None,
    ) -> dict[str, Any]:
        """Compare vocabulary inside vs outside Constructed Action spans."""
        tiers = self._cached_available_tiers()
        if ca_tier_ids is None:
            ca_tier_ids = [
                t["tier_id"] for t in tiers if t["suggested_category"] == "constructed_action"
            ]
        if gloss_tier_ids is None:
            gloss_tier_ids = [t["tier_id"] for t in tiers if t["suggested_category"] == "gloss"]

        if not ca_tier_ids or not gloss_tier_ids:
            return {
                "ca_spans": 0,
                "files_with_ca": 0,
                "total_ca_duration_ms": 0,
                "inside": {
                    "total_tokens": 0,
                    "vocabulary_size": 0,
                    "top_signs": [],
                    "avg_sign_duration_ms": 0,
                },
                "outside": {
                    "total_tokens": 0,
                    "vocabulary_size": 0,
                    "top_signs": [],
                    "avg_sign_duration_ms": 0,
                },
                "distinctive_signs": [],
            }

        ca_ph = ", ".join("?" for _ in ca_tier_ids)
        gl_ph = ", ".join("?" for _ in gloss_tier_ids)

        file_clause = ""
        file_params: list[Any] = []
        if metadata_filter:
            for key, value in metadata_filter.items():
                file_clause += (
                    " AND g.file_id IN "
                    "(SELECT file_id FROM file_metadata WHERE key = ? AND value = ?)"
                )
                file_params.extend([key, value])

        with self._rlock:
            conn = self._get_conn()

            # CA span summary (lightweight aggregate query)
            ca_summary = conn.execute(
                f"SELECT COUNT(*) AS ca_spans, "
                f"COUNT(DISTINCT s.file_id) AS files_with_ca, "
                f"COALESCE(SUM(s.end_ms - s.start_ms), 0) AS total_ca_dur "
                f"FROM segments s JOIN tiers t ON s.tier_rowid = t.id "
                f"WHERE t.tier_id IN ({ca_ph}) "
                f"AND s.label IS NOT NULL AND s.label != ''",
                ca_tier_ids,
            ).fetchone()

            # Glosses INSIDE CA — SQL JOIN with midpoint containment
            inside_sql = (
                f"SELECT g.label_lower AS gloss, COUNT(*) AS freq, "
                f"AVG(g.duration_ms) AS avg_dur "
                f"FROM segments g "
                f"JOIN tiers gt ON g.tier_rowid = gt.id "
                f"WHERE gt.tier_id IN ({gl_ph}) "
                f"AND g.label IS NOT NULL AND g.label != '' AND g.duration_ms > 0 "
                f"{file_clause} "
                f"AND EXISTS ("
                f"  SELECT 1 FROM segments ca "
                f"  JOIN tiers ct ON ca.tier_rowid = ct.id "
                f"  WHERE ct.tier_id IN ({ca_ph}) "
                f"  AND ca.file_id = g.file_id "
                f"  AND ca.label IS NOT NULL AND ca.label != '' "
                f"  AND ca.start_ms <= (g.start_ms + g.end_ms) / 2 "
                f"  AND ca.end_ms >= (g.start_ms + g.end_ms) / 2"
                f") "
                f"GROUP BY g.label_lower ORDER BY freq DESC"
            )
            inside_rows = conn.execute(
                inside_sql, gloss_tier_ids + file_params + ca_tier_ids
            ).fetchall()

            # Glosses OUTSIDE CA — NOT EXISTS
            outside_sql = (
                f"SELECT g.label_lower AS gloss, COUNT(*) AS freq, "
                f"AVG(g.duration_ms) AS avg_dur "
                f"FROM segments g "
                f"JOIN tiers gt ON g.tier_rowid = gt.id "
                f"WHERE gt.tier_id IN ({gl_ph}) "
                f"AND g.label IS NOT NULL AND g.label != '' AND g.duration_ms > 0 "
                f"{file_clause} "
                f"AND NOT EXISTS ("
                f"  SELECT 1 FROM segments ca "
                f"  JOIN tiers ct ON ca.tier_rowid = ct.id "
                f"  WHERE ct.tier_id IN ({ca_ph}) "
                f"  AND ca.file_id = g.file_id "
                f"  AND ca.label IS NOT NULL AND ca.label != '' "
                f"  AND ca.start_ms <= (g.start_ms + g.end_ms) / 2 "
                f"  AND ca.end_ms >= (g.start_ms + g.end_ms) / 2"
                f") "
                f"GROUP BY g.label_lower ORDER BY freq DESC"
            )
            outside_rows = conn.execute(
                outside_sql, gloss_tier_ids + file_params + ca_tier_ids
            ).fetchall()

        inside_counter = {r["gloss"]: r["freq"] for r in inside_rows}
        outside_counter = {r["gloss"]: r["freq"] for r in outside_rows}
        inside_total_dur = sum(r["avg_dur"] * r["freq"] for r in inside_rows if r["avg_dur"])
        inside_total_count = sum(r["freq"] for r in inside_rows)
        outside_total_dur = sum(r["avg_dur"] * r["freq"] for r in outside_rows if r["avg_dur"])
        outside_total_count = sum(r["freq"] for r in outside_rows)

        def _vocab_summary(counter: dict[str, int], total_dur: float, total_count: int) -> dict:
            return {
                "total_tokens": sum(counter.values()),
                "vocabulary_size": len(counter),
                "top_signs": [
                    {"sign": s, "freq": f}
                    for s, f in sorted(counter.items(), key=lambda x: x[1], reverse=True)[:top_n]
                ],
                "avg_sign_duration_ms": round(total_dur / total_count, 1) if total_count else 0,
            }

        # Distinctive signs: ratio of inside_rate / outside_rate
        total_inside = sum(inside_counter.values()) or 1
        total_outside = sum(outside_counter.values()) or 1
        all_signs = set(inside_counter) | set(outside_counter)
        distinctive: list[dict[str, Any]] = []
        for sign in all_signs:
            in_freq = inside_counter.get(sign, 0)
            out_freq = outside_counter.get(sign, 0)
            in_rate = in_freq / total_inside
            out_rate = out_freq / total_outside
            if in_rate > 0 and out_rate > 0:
                ratio = in_rate / out_rate
            elif in_rate > 0:
                ratio = 10.0  # CA-only
            else:
                ratio = 0.1  # non-CA-only
            if ratio >= 2.0 or ratio <= 0.5:
                distinctive.append(
                    {
                        "sign": sign,
                        "inside_freq": in_freq,
                        "outside_freq": out_freq,
                        "ratio": round(ratio, 3),
                        "direction": "ca_preferred" if ratio >= 2.0 else "non_ca_preferred",
                    }
                )
        distinctive.sort(key=lambda x: abs(math.log(max(x["ratio"], 0.01))), reverse=True)

        return {
            "ca_spans": ca_summary["ca_spans"],
            "files_with_ca": ca_summary["files_with_ca"],
            "total_ca_duration_ms": ca_summary["total_ca_dur"],
            "inside": _vocab_summary(inside_counter, inside_total_dur, inside_total_count),
            "outside": _vocab_summary(outside_counter, outside_total_dur, outside_total_count),
            "distinctive_signs": distinctive[:top_n],
        }

    # ------------------------------------------------------------------
    # Clause structure analysis
    # ------------------------------------------------------------------

    def clause_structure_analysis(
        self,
        clause_tier_ids: list[str] | None = None,
        gloss_tier_ids: list[str] | None = None,
        top_n: int = 50,
        metadata_filter: dict[str, str] | None = None,
    ) -> dict[str, Any]:
        """Analyze internal structure of different clause types."""
        tiers = self._cached_available_tiers()
        if clause_tier_ids is None:
            clause_tier_ids = [t["tier_id"] for t in tiers if t["suggested_category"] == "clause"]
        if gloss_tier_ids is None:
            gloss_tier_ids = [t["tier_id"] for t in tiers if t["suggested_category"] == "gloss"]

        if not clause_tier_ids or not gloss_tier_ids:
            return {"clause_types": [], "total_clauses": 0, "avg_signs_per_clause": 0}

        cl_ph = ", ".join("?" for _ in clause_tier_ids)
        gl_ph = ", ".join("?" for _ in gloss_tier_ids)

        file_clause = ""
        file_params: list[Any] = []
        if metadata_filter:
            for key, value in metadata_filter.items():
                file_clause += (
                    " AND c.file_id IN "
                    "(SELECT file_id FROM file_metadata WHERE key = ? AND value = ?)"
                )
                file_params.extend([key, value])

        with self._rlock:
            conn = self._get_conn()

            # Clause summary stats via SQL aggregate
            summary_sql = (
                f"SELECT TRIM(c.label) AS clause_type, ct.tier_id, "
                f"COUNT(*) AS cnt, "
                f"AVG(c.duration_ms) AS avg_dur "
                f"FROM segments c JOIN tiers ct ON c.tier_rowid = ct.id "
                f"WHERE ct.tier_id IN ({cl_ph}) "
                f"AND c.label IS NOT NULL AND c.label != '' "
                f"{file_clause} "
                f"GROUP BY TRIM(c.label), ct.tier_id "
                f"ORDER BY cnt DESC"
            )
            summary_rows = conn.execute(summary_sql, clause_tier_ids + file_params).fetchall()

            # Signs contained in clauses — SQL JOIN with containment
            contained_sql = (
                f"SELECT TRIM(c.label) AS clause_type, g.label_lower AS gloss, "
                f"COUNT(*) AS freq, c.id AS clause_id "
                f"FROM segments c "
                f"JOIN tiers ct ON c.tier_rowid = ct.id "
                f"JOIN segments g ON g.file_id = c.file_id "
                f"JOIN tiers gt ON g.tier_rowid = gt.id "
                f"WHERE ct.tier_id IN ({cl_ph}) "
                f"AND c.label IS NOT NULL AND c.label != '' "
                f"{file_clause} "
                f"AND gt.tier_id IN ({gl_ph}) "
                f"AND g.label IS NOT NULL AND g.label != '' "
                f"AND g.start_ms >= c.start_ms AND g.end_ms <= c.end_ms "
                f"GROUP BY TRIM(c.label), g.label_lower "
                f"ORDER BY TRIM(c.label), freq DESC"
            )
            contained_rows = conn.execute(
                contained_sql,
                clause_tier_ids + file_params + gloss_tier_ids,
            ).fetchall()

            # Per-clause sign counts for avg_signs_per_clause
            signs_per_clause_sql = (
                f"SELECT TRIM(c.label) AS clause_type, c.id AS clause_id, "
                f"COUNT(g.id) AS sign_count "
                f"FROM segments c "
                f"JOIN tiers ct ON c.tier_rowid = ct.id "
                f"LEFT JOIN segments g ON g.file_id = c.file_id "
                f"  AND g.tier_rowid IN (SELECT t.id FROM tiers t WHERE t.tier_id IN ({gl_ph})) "
                f"  AND g.label IS NOT NULL AND g.label != '' "
                f"  AND g.start_ms >= c.start_ms AND g.end_ms <= c.end_ms "
                f"WHERE ct.tier_id IN ({cl_ph}) "
                f"AND c.label IS NOT NULL AND c.label != '' "
                f"{file_clause} "
                f"GROUP BY c.id"
            )
            spc_rows = conn.execute(
                signs_per_clause_sql,
                gloss_tier_ids + clause_tier_ids + file_params,
            ).fetchall()

        # Build per-type aggregates from SQL results
        type_data: dict[str, dict[str, Any]] = {}
        for r in summary_rows:
            ctype = r["clause_type"]
            type_data[ctype] = {
                "type": ctype,
                "tier": r["tier_id"],
                "count": r["cnt"],
                "avg_duration_ms": round(r["avg_dur"], 1) if r["avg_dur"] else 0,
                "sign_counter": Counter(),
                "signs_per_clause_list": [],
            }

        # Populate sign counters from contained_rows
        for r in contained_rows:
            ctype = r["clause_type"]
            if ctype in type_data:
                type_data[ctype]["sign_counter"][r["gloss"]] += r["freq"]

        # Populate signs per clause from spc_rows
        spc_by_type: dict[str, list[int]] = defaultdict(list)
        for r in spc_rows:
            ctype = r["clause_type"]
            spc_by_type[ctype].append(r["sign_count"])

        total_clauses = 0
        total_signs_in_clauses = 0
        clause_types: list[dict[str, Any]] = []
        for ctype, td in type_data.items():
            spc_list = spc_by_type.get(ctype, [])
            total_spc = sum(spc_list)
            total_clauses += td["count"]
            total_signs_in_clauses += total_spc
            clause_types.append(
                {
                    "type": td["type"],
                    "tier": td["tier"],
                    "count": td["count"],
                    "avg_duration_ms": td["avg_duration_ms"],
                    "avg_signs_per_clause": round(total_spc / len(spc_list), 2) if spc_list else 0,
                    "vocabulary_size": len(td["sign_counter"]),
                    "top_signs": [
                        {"sign": s, "freq": f} for s, f in td["sign_counter"].most_common(top_n)
                    ],
                }
            )
        clause_types.sort(key=lambda x: x["count"], reverse=True)

        avg_spc = total_signs_in_clauses / total_clauses if total_clauses > 0 else 0

        return {
            "clause_types": clause_types,
            "total_clauses": total_clauses,
            "avg_signs_per_clause": round(avg_spc, 2),
        }

    # ------------------------------------------------------------------
    # Semantic similarity (replaces minimal pairs)
    # ------------------------------------------------------------------

    def semantic_similarity(
        self,
        top_n: int = 200,
        min_count: int = 5,
        metadata_filter: dict[str, str] | None = None,
        tier_roles: list[str] | None = None,
    ) -> dict[str, Any]:
        """Find semantically related glosses using WordNet + distributional similarity."""
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)
        tier_clause, tier_params = self._build_tier_filter_sql(tier_roles)

        with self._rlock:
            rows = (
                self._get_conn()
                .execute(
                    f"SELECT s.label_lower AS gloss, COUNT(*) AS freq "
                    f"FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' {file_clause} {tier_clause} "
                    f"GROUP BY s.label_lower HAVING freq >= ? "
                    f"ORDER BY freq DESC LIMIT ?",
                    file_params + tier_params + [min_count, top_n],
                )
                .fetchall()
            )

        glosses = {r["gloss"]: r["freq"] for r in rows}
        if not glosses:
            return {"clusters": [], "pairs": [], "wordnet_matched": 0, "unmatched": 0}

        # Try WordNet
        clusters: list[dict[str, Any]] = []
        pairs: list[dict[str, Any]] = []
        matched_glosses: set[str] = set()

        try:
            import nltk
            from nltk.corpus import wordnet as wn

            # Ensure WordNet data available
            try:
                wn.synsets("test")
            except LookupError:
                nltk.download("wordnet", quiet=True)
                nltk.download("omw-1.4", quiet=True)

            # Map glosses to synsets
            gloss_synsets: dict[str, list] = {}
            for g in glosses:
                syns = wn.synsets(g) or wn.synsets(g.upper()) or wn.synsets(g.replace("-", "_"))
                if syns:
                    gloss_synsets[g] = syns[:2]  # keep top 2 senses only
                    matched_glosses.add(g)

            # Cluster by shared hypernyms
            hypernym_groups: dict[str, list[str]] = defaultdict(list)
            for g, syns in gloss_synsets.items():
                for syn in syns:
                    for hyp in syn.hypernyms()[:1]:
                        hypernym_groups[hyp.name()].append(g)

            for hyp_name, members in hypernym_groups.items():
                if len(members) >= 2:
                    clusters.append(
                        {
                            "category": hyp_name.split(".")[0].replace("_", " "),
                            "glosses": sorted(set(members)),
                            "synset_name": hyp_name,
                        }
                    )

            # Wu-Palmer similarity — primary sense only, cap at 60 glosses
            _WP_CAP = 60
            wp_glosses = list(gloss_synsets.keys())[:_WP_CAP]
            # Use only primary (first) synset per gloss for speed
            primary = {g: syns[0] for g, syns in gloss_synsets.items() if g in wp_glosses[:_WP_CAP]}
            for i in range(len(wp_glosses)):
                g1 = wp_glosses[i]
                s1 = primary.get(g1)
                if s1 is None:
                    continue
                s1_hyps = set(s1.hypernyms())
                for j in range(i + 1, len(wp_glosses)):
                    g2 = wp_glosses[j]
                    s2 = primary.get(g2)
                    if s2 is None:
                        continue
                    sim = s1.wup_similarity(s2)
                    if sim and sim >= 0.5:
                        if s1 == s2:
                            rel = "synonym"
                        elif s2 in s1_hyps or s1 in s2.hypernyms():
                            rel = "hypernym"
                        else:
                            rel = "co-hyponym"
                        pairs.append(
                            {
                                "gloss1": g1,
                                "gloss2": g2,
                                "similarity": round(sim, 3),
                                "relation_type": rel,
                            }
                        )

        except ImportError:
            _logger.warning("nltk not available, falling back to distributional similarity only")

        # Distributional similarity for unmatched glosses (bulk SQL, no N+1)
        unmatched = set(glosses) - matched_glosses
        if unmatched and len(unmatched) >= 2:
            tier_extra = self._tier_role_clause_for_iter(None, tier_roles)
            # Single bulk query ordered by file + time, process per-file in one pass
            with self._rlock:
                conn = self._get_conn()
                all_segs = conn.execute(
                    f"SELECT s.file_id, s.label_lower AS label FROM segments s "
                    f"WHERE s.label IS NOT NULL AND s.label != '' "
                    f"{tier_extra} ORDER BY s.file_id, s.start_ms",
                ).fetchall()

            # Build co-occurrence vectors per file boundary
            cooc: dict[str, Counter[str]] = defaultdict(Counter)
            cur_file: int | None = None
            file_labels: list[str] = []
            for row in all_segs:
                fid = row["file_id"]
                if fid != cur_file:
                    if file_labels:
                        self._build_cooc_window(file_labels, unmatched, cooc)
                    cur_file = fid
                    file_labels = []
                file_labels.append(row["label"])
            if file_labels:
                self._build_cooc_window(file_labels, unmatched, cooc)

            # Cosine similarity — cap at 80 glosses
            _DIST_CAP = 80
            unmatched_list = [g for g in unmatched if g in cooc][:_DIST_CAP]
            for i in range(len(unmatched_list)):
                v1 = cooc[unmatched_list[i]]
                mag1 = math.sqrt(sum(v**2 for v in v1.values()))
                if mag1 == 0:
                    continue
                for j in range(i + 1, len(unmatched_list)):
                    v2 = cooc[unmatched_list[j]]
                    shared_keys = set(v1) & set(v2)
                    if not shared_keys:
                        continue
                    dot = sum(v1[k] * v2[k] for k in shared_keys)
                    mag2 = math.sqrt(sum(v**2 for v in v2.values()))
                    if mag2 > 0:
                        sim = dot / (mag1 * mag2)
                        if sim >= 0.3:
                            pairs.append(
                                {
                                    "gloss1": unmatched_list[i],
                                    "gloss2": unmatched_list[j],
                                    "similarity": round(sim, 3),
                                    "relation_type": "distributional",
                                }
                            )

        pairs.sort(key=lambda x: x["similarity"], reverse=True)

        return {
            "clusters": clusters[:50],
            "pairs": pairs[:100],
            "wordnet_matched": len(matched_glosses),
            "unmatched": len(set(glosses) - matched_glosses),
        }

    @staticmethod
    def _build_cooc_window(
        labels: list[str],
        targets: set[str],
        cooc: dict[str, Counter[str]],
        window: int = 3,
    ) -> None:
        """Build co-occurrence counts for target labels within a sliding window."""
        for i, lab in enumerate(labels):
            if lab in targets:
                lo = max(0, i - window)
                hi = min(len(labels), i + window + 1)
                for j in range(lo, hi):
                    if j != i:
                        cooc[lab][labels[j]] += 1

    # ------------------------------------------------------------------
    # Non-Manual Annotator Support
    # ------------------------------------------------------------------

    def unannotated_segments(
        self,
        nm_tier_ids: list[str] | None = None,
        gloss_filter: str | None = None,
        metadata_filter: dict[str, str] | None = None,
        limit: int = 200,
    ) -> list[dict[str, Any]]:
        """Find gloss-tier segments with NO overlapping NM annotations.

        Returns segments on gloss tiers that have no temporally overlapping
        segment on any non-manual tier, useful for building annotation queues.

        When *nm_tier_ids* is None, auto-detects NM tiers from
        ``available_tiers()`` where ``suggested_category == "nonmanual"``.
        """
        if nm_tier_ids is None:
            cached = self._cached_available_tiers()
            nm_tier_ids = [t["tier_id"] for t in cached if t["suggested_category"] == "nonmanual"]
        if not nm_tier_ids:
            # No NM tiers found — all gloss segments are "unannotated"
            nm_tier_ids = ["__no_nm_tiers__"]

        gloss_sub = self._GLOSS_TIER_SUBQUERY
        file_clause, file_params = self._build_file_filter_sql(metadata_filter)

        label_clause = ""
        label_params: list[Any] = []
        if gloss_filter:
            label_clause = "AND s.label_lower LIKE ?"
            label_params = [f"%{gloss_filter.lower()}%"]

        nm_ph = ", ".join("?" for _ in nm_tier_ids)
        sql = (
            "SELECT s.label, s.start_ms, s.end_ms, "
            "(s.end_ms - s.start_ms) AS duration_ms, "
            "s.annotation_id, t.tier_id, t.participant, "
            "f.path AS file_path, f.name AS file_name, f.stem AS file_stem "
            "FROM segments s "
            "JOIN tiers t ON s.tier_rowid = t.id "
            "JOIN files f ON s.file_id = f.id "
            f"WHERE s.tier_rowid IN ({gloss_sub}) "
            "AND s.label IS NOT NULL AND s.label != '' "
            f"{label_clause} {file_clause} "
            "AND NOT EXISTS ("
            "  SELECT 1 FROM segments nm "
            "  JOIN tiers nmt ON nm.tier_rowid = nmt.id "
            f"  WHERE nmt.tier_id IN ({nm_ph}) "
            "  AND nm.file_id = s.file_id "
            "  AND nm.start_ms < s.end_ms AND nm.end_ms > s.start_ms "
            "  AND nm.label IS NOT NULL AND nm.label != ''"
            ") "
            "ORDER BY f.name, s.start_ms "
            "LIMIT ?"
        )
        params = label_params + file_params + nm_tier_ids + [limit]

        with self._rlock:
            rows = self._get_conn().execute(sql, params).fetchall()

        return [
            {
                "label": r["label"],
                "start_ms": r["start_ms"],
                "end_ms": r["end_ms"],
                "duration_ms": r["duration_ms"],
                "annotation_id": r["annotation_id"],
                "tier": r["tier_id"],
                "participant": r["participant"],
                "file_path": r["file_path"],
                "file_name": r["file_name"],
                "file_stem": r["file_stem"],
            }
            for r in rows
        ]

    def nm_label_vocabulary(
        self,
        nm_tier_ids: list[str] | None = None,
    ) -> dict[str, list[dict[str, Any]]]:
        """Return distinct NM labels already in the corpus, grouped by tier.

        When *nm_tier_ids* is None, auto-detects NM tiers from
        ``available_tiers()`` where ``suggested_category == "nonmanual"``.

        Returns ``{"Head": [{"label": "nod", "count": 42}, ...], ...}``.
        """
        if nm_tier_ids is None:
            cached = self._cached_available_tiers()
            nm_tier_ids = [t["tier_id"] for t in cached if t["suggested_category"] == "nonmanual"]
        if not nm_tier_ids:
            return {}

        nm_ph = ", ".join("?" for _ in nm_tier_ids)
        sql = (
            "SELECT t.tier_id, s.label, COUNT(*) AS cnt "
            "FROM segments s "
            "JOIN tiers t ON s.tier_rowid = t.id "
            f"WHERE t.tier_id IN ({nm_ph}) "
            "AND s.label IS NOT NULL AND s.label != '' "
            "GROUP BY t.tier_id, s.label "
            "ORDER BY t.tier_id, cnt DESC"
        )

        with self._rlock:
            rows = self._get_conn().execute(sql, nm_tier_ids).fetchall()

        result: dict[str, list[dict[str, Any]]] = {}
        for r in rows:
            tier = r["tier_id"]
            if tier not in result:
                result[tier] = []
            result[tier].append({"label": r["label"], "count": r["cnt"]})
        return result

    # ------------------------------------------------------------------
    # Source tagging
    # ------------------------------------------------------------------

    def tag_file_source(self, file_id: int, source: str) -> None:
        """Tag a file's annotation source as human, model, or unknown."""
        if source not in ("human", "model", "unknown"):
            raise ValueError(f"Invalid source {source!r}. Must be human, model, or unknown.")
        with self._wlock:
            conn = self._get_write_conn()
            conn.execute(
                "INSERT OR REPLACE INTO file_metadata "
                "(file_id, key, value, source) "
                "VALUES (?, 'annotation_source', ?, 'system')",
                (file_id, source),
            )
            conn.commit()

    def get_file_sources(self) -> list[dict[str, Any]]:
        """Return all files with their source tags (human/model/unknown).

        Uses heuristic fallback: stems ending with ``_segments`` or
        ``_spotted`` default to ``'model'``; everything else to ``'human'``.
        """
        sql = (
            "SELECT f.id, f.path, f.name, f.stem, "
            "COALESCE(fm.value, "
            "  CASE WHEN f.stem LIKE '%_segments' "
            "       OR f.stem LIKE '%_spotted' "
            "  THEN 'model' ELSE 'human' END"
            ") AS source "
            "FROM files f "
            "LEFT JOIN file_metadata fm "
            "  ON fm.file_id = f.id AND fm.key = 'annotation_source' "
            "ORDER BY f.name"
        )
        with self._rlock:
            rows = self._get_conn().execute(sql).fetchall()
        return [
            {
                "file_id": r["id"],
                "path": r["path"],
                "name": r["name"],
                "stem": r["stem"],
                "source": r["source"],
            }
            for r in rows
        ]

    def files_by_source(self, source: str) -> list[dict[str, Any]]:
        """Return files matching a given source tag.

        Uses the same heuristic fallback as :meth:`get_file_sources`.
        """
        if source not in ("human", "model", "unknown"):
            raise ValueError(f"Invalid source {source!r}. Must be human, model, or unknown.")
        sql = (
            "SELECT f.id, f.path, f.name "
            "FROM files f "
            "LEFT JOIN file_metadata fm "
            "  ON fm.file_id = f.id AND fm.key = 'annotation_source' "
            "WHERE COALESCE(fm.value, "
            "  CASE WHEN f.stem LIKE '%_segments' "
            "       OR f.stem LIKE '%_spotted' "
            "  THEN 'model' ELSE 'human' END"
            ") = ? "
            "ORDER BY f.name"
        )
        with self._rlock:
            rows = self._get_conn().execute(sql, (source,)).fetchall()
        return [
            {
                "file_id": r["id"],
                "path": r["path"],
                "name": r["name"],
            }
            for r in rows
        ]

    # ------------------------------------------------------------------
    # AI vs Human evaluation
    # ------------------------------------------------------------------

    def _get_source_for_stem(self, stem: str) -> dict[str, Any]:
        """Find human and model files for a video stem.

        Strips ``_segments`` / ``_spotted`` suffixes to match the base
        video stem, then looks for files from each source.
        """
        # Normalise stem: strip AI suffixes
        base = stem
        for suffix in ("_segments", "_spotted"):
            if base.endswith(suffix):
                base = base[: -len(suffix)]

        sql = (
            "SELECT f.id, f.path, f.name, f.stem, f.segment_count, "
            "  COALESCE(fm.value, "
            "    CASE WHEN f.stem LIKE '%_segments' "
            "         OR f.stem LIKE '%_spotted' "
            "    THEN 'model' ELSE 'human' END"
            "  ) AS source "
            "FROM files f "
            "LEFT JOIN file_metadata fm "
            "  ON fm.file_id = f.id AND fm.key = 'annotation_source' "
            "WHERE f.stem = ? "
            "   OR f.stem = ? || '_segments' "
            "   OR f.stem = ? || '_spotted' "
            "   OR f.stem LIKE ? || '%' "
            "ORDER BY source"
        )
        with self._rlock:
            rows = self._get_conn().execute(sql, (base, base, base, base)).fetchall()

        result: dict[str, Any] = {"base_stem": base, "human": None, "model": None}
        for r in rows:
            src = r["source"]
            info = {
                "file_id": r["id"],
                "path": r["path"],
                "name": r["name"],
                "stem": r["stem"],
                "segment_count": r["segment_count"],
            }
            if src == "human" and result["human"] is None:
                result["human"] = info
            elif src == "model" and result["model"] is None:
                result["model"] = info
        return result

    def compare_sources(self, file_stem: str, tolerance_ms: int = 100) -> dict[str, Any]:
        """Compare AI segments against human annotations for a video.

        Uses greedy nearest-neighbour matching: each AI segment is
        matched to the closest human segment (by midpoint) within
        *tolerance_ms*.

        Returns precision, recall, F1, boundary errors, and per-segment
        match details.
        """
        pair = self._get_source_for_stem(file_stem)
        if pair["human"] is None:
            raise ValueError(f"No human annotation found for stem '{file_stem}'")
        if pair["model"] is None:
            raise ValueError(f"No model annotation found for stem '{file_stem}'")

        # Load segments from both files
        def _load_segs(file_id: int) -> list[dict]:
            sql = (
                "SELECT s.start_ms, s.end_ms, s.label "
                "FROM segments s "
                "WHERE s.file_id = ? "
                "ORDER BY s.start_ms"
            )
            with self._rlock:
                rows = self._get_conn().execute(sql, (file_id,)).fetchall()
            return [dict(r) for r in rows]

        human_segs = _load_segs(pair["human"]["file_id"])
        model_segs = _load_segs(pair["model"]["file_id"])

        if not human_segs and not model_segs:
            return {
                "human_file": pair["human"],
                "model_file": pair["model"],
                "precision": 0.0,
                "recall": 0.0,
                "f1": 0.0,
                "mean_start_error_ms": 0.0,
                "mean_end_error_ms": 0.0,
                "over_segmentation_ratio": 0.0,
                "tolerance_ms": tolerance_ms,
                "matched_pairs": [],
                "unmatched_human": human_segs,
                "unmatched_model": model_segs,
            }

        # Greedy nearest-neighbour matching
        human_matched = set()
        matched_pairs = []

        for m_seg in model_segs:
            m_mid = (m_seg["start_ms"] + m_seg["end_ms"]) / 2
            best_idx = None
            best_dist = float("inf")

            for h_idx, h_seg in enumerate(human_segs):
                if h_idx in human_matched:
                    continue
                h_mid = (h_seg["start_ms"] + h_seg["end_ms"]) / 2
                dist = abs(m_mid - h_mid)
                if dist < best_dist:
                    best_dist = dist
                    best_idx = h_idx

            if best_idx is not None:
                h_seg = human_segs[best_idx]
                start_err = abs(m_seg["start_ms"] - h_seg["start_ms"])
                end_err = abs(m_seg["end_ms"] - h_seg["end_ms"])
                if start_err <= tolerance_ms and end_err <= tolerance_ms:
                    human_matched.add(best_idx)
                    matched_pairs.append(
                        {
                            "human": h_seg,
                            "model": m_seg,
                            "start_error_ms": start_err,
                            "end_error_ms": end_err,
                        }
                    )

        n_matched = len(matched_pairs)
        precision = n_matched / len(model_segs) if model_segs else 0.0
        recall = n_matched / len(human_segs) if human_segs else 0.0
        f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0

        mean_start = (
            sum(p["start_error_ms"] for p in matched_pairs) / n_matched if n_matched else 0.0
        )
        mean_end = sum(p["end_error_ms"] for p in matched_pairs) / n_matched if n_matched else 0.0
        over_seg = len(model_segs) / len(human_segs) if human_segs else 0.0

        unmatched_human = [human_segs[i] for i in range(len(human_segs)) if i not in human_matched]
        unmatched_model = [m for m in model_segs if not any(p["model"] is m for p in matched_pairs)]

        return {
            "human_file": pair["human"],
            "model_file": pair["model"],
            "precision": round(precision, 4),
            "recall": round(recall, 4),
            "f1": round(f1, 4),
            "mean_start_error_ms": round(mean_start, 1),
            "mean_end_error_ms": round(mean_end, 1),
            "over_segmentation_ratio": round(over_seg, 2),
            "tolerance_ms": tolerance_ms,
            "matched_pairs": matched_pairs,
            "unmatched_human": unmatched_human,
            "unmatched_model": unmatched_model,
        }

    def batch_compare_sources(self, tolerance_ms: int = 100) -> list[dict[str, Any]]:
        """Compare all AI vs human annotation pairs in the corpus.

        Finds video stems that have both human and model files, then
        runs :meth:`compare_sources` for each.  Returns summary-level
        metrics (no per-segment details).
        """
        # Find all base stems that have BOTH human and model files
        sql = (
            "SELECT f.stem, "
            "  COALESCE(fm.value, "
            "    CASE WHEN f.stem LIKE '%_segments' "
            "         OR f.stem LIKE '%_spotted' "
            "    THEN 'model' ELSE 'human' END"
            "  ) AS source "
            "FROM files f "
            "LEFT JOIN file_metadata fm "
            "  ON fm.file_id = f.id AND fm.key = 'annotation_source'"
        )
        with self._rlock:
            rows = self._get_conn().execute(sql).fetchall()

        # Group by normalised base stem
        stem_sources: dict[str, set[str]] = {}
        for r in rows:
            stem = r["stem"]
            base = stem
            for suffix in ("_segments", "_spotted"):
                if base.endswith(suffix):
                    base = base[: -len(suffix)]
            stem_sources.setdefault(base, set()).add(r["source"])

        # Only process stems with both sources
        results = []
        for base, sources in sorted(stem_sources.items()):
            if "human" not in sources or "model" not in sources:
                continue
            try:
                comp = self.compare_sources(base, tolerance_ms)
                results.append(
                    {
                        "file_stem": base,
                        "precision": comp["precision"],
                        "recall": comp["recall"],
                        "f1": comp["f1"],
                        "human_count": len(comp.get("matched_pairs", []))
                        + len(comp.get("unmatched_human", [])),
                        "model_count": len(comp.get("matched_pairs", []))
                        + len(comp.get("unmatched_model", [])),
                        "mean_boundary_error_ms": round(
                            (comp["mean_start_error_ms"] + comp["mean_end_error_ms"]) / 2,
                            1,
                        ),
                    }
                )
            except ValueError:
                continue

        return results

    def spotting_accuracy(self, file_stem: str, rank_threshold: int = 5) -> dict[str, Any]:
        """Evaluate spotting accuracy against human labels.

        Compares the top-N spotted gloss tiers against human annotation
        labels for segments in the same time range.

        Returns top-1 accuracy, top-N accuracy, and confusion pairs.
        """
        pair = self._get_source_for_stem(file_stem)
        if pair["human"] is None:
            raise ValueError(f"No human annotation found for stem '{file_stem}'")
        if pair["model"] is None:
            raise ValueError(f"No model annotation found for stem '{file_stem}'")

        # Load human segments
        sql_human = (
            "SELECT s.start_ms, s.end_ms, s.label "
            "FROM segments s "
            "WHERE s.file_id = ? AND s.label IS NOT NULL AND s.label != '' "
            "ORDER BY s.start_ms"
        )
        with self._rlock:
            conn = self._get_conn()
            human_rows = conn.execute(sql_human, (pair["human"]["file_id"],)).fetchall()

        # Load model segments (from spotted tiers)
        sql_model = (
            "SELECT t.tier_id, s.start_ms, s.end_ms, s.label "
            "FROM segments s "
            "JOIN tiers t ON t.id = s.tier_rowid "
            "WHERE s.file_id = ? "
            "ORDER BY s.start_ms, t.tier_id"
        )
        with self._rlock:
            model_rows = conn.execute(sql_model, (pair["model"]["file_id"],)).fetchall()

        if not human_rows or not model_rows:
            return {
                "file_stem": file_stem,
                "top_1_accuracy": 0.0,
                "top_5_accuracy": 0.0,
                "total_segments": len(human_rows),
                "confusion_pairs": [],
            }

        # Group model segments by time slot → {(start, end): {tier: label}}
        model_by_time: dict[tuple[int, int], dict[str, str]] = {}
        for r in model_rows:
            key = (r["start_ms"], r["end_ms"])
            model_by_time.setdefault(key, {})[r["tier_id"]] = r["label"]

        top1_correct = 0
        topn_correct = 0
        confusion = []

        for h in human_rows:
            human_label = h["label"].upper().strip()
            h_mid = (h["start_ms"] + h["end_ms"]) / 2

            # Find closest model time slot
            best_key = None
            best_dist = float("inf")
            for ms, me in model_by_time:
                m_mid = (ms + me) / 2
                d = abs(h_mid - m_mid)
                if d < best_dist:
                    best_dist = d
                    best_key = (ms, me)

            if best_key is None:
                continue

            tiers = model_by_time[best_key]
            # Check rank tiers
            matched_any = False
            rank1_gloss = None
            for rank in range(1, rank_threshold + 1):
                tier_name = f"Rank-{rank}"
                if tier_name in tiers:
                    gloss = tiers[tier_name].upper().strip()
                    if rank == 1:
                        rank1_gloss = gloss
                    if gloss == human_label:
                        if rank == 1:
                            top1_correct += 1
                        topn_correct += 1
                        matched_any = True
                        break

            if not matched_any and rank1_gloss:
                confusion.append(
                    {
                        "human_gloss": human_label,
                        "predicted_gloss": rank1_gloss,
                        "start_ms": h["start_ms"],
                        "end_ms": h["end_ms"],
                    }
                )

        total = len(human_rows)
        return {
            "file_stem": file_stem,
            "top_1_accuracy": round(top1_correct / total, 4) if total else 0.0,
            "top_5_accuracy": round(topn_correct / total, 4) if total else 0.0,
            "total_segments": total,
            "confusion_pairs": confusion,
        }

    # ------------------------------------------------------------------
    # Cross-workspace comparison
    # ------------------------------------------------------------------

    @staticmethod
    def compare_gloss_across_dbs(
        gloss: str, db_paths: Sequence[str | Path]
    ) -> list[dict[str, Any]]:
        """Compare a gloss across multiple workspace databases.

        Opens each DB read-only, collects duration stats, returns
        per-workspace comparison.
        """
        results: list[dict[str, Any]] = []
        for db_path in db_paths:
            db_path = Path(db_path)
            if not db_path.exists():
                continue
            db = CorpusDB(db_path)
            try:
                stats = db.gloss_duration_stats(gloss)
                stats["db_path"] = str(db_path)
                stats["workspace"] = db_path.parent.name
                results.append(stats)
            finally:
                db.close()
        return results


# ============================================================================
# Apply caching decorators to CorpusDB methods
# ============================================================================

# Expensive read methods — results are cached until the next write operation.
_CACHED_READ_METHODS = [
    "corpus_summary",
    "top_glosses",
    "concordance",
    "ngram_frequency",
    "cooccurrence",
    "vocabulary_stats",
    "cooccurrence_matrix",
    "all_duration_stats",
    "minimal_pairs",
    "dashboard_stats",
    "frequency_distribution",
    "duration_histogram",
    "lexical_richness",
    "per_signer_richness",
    "sign_transitions",
    "fluency_stats",
    "per_signer_fluency",
    "grouped_comparison",
    "top_glosses_for_group",
    "hand_comparison",
    "available_tiers",
    "multi_tier_concordance",
    "nonmanual_cooccurrence",
    "constructed_action_analysis",
    "clause_structure_analysis",
    "semantic_similarity",
    "get_file_sources",
    "files_by_source",
    "compare_sources",
    "batch_compare_sources",
    "spotting_accuracy",
]
for _name in _CACHED_READ_METHODS:
    setattr(CorpusDB, _name, _cached_read(getattr(CorpusDB, _name)))

# Write methods — clear all caches after execution.
_CACHE_CLEARING_METHODS = [
    "ingest_synthetic",
    "remove_file",
    "set_file_metadata",
    "set_tier_role",
    "set_tier_roles_bulk",
    "delete_tier_role",
    "delete_all_tier_roles",
    "apply_path_as_participant",
    "apply_path_as_tier",
    "apply_substring_extraction",
    "apply_segment_as_metadata",
    "apply_delimiter_extraction",
    "tag_file_source",
]
for _name in _CACHE_CLEARING_METHODS:
    setattr(CorpusDB, _name, _cache_clearing(getattr(CorpusDB, _name)))
