# FILE-SIZE-OK: Cohesive reconciliation engine (discover/repair/verify)
"""Kintsugi: Unified gap reconciliation for the ClickHouse cache (Issue #115, #238).

Named after the Japanese art of repairing broken pottery with gold lacquer
(金継ぎ). Interior gaps ("shards") and the trailing edge are discovered and
repaired autonomously, making the dataset stronger than before.

**Stathera-only anomaly detection.**  All anomalies detected via trade-ID
continuity (tid_delta != 0).  No time-based gap detection.  Both gaps
(tid_delta > 0) and overlaps (tid_delta < 0) are discovered and repaired.

**Day-recompute repair.**  When any Stathera anomaly is detected, the entire
Ouroboros day(s) containing it are deleted and reprocessed from scratch.
Vision-first (Binance Vision ZIPs), REST fallback via time-based fetch.
No Ariadne anchor needed — each day is independently processable.
This eliminates both missing-trade gaps and overlap artifacts from
concurrent sidecar + backfill writes.

**Trailing-edge fill (#238 Phase 2).**  After interior shard repairs, fills
the gap between the latest cached bar and now using Vision-first + REST
fallback (backfill_all_gaps).  Absorbed from the former gap-backfill daemon.

Architecture
------------
::

    Stathera audit (ClickHouse)  →  discover_shards()  →  day-recompute → verify
         (gaps + overlaps)              │
                            ┌───────────┘
                            ▼
                 Vision ZIP (if available) → _try_vision_day_replace()
                            │ fallback
                            ▼
                 REST time-based fetch → _process_gap_fallback_starttime()
                            │
    Trailing edge (latest bar → now)  →  backfill_all_gaps()  ◄─────┘

Usage
-----
Single pass::

    >>> from opendeviationbar.kintsugi import kintsugi_pass
    >>> results = kintsugi_pass()

Daemon mode::

    >>> from opendeviationbar.kintsugi import kintsugi_daemon
    >>> kintsugi_daemon()  # Long-running, adaptive polling
"""

from __future__ import annotations

import json
import logging
import os
import socket as _socket
import time
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from opendeviationbar.config.kintsugi import KintsugiConfig

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# NDJSON telemetry (persists independently of ClickHouse)
# ---------------------------------------------------------------------------

_KINTSUGI_LOG_DIR = Path.home() / "opendeviationbar-py" / "logs"
_KINTSUGI_NDJSON = _KINTSUGI_LOG_DIR / "kintsugi.ndjson"
# ---------------------------------------------------------------------------
# Repair configuration — SSoT in config/kintsugi.py
# ---------------------------------------------------------------------------

def _kintsugi_cfg() -> KintsugiConfig:
    from opendeviationbar.config.kintsugi import KintsugiConfig
    return KintsugiConfig()



def _emit_telemetry(event: str, **fields: object) -> None:
    """Append structured NDJSON telemetry line to kintsugi.ndjson.

    Persists independently of ClickHouse — survives tunnel drops.
    Rotates at 10 MB, keeping one backup (.ndjson.1).
    """
    try:
        _KINTSUGI_LOG_DIR.mkdir(parents=True, exist_ok=True)
        # Rotate if >10 MB
        if (
            _KINTSUGI_NDJSON.exists()
            and _KINTSUGI_NDJSON.stat().st_size > _kintsugi_cfg().ndjson_max_bytes
        ):
            rotated = _KINTSUGI_NDJSON.with_suffix(".ndjson.1")
            rotated.unlink(missing_ok=True)
            _KINTSUGI_NDJSON.rename(rotated)
        record = {
            "ts": datetime.now(UTC).isoformat(),
            "pid": os.getpid(),
            "event": event,
            **fields,
        }
        with _KINTSUGI_NDJSON.open("a") as f:
            f.write(json.dumps(record, default=str) + "\n")
    except OSError:
        pass  # Best-effort — never crash kintsugi for telemetry


def _sd_notify(state: str) -> None:
    """Send sd_notify message to systemd (if NOTIFY_SOCKET is set).

    This is a minimal implementation — no external dependency needed.
    Used for WatchdogSec integration (Issue #117).
    """
    addr = os.environ.get("NOTIFY_SOCKET")
    if not addr:
        return
    try:
        sock = _socket.socket(_socket.AF_UNIX, _socket.SOCK_DGRAM)
        try:
            if addr.startswith("@"):
                addr = "\0" + addr[1:]
            sock.sendto(state.encode(), addr)
        finally:
            sock.close()
    except OSError:
        logger.debug("sd_notify(%s) failed", state, exc_info=True)


# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------


# Symbols that get bandwidth priority when multiple gaps compete.
# BTCUSDT is the primary trading pair — its recency is non-negotiable.
_PRIORITY_SYMBOLS = ("BTCUSDT",)

# Transient error patterns — don't count toward max attempts (Issue #156)
_TRANSIENT_ERROR_PATTERNS = (
    "timeout",
    "connection",
    "rate limit",
    "503",
    "502",
    "500",
    "network",
    "timed out",
    "reset by peer",
)


# ---------------------------------------------------------------------------
# Data classes
# ---------------------------------------------------------------------------


@dataclass
class Shard:
    """A Stathera anomaly — consecutive bars with tid_delta != 0 (Issue #158).

    Covers both gaps (tid_delta > 0, missing trades) and overlaps
    (tid_delta < 0, duplicate trade-ID ranges from concurrent writers).

    Identified by anchor_last_tid and resume_first_tid. Repaired by
    day-recompute: delete all bars for the affected Ouroboros day(s) and
    reprocess from scratch via Vision or REST time-based fetch.
    """

    symbol: str
    threshold_decimal_bps: int
    gap_start_ms: int
    gap_end_ms: int
    gap_hours: float
    ouroboros_boundaries: list[int] = field(default_factory=list)
    # Issue #158: Stathera trade-ID fields
    anchor_last_tid: int | None = None  # last_agg_trade_id of bar before anomaly
    resume_first_tid: int | None = None  # first_agg_trade_id of bar after anomaly
    missing_trade_count: int = 0  # abs(tid_delta) — trade count for gap or overlap
    is_overlap: bool = False  # True when tid_delta < 0 (overlap, not gap)


@dataclass
class KintsugiResult:
    """Result of a single repair operation.

    Repairs ALWAYS use day-start fills (Vision ZIP or REST startTime=midnight_ms).
    Ariadne fromId pagination is NOT used in kintsugi repairs.
    """

    shard: Shard
    healed: bool
    bars_written: int
    duration_seconds: float
    had_anchor_tid: bool  # True when shard came from gap between two known bars (interior gap)
    ouroboros_splits: int  # Number of boundary sub-repairs
    error: str | None = None


# ---------------------------------------------------------------------------
# Shard discovery
# ---------------------------------------------------------------------------


def discover_shards(
    *,
    symbol: str | None = None,
    threshold: int | None = None,
) -> list[Shard]:
    """Discover shards via Stathera trade-ID continuity (Issue #158).

    Uses direct ClickHouse Stathera audit query. No legacy fallback.
    """
    return _discover_shards_stathera(
        symbol=symbol,
        threshold=threshold,
    )


def _discover_shards_stathera(
    *,
    symbol: str | None = None,
    threshold: int | None = None,
) -> list[Shard]:
    """Discover shards via Stathera trade-ID continuity (Issue #158).

    Connects directly to ClickHouse and runs the Stathera audit query.
    Each anomaly (tid_delta != 0) becomes a Shard with exact trade-ID anchors.

    Parameters
    ----------
    symbol : str | None
        Filter to specific symbol.
    threshold : int | None
        Filter to specific threshold.

    Returns
    -------
    list[Shard]
        Discovered shards with Stathera trade-ID anchors.
    """
    from opendeviationbar.clickhouse import OpenDeviationBarCache
    from opendeviationbar.ouroboros import (
        get_operational_ouroboros_mode,
    )

    ouroboros_mode = get_operational_ouroboros_mode()

    with OpenDeviationBarCache() as cache:
        # Discover (symbol, threshold) pairs
        if symbol and threshold:
            pairs = [(symbol, threshold)]
        else:
            result = cache.client.query(
                """
                SELECT DISTINCT symbol, threshold_decimal_bps
                FROM opendeviationbar_cache.open_deviation_bars
                WHERE first_agg_trade_id > 0
                ORDER BY symbol, threshold_decimal_bps
                """
            )
            pairs = [(row[0], row[1]) for row in result.result_rows]
            if symbol:
                pairs = [(s, t) for s, t in pairs if s == symbol]
            if threshold:
                pairs = [(s, t) for s, t in pairs if t == threshold]

        # Filter out disabled symbols (respect the Symbol Registry gate).
        from opendeviationbar.symbol_registry import filter_pairs_by_registry

        pairs = filter_pairs_by_registry(pairs)

        # Filter out pairs where threshold < symbol's min_threshold.
        # These are legacy data fragments from before Issue #89 enforcement;
        # kintsugi cannot repair them so skip at discovery.
        # Use get_min_threshold_for_symbol() directly — resolve_and_validate_threshold()
        # fires telemetry (NDJSON + Pushover) before raising, spamming Telegram every cycle.
        from opendeviationbar.threshold import get_min_threshold_for_symbol

        valid_pairs: list[tuple[str, int]] = []
        for sym, thr in pairs:
            min_thr = get_min_threshold_for_symbol(sym)
            if thr >= min_thr:
                valid_pairs.append((sym, thr))
            else:
                logger.debug(
                    "discover: skipping %s@%d (below min_threshold %d)",
                    sym,
                    thr,
                    min_thr,
                )

        shards: list[Shard] = []

        for sym, thr in valid_pairs:
            # Run Stathera audit query per pair
            from opendeviationbar.clickhouse.stathera_query import (
                FULL_TABLE,
                STATHERA_CORE_COLUMNS,
                per_pair_filter,
                per_pair_window,
            )

            result = cache.client.query(
                f"""
                SELECT close_time_ms, first_agg_trade_id,
                       last_agg_trade_id, prev_close_time_ms,
                       prev_last_tid, tid_delta, rn
                FROM (
                    SELECT {STATHERA_CORE_COLUMNS}
                    FROM {FULL_TABLE} FINAL
                    {per_pair_filter()}
                    {per_pair_window()}
                )
                WHERE rn > 1 AND tid_delta != 0
                """,
                parameters={
                    "symbol": sym,
                    "threshold": thr,
                    "ouroboros": ouroboros_mode,
                },
            )

            for row in result.result_rows:
                close_time_ms = row[0]
                _first_tid = row[1]
                _last_tid = row[2]
                prev_close_time_ms = row[3]
                prev_last_tid = row[4]
                tid_delta = row[5]
                rn = row[6]

                # Skip first row (no previous bar to compare)
                if rn <= 1 or tid_delta is None:
                    continue

                # Skip clean bars (tid_delta == 0)
                if tid_delta == 0:
                    continue

                # Both gaps (tid_delta > 0) and overlaps (tid_delta < 0)
                # trigger day-recompute: DELETE entire day(s) + Ariadne reprocess.
                is_overlap = tid_delta < 0

                # Compute gap hours from timestamps
                gap_ms = close_time_ms - (prev_close_time_ms or 0)
                gap_hours = gap_ms / 3600000.0

                shard = Shard(
                    symbol=sym,
                    threshold_decimal_bps=thr,
                    gap_start_ms=prev_close_time_ms or 0,
                    gap_end_ms=close_time_ms,
                    gap_hours=gap_hours,
                    anchor_last_tid=prev_last_tid,
                    resume_first_tid=_first_tid,
                    missing_trade_count=abs(tid_delta),
                    is_overlap=is_overlap,
                )
                shards.append(shard)

    _enrich_shards(shards)

    # Emit GAP_DETECTED hooks
    if shards:
        try:
            from opendeviationbar.hooks import HookEvent, emit_hook

            for shard in shards:
                emit_hook(
                    HookEvent.GAP_DETECTED,
                    symbol=shard.symbol,
                    threshold_dbps=shard.threshold_decimal_bps,
                    gap_hours=shard.gap_hours,
                )
        except (ImportError, OSError, RuntimeError, ValueError):
            logger.debug("failed to emit GAP_DETECTED hooks", exc_info=True)

    return shards


def _is_transient_error(error_message: str) -> bool:
    """Check if an error message indicates a transient (retriable) failure.

    Transient errors (network, timeout, rate limit, server errors) should
    not count toward the per-shard max attempt limit (Issue #156).
    """
    lower = error_message.lower()
    return any(pattern in lower for pattern in _TRANSIENT_ERROR_PATTERNS)


def _shard_sort_key(shard: Shard) -> tuple[int, int]:
    """Sort key: recency first, then BTCUSDT priority.

    Most-recent gaps are repaired first. Among same-recency gaps,
    BTCUSDT (and other priority symbols) get bandwidth first.
    """
    symbol_rank = 0 if shard.symbol in _PRIORITY_SYMBOLS else 1
    return (-shard.gap_end_ms, symbol_rank)


def _get_dismissed_set() -> set[tuple[str, int, int]]:
    """Batch query: shards confirmed as 0-bar (sub-threshold) in last 30 days.

    Returns set of (symbol, threshold_decimal_bps, gap_start_ms) tuples.
    Single query replaces N per-shard queries (Issue #179 P1-1).
    """
    try:
        from opendeviationbar.clickhouse import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            result = cache.client.query(
                "SELECT DISTINCT symbol, threshold_decimal_bps, gap_start_ms "
                "FROM opendeviationbar_cache.kintsugi_log "
                "WHERE result = 'clean' AND bars_written = 0 "
                "AND timestamp > now() - INTERVAL 30 DAY"
            )
            return {(row[0], row[1], row[2]) for row in result.result_rows}
    except (OSError, RuntimeError, ConnectionError):
        logger.debug("_get_dismissed_set query failed", exc_info=True)
        return set()


def _get_rate_limited_set() -> set[tuple[str, int, int]]:
    """Batch query: shards with >= 3 non-transient attempts in last 24h.

    Returns set of (symbol, threshold_decimal_bps, gap_start_ms) tuples.
    Single query replaces N per-shard queries (Issue #179 P1-1).
    """
    try:
        from opendeviationbar.clickhouse import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            result = cache.client.query(
                "SELECT symbol, threshold_decimal_bps, gap_start_ms "
                "FROM opendeviationbar_cache.kintsugi_log "
                "WHERE result != 'clean' "
                "AND timestamp > now() - INTERVAL 24 HOUR "
                "GROUP BY symbol, threshold_decimal_bps, gap_start_ms "
                "HAVING count() >= 3"
            )
            return {(row[0], row[1], row[2]) for row in result.result_rows}
    except (OSError, RuntimeError, ConnectionError):
        logger.debug("_get_rate_limited_set query failed", exc_info=True)
        return set()


def _select_repair_candidates(
    shards: list[Shard],
    max_repairs: int,
) -> list[Shard]:
    """Select shards eligible for repair this pass.

    Applies three filters in order:
    1. **Dismissed**: Skip shards confirmed as 0-bar (gap too short) in last 30d.
    2. **Budget**: At most ``max_repairs`` shards per pass.
    3. **Rate limit**: Skip shards with ≥3 non-transient attempts in 24h.

    Uses batch ClickHouse queries (2 queries total) instead of per-shard
    queries (was ~2xN queries for N shards). Issue #179 P1-1.
    """
    dismissed = _get_dismissed_set()
    rate_limited = _get_rate_limited_set()

    candidates: list[Shard] = []
    budget = max_repairs
    n_dismissed = 0
    n_rate_limited = 0

    for shard in shards:
        key = (shard.symbol, shard.threshold_decimal_bps, shard.gap_start_ms)

        if key in dismissed:
            n_dismissed += 1
            continue

        if budget <= 0:
            break

        if key in rate_limited:
            n_rate_limited += 1
            continue

        budget -= 1
        candidates.append(shard)

    if n_dismissed > 0:
        logger.info("skipped %d dismissed shards (0-bar in last 30d)", n_dismissed)
    if n_rate_limited > 0:
        logger.info("skipped %d rate-limited shards", n_rate_limited)

    return candidates


def _enrich_shards(shards: list[Shard]) -> None:
    """Add Ouroboros boundaries to shards."""
    from opendeviationbar.ouroboros import get_ouroboros_boundaries

    if not shards:
        return

    try:
        for shard in shards:
            # Ouroboros boundaries within the gap
            if shard.gap_start_ms > 0 and shard.gap_end_ms > 0:
                start_date = datetime.fromtimestamp(
                    shard.gap_start_ms / 1000,
                    tz=UTC,
                ).date()
                end_date = datetime.fromtimestamp(
                    shard.gap_end_ms / 1000,
                    tz=UTC,
                ).date()
                from opendeviationbar.ouroboros import (
                    get_operational_ouroboros_mode,
                )

                boundaries = get_ouroboros_boundaries(
                    start_date,
                    end_date,
                    get_operational_ouroboros_mode(),
                )
                shard.ouroboros_boundaries = [b.timestamp_ms for b in boundaries]
    except (OSError, RuntimeError, ConnectionError) as e:
        logger.warning("failed to enrich shards: %s", e)


# ---------------------------------------------------------------------------
# Shard repair
# ---------------------------------------------------------------------------


def repair_shard(
    shard: Shard,
    *,
    include_microstructure: bool = True,
    verbose: bool = False,
) -> KintsugiResult:
    """Repair a single shard via backfill_gap() (Vision-first + REST fallback)."""
    # Validate threshold before attempting repair — reject orphaned invalid data.
    # Use get_min_threshold_for_symbol() directly to avoid telemetry spam.
    from opendeviationbar.threshold import get_min_threshold_for_symbol

    min_thr = get_min_threshold_for_symbol(shard.symbol)
    if shard.threshold_decimal_bps < min_thr:
        return KintsugiResult(
            shard=shard,
            healed=False,
            bars_written=0,
            duration_seconds=0.0,
            had_anchor_tid=False,
            ouroboros_splits=0,
            error=(
                f"threshold {shard.threshold_decimal_bps} dbps "
                f"below minimum for {shard.symbol}"
            ),
        )

    t0 = time.monotonic()
    n_splits = max(len(shard.ouroboros_boundaries), 1)
    had_anchor_tid = shard.anchor_last_tid is not None

    try:
        total_bars = _repair(
            shard,
            include_microstructure=include_microstructure,
            verbose=verbose,
        )
    except (OSError, RuntimeError, ConnectionError, ValueError) as e:
        elapsed = time.monotonic() - t0
        logger.exception(
            "repair failed for %s@%d",
            shard.symbol,
            shard.threshold_decimal_bps,
        )
        _notify_repair_failure(shard, str(e))
        return KintsugiResult(
            shard=shard,
            healed=False,
            bars_written=0,
            duration_seconds=elapsed,
            had_anchor_tid=had_anchor_tid,
            ouroboros_splits=n_splits,
            error=str(e),
        )

    elapsed = time.monotonic() - t0

    # _repair() returns -1 when today-guard defers the shard. This is NOT a
    # "clean" result — the shard wasn't attempted, just deferred to tomorrow.
    # Must NOT be logged to kintsugi_log (would be dismissed for 30 days).
    if total_bars < 0:
        return KintsugiResult(
            shard=shard,
            healed=False,
            bars_written=0,
            duration_seconds=elapsed,
            had_anchor_tid=had_anchor_tid,
            ouroboros_splits=n_splits,
            error="deferred_today_guard",
        )

    healed = total_bars > 0

    # 0 bars is a valid outcome — gap exists in trade IDs but price didn't
    # breach the threshold.  This is clean (not an error).  Logged to
    # kintsugi_log so _get_dismissed_set() can skip the shard on future passes.
    error = None

    state_label = "HEALED" if healed else "NO_BARS"
    logger.info(
        "repair %s for %s@%d: %d bars in %.1fs (day-start-fill, anchored=%s, splits=%d)",
        state_label,
        shard.symbol,
        shard.threshold_decimal_bps,
        total_bars,
        elapsed,
        had_anchor_tid,
        n_splits,
    )

    return KintsugiResult(
        shard=shard,
        healed=healed,
        bars_written=total_bars,
        duration_seconds=elapsed,
        had_anchor_tid=had_anchor_tid,
        ouroboros_splits=n_splits,
        error=error,
    )


def _notify_repair_failure(shard: Shard, error_msg: str) -> None:
    """Send immediate LOUD Telegram alert on repair failure.

    Reports trade-ID anchors and anomaly details (gap or overlap).
    Always LOUD (disable_notification=False) — repair failures require
    operator attention.
    """
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram

        anomaly_type = "overlap" if shard.is_overlap else "gap"
        lines = [
            "<b>KINTSUGI REPAIR FAILED</b>",
            f"<b>Pair:</b> {shard.symbol}@{shard.threshold_decimal_bps}",
            f"<b>Stathera {anomaly_type}:</b> {shard.missing_trade_count:,} trades",
            f"<b>Anchor:</b> tid {shard.anchor_last_tid} → {shard.resume_first_tid}",
            f"<b>Error:</b> <code>{error_msg[:200]}</code>",
            "\n<b>Next:</b> Will retry on next pass (max 3/24h).",
            "<b>If persistent:</b> Check Binance Vision data availability, "
            "REST rate limits, CH write health",
            _build_context_block("kintsugi"),
        ]
        send_telegram("\n".join(lines), disable_notification=False)
    except Exception:
        logger.debug("Failed to send repair failure Telegram alert", exc_info=True)




def _repair(
    shard: Shard,
    *,
    include_microstructure: bool = True,
    verbose: bool = False,
) -> int:
    """Repair shard via day-recompute: delete + reprocess entire Ouroboros day(s).

    When a Stathera anomaly (gap or overlap) is detected, the affected
    Ouroboros day(s) are reprocessed from scratch:
      1. Identify all Ouroboros days spanned by the anomaly
      2. Try Vision ZIP first (include_incomplete=True for orphan flush)
      3. Fall back to REST time-based fetch (no Ariadne anchor needed)
      4. Atomic per-chunk DELETE+INSERT to ClickHouse

    This eliminates both missing-trade gaps and overlap artifacts from
    concurrent sidecar + backfill writes.
    """
    if shard.anchor_last_tid is None:
        msg = (
            f"kintsugi: shard {shard.symbol}@{shard.threshold_decimal_bps} "
            f"has no anchor_last_tid — cannot repair interior gap "
            f"(gap_start_ms={shard.gap_start_ms})"
        )
        logger.error(msg)
        _notify_repair_failure(shard, msg)
        raise RuntimeError(msg)

    from opendeviationbar.ouroboros import get_operational_ouroboros_mode

    ouroboros_mode = get_operational_ouroboros_mode()

    # Identify the Ouroboros day boundaries spanning the gap.
    # Normalize: gap_start_ms/gap_end_ms may be inverted when lagInFrame()
    # operates in trade-ID order and concurrent writers produce bars with
    # close_time_ms out of trade-ID order.
    lo_ms = min(shard.gap_start_ms, shard.gap_end_ms)
    hi_ms = max(shard.gap_start_ms, shard.gap_end_ms)
    gap_start_date = datetime.fromtimestamp(lo_ms / 1000, tz=UTC).date()
    gap_end_date = datetime.fromtimestamp(hi_ms / 1000, tz=UTC).date()

    # Guard: avoid repairing today's date when the gap is at the trailing
    # edge — the sidecar is actively writing bars there, so DELETE + rewrite
    # races with sidecar INSERTs.
    #
    # However, INTERIOR gaps (resume_first_tid is set = committed bars exist
    # AFTER the gap) are safe to repair even on today's date. The per-chunk
    # atomic replace (v13.3+) only touches the gap's trade-ID range, which is
    # disjoint from the sidecar's live writing region. This is a pure Stathera
    # check — trade-ID sequentiality, not time-based.
    #
    # Without this exception, mid-session ClickHouse blips leave gaps that
    # are deferred until midnight, defeating the purpose of kintsugi.
    today_utc = datetime.now(tz=UTC).date()
    if gap_end_date >= today_utc:
        gap_is_interior = shard.resume_first_tid is not None
        if gap_is_interior:
            logger.info(
                "kintsugi day-recompute: ALLOWING interior intra-day repair for "
                "%s@%d — committed bars exist after gap "
                "(anchor=%s → resume=%s)",
                shard.symbol,
                shard.threshold_decimal_bps,
                shard.anchor_last_tid,
                shard.resume_first_tid,
            )
            # Fall through to repair — gap is sealed, no sidecar contention
        else:
            # Trailing-edge gap: no committed bars after it, sidecar is actively
            # writing here. Clamp to yesterday so multi-day gaps still get their
            # historical days repaired.
            gap_end_date = today_utc - timedelta(days=1)
            if gap_end_date < gap_start_date:
                logger.info(
                    "kintsugi day-recompute: DEFERRING %s@%d — trailing-edge gap "
                    "on today (%s), sidecar is actively writing. Will retry tomorrow.",
                    shard.symbol,
                    shard.threshold_decimal_bps,
                    today_utc,
                )
                return -1  # sentinel: deferred, not clean — must NOT be dismissed

    # In day-mode Ouroboros, process up to _kintsugi_cfg().batch_days days per repair.
    # Each day is independently processed (processor reset at boundaries)
    # so the day-mode invariant is always preserved. Multi-day batching
    # dramatically speeds up backfill for large gaps (e.g., 288-day gap
    # at BTCUSDT@1000 takes 288 passes at 1 day/pass vs 10 passes at 30).
    # Fair queuing is preserved by max_repairs per pass.
    num_days = (gap_end_date - gap_start_date).days + 1
    batch_days = min(num_days, _kintsugi_cfg().batch_days) if ouroboros_mode == "day" else num_days
    if num_days > batch_days:
        gap_end_date = gap_start_date + timedelta(days=batch_days - 1)
        logger.info(
            "kintsugi day-recompute: %s@%d gap spans %d days (%s to %s), "
            "batching %d days (remaining %d on next pass)",
            shard.symbol,
            shard.threshold_decimal_bps,
            num_days,
            gap_start_date,
            gap_end_date,
            batch_days,
            num_days - batch_days,
        )

    # Hold the Redis write lock for the ENTIRE batch sequence.
    # Without this, sidecar or backfill can write bars between our DELETE and
    # INSERT, creating Stathera overlaps. The lock scope is per-(symbol, threshold)
    # which is exactly the granularity we need — no other writer should touch
    # this pair while we're recomputing.
    from opendeviationbar.clickhouse.write_lock import cache_write_lock

    # Build list of (date, day_start_ms, day_end_ms) for each day in the batch.
    days_to_repair: list[tuple[object, int, int]] = []
    current_date = gap_start_date
    while current_date <= gap_end_date:
        day_start_ms = int(
            datetime(
                current_date.year, current_date.month, current_date.day, tzinfo=UTC
            ).timestamp()
            * 1000
        )
        day_end_ms = int(
            datetime(
                current_date.year,
                current_date.month,
                current_date.day,
                23,
                59,
                59,
                999000,
                tzinfo=UTC,
            ).timestamp()
            * 1000
        )
        days_to_repair.append((current_date, day_start_ms, day_end_ms))
        current_date += timedelta(days=1)


    total_bars = 0
    with cache_write_lock(
        shard.symbol,
        shard.threshold_decimal_bps,
        timeout=(_kintsugi_cfg().repair_timeout_s + 60) * batch_days,
    ):
        if _kintsugi_cfg().batch_workers <= 1 or len(days_to_repair) <= 1:
            # Sequential path (single worker or single day).
            for i, (repair_date, day_start_ms, day_end_ms) in enumerate(
                days_to_repair
            ):
                logger.info(
                    "kintsugi day-recompute: %s@%d — replacing bars for %s "
                    "(%d/%d in batch, crash-safe per-chunk replace)",
                    shard.symbol,
                    shard.threshold_decimal_bps,
                    repair_date,
                    i + 1,
                    len(days_to_repair),
                )
                bars = _repair_under_lock(
                    shard=shard,
                    day_start_ms=day_start_ms,
                    day_end_ms=day_end_ms,
                    repair_date=repair_date,
                    ouroboros_mode=ouroboros_mode,
                    include_microstructure=include_microstructure,
                    verbose=verbose,
                )
                total_bars += bars
        else:
            # Parallel path: each day is independent (separate trades fetch,
            # fresh processor, independent CH write). ClickHouse mutation
            # check (>5 pending) in _repair_under_lock() provides natural
            # backpressure — days that hit mutation pressure return 0 and
            # are rediscovered on the next pass.
            import concurrent.futures

            workers = min(_kintsugi_cfg().batch_workers, len(days_to_repair))
            logger.info(
                "kintsugi day-recompute: %s@%d — parallel batch of %d days "
                "with %d workers",
                shard.symbol,
                shard.threshold_decimal_bps,
                len(days_to_repair),
                workers,
            )

            def _repair_day(
                args: tuple[int, tuple[object, int, int]],
            ) -> int:
                i, (repair_date, day_start_ms, day_end_ms) = args
                logger.info(
                    "kintsugi day-recompute: %s@%d — replacing bars for %s "
                    "(%d/%d in batch, worker-parallel)",
                    shard.symbol,
                    shard.threshold_decimal_bps,
                    repair_date,
                    i + 1,
                    len(days_to_repair),
                )
                return _repair_under_lock(
                    shard=shard,
                    day_start_ms=day_start_ms,
                    day_end_ms=day_end_ms,
                    repair_date=repair_date,
                    ouroboros_mode=ouroboros_mode,
                    include_microstructure=include_microstructure,
                    verbose=verbose,
                )

            with concurrent.futures.ThreadPoolExecutor(
                max_workers=workers
            ) as executor:
                futures = [
                    executor.submit(_repair_day, (i, day_info))
                    for i, day_info in enumerate(days_to_repair)
                ]
                for future in concurrent.futures.as_completed(futures):
                    try:
                        total_bars += future.result()
                    except Exception:
                        logger.exception(
                            "kintsugi day-recompute: parallel day repair failed "
                            "for %s@%d — will retry on next pass",
                            shard.symbol,
                            shard.threshold_decimal_bps,
                        )

    return total_bars


def _try_vision_day_replace(
    *,
    shard: Shard,
    repair_date: object,  # datetime.date
    include_microstructure: bool,
) -> int:
    """Try Vision archives for a single repair day (rate-limit-free, deterministic).

    Uses the same Redis negative cache as backfill.py to avoid repeated probes
    for known-unavailable dates. Vision data is preferred because:
    - No API rate limits (ZIP download from data.binance.vision)
    - Deterministic (same ZIP = same trades every time)
    - Faster for full-day fetches (single HTTP GET vs hundreds of REST pages)

    Streams bars from Vision, writes with overlap_strategy="replace" for crash
    safety. Returns 0 on any failure (caller falls back to REST Ariadne).
    """
    from opendeviationbar.backfill import (
        _is_vision_cached_unavailable,
        _mark_vision_unavailable,
    )

    # T-1 cutoff: Vision archives for today/yesterday may not exist yet
    today_utc = datetime.now(tz=UTC).date()
    vision_cutoff = today_utc - timedelta(days=1)

    date_str = str(repair_date)  # YYYY-MM-DD

    if repair_date >= vision_cutoff:
        logger.debug(
            "kintsugi Vision: %s@%d — %s not eligible (>= T-1 cutoff %s)",
            shard.symbol,
            shard.threshold_decimal_bps,
            date_str,
            vision_cutoff,
        )
        return 0

    if _is_vision_cached_unavailable(shard.symbol, date_str):
        logger.debug(
            "kintsugi Vision: %s@%d — %s cached-unavailable (Redis negative cache)",
            shard.symbol,
            shard.threshold_decimal_bps,
            date_str,
        )
        return 0

    logger.info(
        "kintsugi Vision: trying %s@%d for %s (rate-limit-free)",
        shard.symbol,
        shard.threshold_decimal_bps,
        date_str,
    )

    try:
        import polars as pl

        from opendeviationbar.clickhouse import OpenDeviationBarCache
        from opendeviationbar.orchestration.helpers import (
            _stream_open_deviation_bars_binance,
        )

        # Stream bars from Vision ZIP for this single day.
        # _stream_open_deviation_bars_binance() uses Rust stream_binance_trades_arrow()
        # which downloads the Vision ZIP and parses trades in chunks.
        bar_frames: list[pl.DataFrame] = []
        for bar_batch in _stream_open_deviation_bars_binance(
            shard.symbol,
            date_str,
            date_str,
            shard.threshold_decimal_bps,
            "spot",  # crypto symbols are always spot
            include_microstructure=include_microstructure,
            include_incomplete=True,  # flush orphan — critical for low-volatility days
        ):
            bar_frames.append(bar_batch)

        if not bar_frames:
            logger.info(
                "kintsugi Vision: %s@%d — no bars produced for %s",
                shard.symbol,
                shard.threshold_decimal_bps,
                date_str,
            )
            return 0

        bars_df = pl.concat(bar_frames, rechunk=False)
        del bar_frames

        if bars_df.is_empty():
            return 0

        # Write bars with "replace" strategy — atomic DELETE+INSERT per chunk.
        # _skip_lock=True: caller already holds cache_write_lock.
        from opendeviationbar import __version__

        source_version = f"{__version__}+kintsugi-vision"
        with OpenDeviationBarCache() as cache:
            written = cache.store_bars_batch(
                symbol=shard.symbol,
                threshold_decimal_bps=shard.threshold_decimal_bps,
                bars=bars_df,
                version=source_version,
                overlap_strategy="replace",
                _skip_lock=True,
            )

        del bars_df

    except Exception as exc:
        # ALL Vision failures mark the negative cache — not just 404s.
        # Network timeouts, DNS errors, connection refused, etc. mean Vision
        # is equally unavailable for this (symbol, date) right now. Without
        # caching these, kintsugi would retry Vision every 60s per pass,
        # burning time on a known-bad path before falling back to REST.
        # The 1-hour TTL ensures we'll re-probe after transient issues resolve.
        _mark_vision_unavailable(shard.symbol, date_str)
        exc_str = str(exc)
        is_404 = (
            "404" in exc_str or "No data" in exc_str or "not found" in exc_str.lower()
        )
        if is_404:
            logger.info(
                "kintsugi Vision: %s@%d — %s not published (cached 1h): %s",
                shard.symbol,
                shard.threshold_decimal_bps,
                date_str,
                exc,
            )
        else:
            logger.warning(
                "kintsugi Vision: %s@%d — %s failed (cached 1h, falling back to REST): %s",
                shard.symbol,
                shard.threshold_decimal_bps,
                date_str,
                exc,
            )
        return 0
    else:
        return written


def _repair_under_lock(
    *,
    shard: Shard,
    day_start_ms: int,
    day_end_ms: int,
    repair_date: object,  # datetime.date — single day being repaired
    ouroboros_mode: str,  # noqa: ARG001
    include_microstructure: bool,
    verbose: bool,  # noqa: ARG001
) -> int:
    """Execute crash-safe repair while the caller holds the write lock.

    This function must ONLY be called from _repair() inside cache_write_lock().
    The lock prevents sidecar/backfill from writing to the same (symbol, threshold)
    concurrently.

    DATA SOURCE PRIORITY: Vision-first, REST day-start fallback.
    1. Check if repair_date is Vision-eligible (before T-1, not in Redis negative cache)
    2. If yes, stream bars from Binance Vision ZIP archives (rate-limit-free, deterministic)
    3. If Vision fails or unavailable, fall back to REST time-based fetch (startTime=UTC midnight)
       No Ariadne fromId pagination — always fetch the full day from UTC 00:00:00.000

    CRASH SAFETY: Uses overlap_strategy="replace" which does atomic per-chunk
    DELETE+INSERT via store_bars_replacing(). If killed mid-repair:
    - Completed chunks: correctly replaced (no data loss)
    - Incomplete chunks: old bars still exist (picked up next pass)
    No bars are ever deleted without immediate replacement.
    """
    from opendeviationbar.clickhouse import OpenDeviationBarCache

    # Step 1: Wait for any pending mutations to drain before starting.
    # High mutation count means ClickHouse is under pressure — defer repair.
    with OpenDeviationBarCache() as cache:
        pending = cache.client.query(
            "SELECT count() FROM system.mutations "
            "WHERE database = 'opendeviationbar_cache' "
            "AND table = 'open_deviation_bars' AND is_done = 0"
        ).result_rows[0][0]
        max_pending_mutations = 5
        if pending > max_pending_mutations:
            logger.info(
                "kintsugi day-recompute: DEFERRING %s@%d — %d pending mutations, "
                "will retry when mutation queue drains",
                shard.symbol,
                shard.threshold_decimal_bps,
                pending,
            )
            return 0

    # Step 2: Try Vision-first for this day (rate-limit-free, deterministic).
    # Vision is preferred over REST because:
    #   - No API rate limits (ZIP download from data.binance.vision)
    #   - Deterministic data (same ZIP always produces same trades)
    #   - Lower latency for historical days (single ZIP vs many REST pages)
    vision_bars = _try_vision_day_replace(
        shard=shard,
        repair_date=repair_date,
        include_microstructure=include_microstructure,
    )
    if vision_bars > 0:
        logger.info(
            "kintsugi day-recompute: Vision wrote %d bars for %s@%d on %s",
            vision_bars,
            shard.symbol,
            shard.threshold_decimal_bps,
            repair_date,
        )
        return vision_bars

    # Step 3: Vision unavailable — fall back to REST time-based fetch.
    # In day-mode Ouroboros, each day is independently processable. We fetch
    # ALL trades for the day by time range (startTime/endTime), process through
    # a fresh processor, and write bars. No Ariadne anchor needed — this makes
    # parallel workers truly independent (no queries against adjacent days' bars).
    logger.info(
        "kintsugi day-recompute: REST time-based fetch for %s@%d on %s",
        shard.symbol,
        shard.threshold_decimal_bps,
        repair_date,
    )

    from opendeviationbar.backfill import _process_gap_fallback_starttime

    return _process_gap_fallback_starttime(
        symbol=shard.symbol,
        threshold_decimal_bps=shard.threshold_decimal_bps,
        start_ms=day_start_ms,
        end_ms=day_end_ms,
        include_microstructure=include_microstructure,
        overlap_strategy="replace",  # atomic per-chunk DELETE+INSERT
        _skip_lock=True,  # prevent deadlock: outer lock already held
    )


# ---------------------------------------------------------------------------
# Verification
# ---------------------------------------------------------------------------


def verify_repair(shard: Shard) -> bool:
    """Verify a Stathera anomaly has been healed via direct ClickHouse query (Issue #158).

    Checks whether any trade-ID discontinuity (gap or overlap) between
    anchor_last_tid and resume_first_tid still exists. Returns True if
    all bars in the range have tid_delta == 0.
    """
    if shard.anchor_last_tid is None or shard.resume_first_tid is None:
        logger.warning(
            "verify_repair: no trade-ID anchors for %s@%d — skipping",
            shard.symbol,
            shard.threshold_decimal_bps,
        )
        return True  # Can't verify without anchors, assume healed

    from opendeviationbar.clickhouse import OpenDeviationBarCache
    from opendeviationbar.ouroboros import get_operational_ouroboros_mode

    ouroboros_mode = get_operational_ouroboros_mode()

    try:
        with OpenDeviationBarCache() as cache:
            result = cache.client.query(
                """
                SELECT
                    first_agg_trade_id - lagInFrame(last_agg_trade_id, 1)
                        OVER (ORDER BY first_agg_trade_id, close_time_ms) - 1
                        AS tid_delta
                FROM opendeviationbar_cache.open_deviation_bars FINAL
                WHERE symbol = {symbol:String}
                  AND threshold_decimal_bps = {threshold:UInt32}
                  AND ouroboros_mode = {ouroboros:String}
                  AND first_agg_trade_id > 0
                  AND last_agg_trade_id > 0
                  AND last_agg_trade_id >= {anchor:Int64}
                  AND first_agg_trade_id <= {resume:Int64}
                ORDER BY first_agg_trade_id, close_time_ms
                """,
                parameters={
                    "symbol": shard.symbol,
                    "threshold": shard.threshold_decimal_bps,
                    "ouroboros": ouroboros_mode,
                    "anchor": shard.anchor_last_tid,
                    "resume": shard.resume_first_tid,
                },
            )

            # Check if any row has tid_delta != 0 (gap or overlap still present)
            for row in result.result_rows:
                td = row[0]
                if td is not None and td != 0:
                    anomaly_type = "gap" if td > 0 else "overlap"
                    logger.info(
                        "verify_repair: %s persists for %s@%d (tid_delta=%d)",
                        anomaly_type,
                        shard.symbol,
                        shard.threshold_decimal_bps,
                        td,
                    )
                    return False

            # (#264) Day-mode oracle: verify no cross-midnight bars in repaired range
            if ouroboros_mode == "day":
                from opendeviationbar.validation.day_mode.oracle import (
                    cross_midnight_count_range,
                )

                violations = cross_midnight_count_range(
                    cache.client,
                    shard.symbol,
                    shard.threshold_decimal_bps,
                    ouroboros_mode,
                    shard.anchor_last_tid,
                    shard.resume_first_tid,
                )
                if violations > 0:
                    logger.error(
                        "Day-mode oracle FAILED: %d cross-midnight bars in [%d, %d] "
                        "for %s@%d",
                        violations,
                        shard.anchor_last_tid,
                        shard.resume_first_tid,
                        shard.symbol,
                        shard.threshold_decimal_bps,
                    )
                    return False

    except Exception:
        logger.exception(
            "verify_repair: ClickHouse query failed for %s@%d",
            shard.symbol,
            shard.threshold_decimal_bps,
        )
        return False

    return True


# ---------------------------------------------------------------------------
# Kintsugi log (ClickHouse)
# ---------------------------------------------------------------------------


def _maybe_log_repair(result: KintsugiResult) -> None:
    """Log repair result, skipping kintsugi_log for transient/deferred errors.

    Transient errors (network, timeout, 5xx) and today-guard deferrals are not
    written to the kintsugi_log table. Transient errors don't count toward the
    3-attempt-per-24h rate limit. Deferred shards must not be dismissed — they
    were skipped because the sidecar is actively writing, not because the gap
    is sub-threshold.
    """
    if result.error == "deferred_today_guard":
        return
    if result.error and _is_transient_error(result.error):
        logger.info(
            "transient error for %s@%d — not counting toward attempt limit: %s",
            result.shard.symbol,
            result.shard.threshold_decimal_bps,
            result.error,
        )
        return
    _log_repair(result)


def _log_repair(result: KintsugiResult) -> None:
    """Write repair result to kintsugi_log ClickHouse table."""
    try:
        from opendeviationbar.clickhouse import OpenDeviationBarCache
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode

        with OpenDeviationBarCache() as cache:
            cache.client.command(
                "INSERT INTO opendeviationbar_cache.kintsugi_log "
                "(symbol, threshold_decimal_bps, priority, "
                "gap_start_ms, gap_end_ms, gap_hours, "
                "action, result, ariadne_from_id, ouroboros_splits, "
                "bars_written, duration_seconds, error_message, ouroboros_mode) "
                "VALUES "
                "({symbol:String}, {threshold:UInt32}, {priority:String}, "
                "{gap_start:Int64}, {gap_end:Int64}, {gap_hours:Float64}, "
                "{action:String}, {result:String}, {ariadne:Int64}, "
                "{splits:UInt8}, {bars:UInt32}, {duration:Float64}, "
                "{error:String}, {ouroboros_mode:String})",
                parameters={
                    "symbol": result.shard.symbol,
                    "threshold": result.shard.threshold_decimal_bps,
                    "priority": "",
                    "gap_start": result.shard.gap_start_ms,
                    "gap_end": result.shard.gap_end_ms,
                    "gap_hours": result.shard.gap_hours,
                    "action": "backfill_gap",
                    "result": (
                        "healed"
                        if result.healed
                        else "clean"
                        if not result.error
                        else "failed"
                    ),
                    "ariadne": (result.shard.anchor_last_tid or 0),  # column name legacy; stores anchor_last_tid for audit trail
                    "splits": result.ouroboros_splits,
                    "bars": result.bars_written,
                    "duration": result.duration_seconds,
                    "error": result.error or "",
                    "ouroboros_mode": get_operational_ouroboros_mode(),
                },
            )
    except (OSError, RuntimeError, ConnectionError) as e:
        logger.warning("failed to log kintsugi result: %s", e)


# ---------------------------------------------------------------------------
# Pass + daemon
# ---------------------------------------------------------------------------


def kintsugi_pass(
    *,
    dry_run: bool = False,
    symbol: str | None = None,
    threshold: int | None = None,
    max_repairs: int | None = None,
    include_microstructure: bool = True,
    verbose: bool = False,
) -> list[KintsugiResult]:
    """Single pass: discover shards (gaps + overlaps), repair via day-recompute.

    Always processes shards in reverse chronological order (newest first).
    Both gaps (tid_delta > 0) and overlaps (tid_delta < 0) are repaired
    by deleting the affected Ouroboros day(s) and reprocessing from scratch.

    Parameters
    ----------
    dry_run
        If True, discover only — no repairs.
    symbol
        Filter to a specific symbol.
    threshold
        Filter to a specific threshold in dbps (e.g., 250).
    max_repairs
        Maximum repairs per pass (default from config).
    include_microstructure
        Include microstructure features in repaired bars.
    verbose
        Enable detailed logging.

    Returns
    -------
    list[KintsugiResult]
        Results for each shard repaired (or discovered in dry_run).
    """
    if max_repairs is None:
        max_repairs = _kintsugi_cfg().max_repairs_per_pass
    pass_t0 = time.monotonic()
    logger.info("kintsugi pass starting (dry_run=%s)", dry_run)
    _emit_telemetry("pass_start", dry_run=dry_run)
    shards = discover_shards(symbol=symbol, threshold=threshold)

    if not shards:
        logger.info("kintsugi: no shards found — cache is clean")
        _emit_telemetry("discovery_complete", total_shards=0)
        return []

    # Sort: recency first, then BTCUSDT priority (Issue #164)
    shards.sort(key=_shard_sort_key)

    _emit_telemetry("discovery_complete", total_shards=len(shards))
    logger.info("kintsugi: discovered %d shards", len(shards))

    if dry_run:
        for s in shards:
            anomaly_type = "OVERLAP" if s.is_overlap else "GAP"
            trade_info = (
                f"{s.missing_trade_count:,} trades"
                if s.missing_trade_count > 0
                else f"gap={s.gap_hours:.1f}h"
            )
            tid_info = (
                f"tid {s.anchor_last_tid}→{s.resume_first_tid}"
                if s.anchor_last_tid is not None and s.resume_first_tid is not None
                else f"ariadne={s.anchor_last_tid is not None}"
            )
            logger.info(
                "  [DRY-RUN] %s@%d %s %s (%s) ouro_splits=%d",
                s.symbol,
                s.threshold_decimal_bps,
                anomaly_type,
                trade_info,
                tid_info,
                len(s.ouroboros_boundaries),
            )
        return []

    results: list[KintsugiResult] = []

    candidates = _select_repair_candidates(shards, max_repairs)
    max_concurrent = _kintsugi_cfg().max_concurrent_repairs

    if max_concurrent <= 1 or len(candidates) <= 1:
        # Sequential path
        for shard in candidates:
            shard_label = f"{shard.symbol}@{shard.threshold_decimal_bps}"
            _emit_telemetry(
                "repair_start",
                symbol=shard.symbol,
                threshold=shard.threshold_decimal_bps,
                gap_hours=shard.gap_hours,
                gap_end_ms=shard.gap_end_ms,
            )
            result = repair_shard(
                shard,
                include_microstructure=include_microstructure,
                verbose=verbose,
            )

            result_state = (
                "healed"
                if result.healed
                else "clean"
                if not result.error
                else "failed"
            )
            _emit_telemetry(
                "repair_complete",
                symbol=shard.symbol,
                threshold=shard.threshold_decimal_bps,
                result=result_state,
                bars_written=result.bars_written,
                duration_s=round(result.duration_seconds, 1),
                error=result.error,
            )

            # Ping watchdog after each repair to prevent timeout during long repairs
            _sd_notify("WATCHDOG=1")
            _emit_telemetry("watchdog_ping", after_shard=shard_label)

            # Verify repair
            if result.healed:
                verified = verify_repair(shard)
                if not verified:
                    if result.bars_written > 0:
                        # Progress was made — remaining gap will be rediscovered
                        # on next kintsugi pass. Don't mark as failed to avoid
                        # counting toward the 3-attempt rate limit.
                        logger.info(
                            "repair for %s@%d wrote %d bars but gap not fully closed — "
                            "remaining gap will be picked up on next pass",
                            shard.symbol,
                            shard.threshold_decimal_bps,
                            result.bars_written,
                        )
                    else:
                        logger.warning(
                            "repair for %s@%d reported healed but gap persists "
                            "and 0 bars written",
                            shard.symbol,
                            shard.threshold_decimal_bps,
                        )
                        result.healed = False
                        result.error = (
                            "verification failed: gap persists after writing 0 bars"
                        )

            _maybe_log_repair(result)
            results.append(result)
    else:
        # Parallel path: different (symbol, threshold) pairs are fully independent
        # Each shard acquires its own per-pair Redis lock in _repair()
        from concurrent.futures import ThreadPoolExecutor, as_completed

        def _repair_one(shard: Shard) -> KintsugiResult:
            _emit_telemetry(
                "repair_start",
                symbol=shard.symbol,
                threshold=shard.threshold_decimal_bps,
                gap_hours=shard.gap_hours,
                gap_end_ms=shard.gap_end_ms,
            )
            return repair_shard(
                shard,
                include_microstructure=include_microstructure,
                verbose=verbose,
            )

        with ThreadPoolExecutor(
            max_workers=min(max_concurrent, len(candidates)),
            thread_name_prefix="kintsugi-shard",
        ) as executor:
            future_to_shard = {
                executor.submit(_repair_one, shard): shard
                for shard in candidates
            }
            for future in as_completed(future_to_shard):
                shard = future_to_shard[future]
                shard_label = f"{shard.symbol}@{shard.threshold_decimal_bps}"
                try:
                    result = future.result()
                except Exception as exc:
                    logger.exception("shard %s repair failed", shard_label)
                    result = KintsugiResult(
                        shard=shard,
                        healed=False,
                        bars_written=0,
                        duration_seconds=0.0,
                        had_anchor_tid=False,
                        ouroboros_splits=0,
                        error=str(exc),
                    )

                result_state = (
                    "healed"
                    if result.healed
                    else "clean"
                    if not result.error
                    else "failed"
                )
                _emit_telemetry(
                    "repair_complete",
                    symbol=shard.symbol,
                    threshold=shard.threshold_decimal_bps,
                    result=result_state,
                    bars_written=result.bars_written,
                    duration_s=round(result.duration_seconds, 1),
                    error=result.error,
                )

                _sd_notify("WATCHDOG=1")
                _emit_telemetry("watchdog_ping", after_shard=shard_label)

                # Verify repair
                if result.healed:
                    verified = verify_repair(shard)
                    if not verified:
                        if result.bars_written > 0:
                            logger.info(
                                "repair for %s@%d wrote %d bars but gap not fully closed — "
                                "remaining gap will be picked up on next pass",
                                shard.symbol,
                                shard.threshold_decimal_bps,
                                result.bars_written,
                            )
                        else:
                            logger.warning(
                                "repair for %s@%d reported healed but gap persists "
                                "and 0 bars written",
                                shard.symbol,
                                shard.threshold_decimal_bps,
                            )
                            result.healed = False
                            result.error = (
                                "verification failed: gap persists after writing 0 bars"
                            )

                _maybe_log_repair(result)
                results.append(result)

    # Summary (four-state: healed / clean / deferred / failed)
    pass_duration = time.monotonic() - pass_t0
    healed = sum(1 for r in results if r.healed)
    clean = sum(1 for r in results if not r.healed and not r.error)
    deferred = sum(1 for r in results if r.error == "deferred_today_guard")
    failed = sum(1 for r in results if r.error and r.error != "deferred_today_guard")
    total_bars = sum(r.bars_written for r in results)
    n_overlaps = sum(1 for r in results if r.shard.is_overlap)
    n_gaps = len(results) - n_overlaps
    logger.info(
        "kintsugi pass complete: %d healed, %d clean, %d deferred, %d failed, "
        "%d bars written (%d gaps, %d overlaps) (%.1fs)",
        healed,
        clean,
        deferred,
        failed,
        total_bars,
        n_gaps,
        n_overlaps,
        pass_duration,
    )
    _emit_telemetry(
        "pass_complete",
        healed=healed,
        clean=clean,
        deferred=deferred,
        failed=failed,
        total_bars=total_bars,
        n_gaps=n_gaps,
        n_overlaps=n_overlaps,
        duration_s=round(pass_duration, 1),
    )

    # Issue #117-119: Emit KINTSUGI_PASS_COMPLETE hook + quiet Telegram
    try:
        from opendeviationbar.hooks import HookEvent, emit_hook

        emit_hook(
            HookEvent.KINTSUGI_PASS_COMPLETE,
            symbol="*",
            n_discovered=len(shards),
            n_healed=healed,
            n_failed=failed,
            total_bars=total_bars,
            pass_duration=pass_duration,
        )
    except (ImportError, OSError, RuntimeError, ValueError):
        logger.debug("failed to emit KINTSUGI_PASS_COMPLETE hook", exc_info=True)
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram

        total_missing = sum(
            r.shard.missing_trade_count
            for r in results
            if r.shard.missing_trade_count > 0
        )
        remaining = len(shards) - len(results)

        # Build human-readable outcome line
        outcomes = []
        if healed:
            outcomes.append(f"{healed} repaired")
        if deferred:
            outcomes.append(f"{deferred} deferred to tomorrow")
        if clean:
            outcomes.append(f"{clean} already clean")
        if failed:
            outcomes.append(f"{failed} FAILED")
        outcome_str = ", ".join(outcomes) if outcomes else "nothing to do"

        # Build human-readable healed details
        healed_details = ""
        healed_results = [r for r in results if r.healed]
        if healed_results:
            detail_lines = []
            for r in healed_results[:_kintsugi_cfg().max_healed_details]:
                kind = "overlap" if r.shard.is_overlap else "gap"
                detail_lines.append(
                    f"  {r.shard.symbol}@{r.shard.threshold_decimal_bps}: "
                    f"{kind}, {r.bars_written} bars written"
                )
            healed_details = "\n".join(detail_lines)
            if len(healed_results) > _kintsugi_cfg().max_healed_details:
                healed_details += (
                    f"\n  ... +{len(healed_results) - _kintsugi_cfg().max_healed_details} more"
                )
            healed_details = f"\n\n{healed_details}"

        pass_msg = (
            f"Kintsugi healed {healed} anomalies in {pass_duration:.0f}s"
            f"{healed_details}\n\n"
            f"<b>Result:</b> {outcome_str}\n"
            f"<b>Bars written:</b> {total_bars:,} from {total_missing:,} trades\n"
            f"<b>Remaining queue:</b> {remaining:,} shards"
            + _build_context_block("kintsugi")
        )
        send_telegram(pass_msg, disable_notification=True)
    except (ImportError, OSError, RuntimeError):
        logger.debug("failed to send kintsugi pass Telegram", exc_info=True)

    # Notify on real failures only (not deferred or clean)
    if failed > 0:
        _notify_kintsugi_failures(results, pass_duration=pass_duration)

    return results


def _notify_kintsugi_failures(
    results: list[KintsugiResult],
    pass_duration: float = 0.0,
) -> None:
    """Send LOUD Telegram alert for failed Stathera repairs.

    Stathera-only: reports trade-ID anchors and missing trade counts.
    """
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        return

    failed = [r for r in results if r.error and r.error != "deferred_today_guard"]
    if not failed:
        return

    healed_count = sum(1 for r in results if r.healed)

    lines = [
        f"Kintsugi failed to repair {len(failed)} anomalies\n",
        f"This pass: {healed_count} repaired, {len(failed)} failed "
        f"({pass_duration:.0f}s)",
    ]
    for r in failed[:_kintsugi_cfg().max_failures_in_alert]:
        anomaly = "overlap" if r.shard.is_overlap else "gap"
        lines.append(
            f"\n  {r.shard.symbol} @ {r.shard.threshold_decimal_bps} dbps — "
            f"{anomaly} ({r.shard.missing_trade_count:,} trades)\n"
            f"  Reason: <code>{r.error or 'unknown'}</code>"
        )
    if len(failed) > _kintsugi_cfg().max_failures_in_alert:
        lines.append(f"\n  ... and {len(failed) - _kintsugi_cfg().max_failures_in_alert} more")
    lines.append("\nWill retry automatically (max 3 attempts per shard per day).")
    lines.append(
        "If this keeps happening, check: Binance Vision data availability, "
        "REST rate limits, ClickHouse write health."
    )
    lines.append(_build_context_block("kintsugi"))

    try:
        send_telegram("\n".join(lines), disable_notification=False)
    except (OSError, RuntimeError, ConnectionError):
        logger.exception("failed to send kintsugi failure alert")


# ---------------------------------------------------------------------------
# Trailing-edge fill (#238 Phase 2)
# ---------------------------------------------------------------------------



def _trailing_edge_fill(
    *,
    symbol: str | None = None,
    threshold: int | None = None,
    include_microstructure: bool = True,
) -> int:
    """Fill trailing edge: latest cached bar → now for all pairs.

    Absorbed from the former gap-backfill daemon (#238 Phase 2).
    Uses Vision-first + REST fallback via backfill_all_gaps().
    """
    try:
        from opendeviationbar.backfill import backfill_all_gaps, backfill_gap

        if symbol and threshold:
            result = backfill_gap(
                symbol,
                threshold,
                include_microstructure=include_microstructure,
            )
            bars = result.bars_written
        else:
            results = backfill_all_gaps(
                include_microstructure=include_microstructure,
            )
            bars = sum(r.bars_written for r in results)

        if bars > 0:
            logger.info("trailing-edge fill: %d bars written", bars)
    except (RuntimeError, OSError, ConnectionError, ValueError) as e:
        logger.warning("trailing-edge fill failed: %s", e)
        return 0
    else:
        return bars


def _periodic_rectification(
    iteration: int,
    *,
    symbol: str | None = None,
    threshold: int | None = None,
) -> None:
    """Run bar rectification every N iterations (~hourly)."""
    if _kintsugi_cfg().rectification_cycle_interval <= 0:
        return
    if iteration % _kintsugi_cfg().rectification_cycle_interval != 0:
        return

    try:
        from opendeviationbar.backfill import (
            _get_cached_symbol_threshold_pairs,
            rectify_recent_bars,
        )

        pairs = _get_cached_symbol_threshold_pairs()
        if symbol:
            pairs = [(s, t) for s, t in pairs if s == symbol]
        if threshold:
            pairs = [(s, t) for s, t in pairs if t == threshold]

        for sym, thr in pairs:
            try:
                rectified = rectify_recent_bars(sym, thr)
                if rectified > 0:
                    logger.info("rectified %d bars for %s@%d", rectified, sym, thr)
            except (RuntimeError, OSError, ConnectionError, ValueError) as e:
                logger.warning("rectification failed for %s@%d: %s", sym, thr, e)
    except (ImportError, RuntimeError) as e:
        logger.warning("rectification skipped: %s", e)


def _pressure_aware_sleep(
    results: list[KintsugiResult],
    trailing_bars: int,
    interval_active_s: int,
    interval_clean_s: int,
    remaining_shards: int = 0,
) -> int:
    """Compute sleep duration adaptively.

    - **Remaining shards > 0**: sleep 0 (continue immediately)
    - **Activity but queue drained**: ``interval_active_s`` (re-discover soon)
    - **Clean (no activity)**: ``interval_clean_s`` (poll infrequently)
    - **Memory pressure**: back off regardless of queue depth
    """
    import gc

    has_activity = any(r.healed or r.error for r in results) or trailing_bars > 0

    if remaining_shards > 0:
        # Work remains — no reason to idle
        sleep_s = 0
    elif has_activity:
        # Queue drained this pass but there was activity — re-discover soon
        sleep_s = interval_active_s
    else:
        # Fully clean — poll infrequently
        sleep_s = interval_clean_s

    # Memory pressure check (ported from gap-backfill)
    try:
        from opendeviationbar.backfill import _check_memory_watermark, _env_int

        memory_max = _env_int("OPENDEVIATIONBAR_MEMORY_MAX_BYTES", 4 * 1024**3)
        watermark = _check_memory_watermark(memory_max)
        if watermark == "gc":
            gc.collect()
            sleep_s = max(sleep_s, 60)  # At least 60s to let GC take effect
            logger.info("memory pressure 'gc' — sleep extended to %ds", sleep_s)
        elif watermark == "abort":
            sleep_s = 900
            logger.warning("memory pressure 'abort' — sleeping 900s for recovery")
    except (ImportError, RuntimeError):
        pass  # Memory check not available, use default sleep

    return sleep_s


def kintsugi_daemon(
    *,
    interval_clean_s: int = 1800,  # SSoT-OK: daemon polling intervals
    interval_active_s: int = 900,  # SSoT-OK: daemon polling intervals
    max_repairs: int | None = None,
    symbol: str | None = None,
    threshold: int | None = None,
    include_microstructure: bool = True,
) -> None:
    """Long-running daemon with adaptive polling.

    Always processes shards in reverse chronological order (newest first).
    Both gaps and overlaps are repaired via day-recompute (DELETE + Ariadne).

    Runs ``kintsugi_pass()`` repeatedly:
    - Every ``interval_clean_s`` when cache is clean (default 30 min)
    - Every ``interval_active_s`` when shards are found (default 15 min)

    The CLI (``scripts/kintsugi.py``) overrides these defaults with aggressive
    intervals (60s active, 300s clean, 10 repairs) for all deployments.

    Stop with Ctrl+C (SIGINT) or SIGTERM.
    """
    if max_repairs is None:
        max_repairs = _kintsugi_cfg().max_repairs_per_pass
    logger.info(
        "kintsugi daemon starting (clean=%ds, active=%ds, max_repairs=%d)",
        interval_clean_s,
        interval_active_s,
        max_repairs,
    )

    # Issue #126: SIGHUP reloads config (enables runtime ouroboros_mode switching)
    import signal

    def _sighup_handler(*_args: object) -> None:
        from opendeviationbar.config import Settings

        Settings.reload_and_clear()
        new_mode = Settings.get().population.ouroboros_mode
        logger.info("SIGHUP: config reloaded (ouroboros_mode=%s)", new_mode)
        # Issue #134: Notify operators of runtime config change
        try:
            from opendeviationbar.notify.telegram import send_telegram

            send_telegram(
                f"<b>SIGHUP: kintsugi config reloaded</b>\n"
                f"<b>ouroboros_mode:</b> {new_mode}\n"
                f"<b>PID:</b> {os.getpid()}",
                disable_notification=True,
            )
        except (ImportError, OSError):
            logger.debug("SIGHUP: Telegram notification failed (non-fatal)")

    signal.signal(signal.SIGHUP, _sighup_handler)

    # Issue #121: Notify on daemon startup (confirms EnvironmentFile worked)
    _sd_notify("READY=1")
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram

        send_telegram(
            "<b>KINTSUGI STARTED</b>\n"
            f"<b>Interval:</b> clean={interval_clean_s}s, active={interval_active_s}s\n"
            f"<b>Max repairs:</b> {max_repairs}\n"
            f"<b>PID:</b> {os.getpid()}" + _build_context_block("kintsugi"),
            disable_notification=True,
        )
    except (ImportError, OSError, RuntimeError):
        logger.debug("failed to send kintsugi startup Telegram", exc_info=True)

    iteration = 0
    try:
        while True:
            iteration += 1
            logger.info("=== kintsugi iteration %d ===", iteration)

            # Phase 1: Interior shard repairs (Stathera anomalies)
            results = kintsugi_pass(
                symbol=symbol,
                threshold=threshold,
                max_repairs=max_repairs,
                include_microstructure=include_microstructure,
            )

            # Issue #117: Ping systemd watchdog after each pass
            _sd_notify("WATCHDOG=1")

            # Phase 2 (#238): Trailing-edge fill (latest bar → now)
            trailing_bars = _trailing_edge_fill(
                symbol=symbol,
                threshold=threshold,
                include_microstructure=include_microstructure,
            )

            # Phase 2 (#238): Periodic bar rectification (every ~1 hour)
            _periodic_rectification(iteration, symbol=symbol, threshold=threshold)

            # Adaptive sleep: skip idle time when repair queue has remaining work.
            # If we repaired max_repairs shards, the queue likely has more.
            budget_exhausted = len(results) >= max_repairs
            remaining_estimate = max_repairs if budget_exhausted else 0
            sleep_s = _pressure_aware_sleep(
                results, trailing_bars, interval_active_s, interval_clean_s,
                remaining_shards=remaining_estimate,
            )

            if sleep_s > 0:
                logger.info(
                    "kintsugi sleeping %ds (%s)",
                    sleep_s,
                    "active" if (results or trailing_bars > 0) else "clean",
                )
                time.sleep(sleep_s)
            else:
                logger.info("kintsugi: shards remaining — continuing immediately")

    except KeyboardInterrupt:
        logger.info("kintsugi daemon stopped after %d iterations", iteration)


# ---------------------------------------------------------------------------
# Dead-letter replay
# ---------------------------------------------------------------------------


def replay_dead_letters() -> int:
    """Replay any dead-letter Parquet files from sidecar flush failures.

    Returns number of bars successfully replayed.
    """
    from opendeviationbar.sidecar import _DEAD_LETTER_DIR

    if not _DEAD_LETTER_DIR.exists():
        return 0

    import polars as pl

    from opendeviationbar.clickhouse import OpenDeviationBarCache

    total_replayed = 0
    for path in sorted(_DEAD_LETTER_DIR.glob("*.parquet")):
        parts = path.stem.split("_")
        if len(parts) < _kintsugi_cfg().min_dead_letter_parts:
            logger.warning("skipping malformed dead-letter: %s", path)
            continue

        symbol = parts[0]
        threshold = int(parts[1])

        try:
            bars_df = pl.read_parquet(path)
            # Issue #158: Stathera source tagging
            from opendeviationbar import __version__

            source_version = f"{__version__}+kintsugi"
            with OpenDeviationBarCache() as cache:
                written = cache.store_bars_batch(
                    symbol=symbol,
                    threshold_decimal_bps=threshold,
                    bars=bars_df,
                    version=source_version,
                    overlap_strategy="replace",  # Issue #158: kintsugi is authoritative
                )
            total_replayed += written
            path.unlink()
            logger.info(
                "replayed dead-letter %s: %d bars",
                path.name,
                written,
            )
        except ValueError as e:
            # (#264) Pre-write gate filters cross-midnight bars.  With
            # the filter-not-raise change, this path only fires for
            # dead letters where ALL bars are cross-midnight (batch
            # reduced to 0 rows, store returns 0).  Delete the file —
            # the trades will be re-fetched by kintsugi with proper
            # day-boundary splitting.
            logger.warning(
                "dead-letter %s permanently unplayable (day-mode gate), removing: %s",
                path.name,
                e,
            )
            path.unlink(missing_ok=True)
        except (OSError, RuntimeError, ConnectionError) as e:
            logger.warning("failed to replay dead-letter %s: %s", path, e)
            break  # Stop on first failure (ClickHouse likely down)

    if total_replayed > 0:
        logger.info("dead-letter replay: %d bars total", total_replayed)

    return total_replayed


__all__ = [
    "_PRIORITY_SYMBOLS",
    "KintsugiResult",
    "Shard",
    "discover_shards",
    "kintsugi_daemon",
    "kintsugi_pass",
    "repair_shard",
    "replay_dead_letters",
    "verify_repair",
]
