# FILE-SIZE-OK: Watchdog logic (Issue #107) is tightly coupled with main loop
"""Layer 3 live streaming sidecar for open deviation bar construction (Issue #91).

Long-running sidecar that streams live trades from Binance WebSocket,
constructs open deviation bars in real-time with full 58-column microstructure
features, and writes them to ClickHouse.

Architecture:
    Binance WS (per symbol) → Rust StreamManager → Python → ClickHouse

The Rust engine handles WebSocket → trade → bar entirely without GIL.
StreamManager wraps LiveBarEngine with adaptive rate limiting (Issue #162),
trade-ID continuity tracking, and shadow validation diagnostics.

NOTE(#279): Current Rust engine uses per-symbol WebSocket connections.
Empirical testing (2026-03-16) showed combined streams (2x9 hybrid)
deliver 4x throughput with zero recv timeouts. Migration planned in #279.
Python only crosses the boundary per completed bar (~1-6/sec), then
writes to ClickHouse via existing ``store_bars_batch()`` path.

Usage
-----
Start sidecar for specific symbols:

>>> from opendeviationbar.sidecar import run_sidecar, SidecarConfig
>>> config = SidecarConfig(symbols=["BTCUSDT"], thresholds=[250, 500])
>>> run_sidecar(config)  # Blocks until interrupted

Start with gap-fill on startup:

>>> config = SidecarConfig(
...     symbols=["BTCUSDT"], thresholds=[250], gap_fill_on_startup=True,
... )
>>> run_sidecar(config)
"""

from __future__ import annotations

import contextlib
import json
import logging
import os
import queue
import signal
import threading
import time
from collections import defaultdict
from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import parse_qs, urlparse

import polars as pl

from opendeviationbar.config.sidecar import SidecarConfig  # re-export (Issue #194)
from opendeviationbar.phi_accrual import PhiAccrualDetector
from opendeviationbar.sidecar_api import (
    ariadne_lookup,
    catchup_lookup,
    gap_fill_trades,
)

if TYPE_CHECKING:
    from opendeviationbar.config.backfill import BackfillConfig
    from opendeviationbar.config.resilience import ResilienceConfig

logger = logging.getLogger(__name__)

__all__ = ["SidecarConfig", "run_sidecar"]


# =============================================================================
# Checkpoint persistence (Issue #93)
# =============================================================================


def _checkpoint_dir() -> Path:
    """Checkpoint directory for streaming state."""
    from platformdirs import user_cache_dir

    d = Path(user_cache_dir("opendeviationbar")) / "checkpoints"
    d.mkdir(parents=True, exist_ok=True)
    return d


def _checkpoint_path(symbol: str, threshold: int) -> Path:
    return _checkpoint_dir() / f"streaming_{symbol}_{threshold}.json"


def _load_checkpoint(symbol: str, threshold: int) -> dict | None:
    path = _checkpoint_path(symbol, threshold)
    if path.exists():
        data = json.loads(path.read_text())
        # Migrate legacy checkpoint key (timestamp_ms → close_time_ms rename)
        if "last_bar_timestamp_ms" in data and "last_bar_close_time_ms" not in data:
            data["last_bar_close_time_ms"] = data.pop("last_bar_timestamp_ms")
            path.write_text(json.dumps(data))
        return data
    return None


def _save_checkpoint(
    symbol: str,
    threshold: int,
    bar_dict: dict,
    *,
    processor_checkpoint: dict | None = None,
) -> None:
    path = _checkpoint_path(symbol, threshold)
    checkpoint = {
        "symbol": symbol,
        "threshold": threshold,
        "last_bar_close_time_ms": bar_dict.get("close_time_ms"),
        "updated_at": time.time(),
    }
    # Issue #111 (Ariadne): Track trade ID at the streaming boundary
    last_trade_id = bar_dict.get("last_agg_trade_id")
    if last_trade_id and last_trade_id > 0:
        checkpoint["last_agg_trade_id"] = last_trade_id
    if processor_checkpoint is not None:
        checkpoint["processor_checkpoint"] = processor_checkpoint
    # Issue #97: Provenance metadata for forensic reconstruction
    from opendeviationbar.checkpoint import _checkpoint_provenance

    checkpoint["provenance"] = _checkpoint_provenance()
    # Atomic write: temp + fsync + rename (Issue #97)
    tmp = path.with_suffix(".tmp")
    data = json.dumps(checkpoint)
    tmp.write_text(data)
    tmp.replace(path)


# =============================================================================
# ClickHouse write path (Issue #96 Task #144 Phase 3: Arrow streaming path)
# =============================================================================


def _write_bars_batch_to_clickhouse(
    symbol: str,
    threshold: int,
    bar_dicts: list[dict],
    *,
    cache: object | None = None,
) -> None:
    """Write a batch of completed bars to ClickHouse.

    Issue #96 Task #144 Phase 3: Streaming latency optimization.
    Issue #117: Accepts optional cached OpenDeviationBarCache instance.
    """
    from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

    if not bar_dicts:
        return

    # Metadata keys (_symbol, _threshold) already .pop()'d at call site
    bars_df = pl.DataFrame(bar_dicts)

    # Issue #158: Stathera source tagging
    from opendeviationbar import __version__

    source_version = f"{__version__}+sidecar"

    if cache is not None:
        cache.store_bars_batch(
            symbol=symbol,
            threshold_decimal_bps=threshold,
            bars=bars_df,
            version=source_version,
            overlap_strategy="off",  # Issue #158: no pre-write check (L1 lock + L2 dedup handle concurrent writes)
        )
    else:
        with OpenDeviationBarCache() as ch_cache:
            ch_cache.store_bars_batch(
                symbol=symbol,
                threshold_decimal_bps=threshold,
                bars=bars_df,
                version=source_version,
                overlap_strategy="off",  # Issue #158: no pre-write check (L1 lock + L2 dedup handle concurrent writes)
            )


# =============================================================================
# Gap-fill on startup
# =============================================================================


def _gap_fill(
    symbols: list[str],
    thresholds: list[int],
    valid_pairs: set[tuple[str, int]] | None = None,
    *,
    force: bool = False,
) -> dict:
    """Run Layer 2 gap backfill for valid symbol x threshold pairs.

    Returns dict keyed by ``"{symbol}@{threshold}"`` → ``BackfillResult``.
    Used by ``_inject_checkpoints()`` to skip stale checkpoints (Issue #96).

    Issue #120: When ``valid_pairs`` is provided, only those pairs are
    backfilled (skips invalid combos like SHIBUSDT@250).

    Issue #261: When ``force=True``, bypasses the trailing-edge guard in
    ``backfill_gap()`` so that gaps < 30 min are filled on watchdog restart.
    """
    try:
        from opendeviationbar.backfill import backfill_gap
    except ImportError:
        logger.warning("backfill module not available, skipping gap-fill")
        return {}

    pairs = (
        valid_pairs
        if valid_pairs is not None
        else {(sym, thr) for sym in symbols for thr in thresholds}
    )

    # Pre-filter to enabled symbols only (disabled symbols are intentional)
    try:
        from opendeviationbar.symbol_registry import filter_pairs_by_registry

        pairs = filter_pairs_by_registry(pairs)
    except ImportError:
        pass

    results: dict = {}
    for symbol, threshold in sorted(pairs):
        key = f"{symbol}@{threshold}"
        try:
            result = backfill_gap(symbol, threshold_decimal_bps=threshold, force=force)
            results[key] = result
            if result and result.bars_written > 0:
                logger.info(
                    "gap-fill: %s@%d — %d bars, gap was %.0fs",
                    symbol,
                    threshold,
                    result.bars_written,
                    result.gap_seconds,
                )
        except Exception:
            logger.exception("gap-fill failed for %s@%d", symbol, threshold)
    return results


# =============================================================================
# Checkpoint inject/extract helpers (Issue #97)
# =============================================================================

_EXPECTED_KEY_PARTS = 2  # "SYMBOL:THRESHOLD" format


def _inject_checkpoints(
    engine: object,
    symbols: list[str],
    thresholds: list[int],
    trace_id: str,
    gap_fill_results: dict | None = None,
    valid_pairs: set[tuple[str, int]] | None = None,
) -> tuple[int, set[tuple[str, int]]]:
    """Inject saved processor checkpoints into engine before start.

    Parameters
    ----------
    gap_fill_results
        Dict keyed by ``"{symbol}@{threshold}"`` → ``BackfillResult`` from
        ``_gap_fill()``.  When gap-fill wrote ≥1 bar for a pair, the
        checkpoint for that pair is skipped — the gap-fill data is
        authoritative and the checkpoint's incomplete bar is obsolete
        (Issue #96).
    valid_pairs
        Issue #120: When provided, only inject checkpoints for these pairs.
        Eliminates useless checkpoint file lookups for invalid pairs.
    """
    from opendeviationbar.hooks import HookEvent, emit_hook
    from opendeviationbar.logging import log_checkpoint_event

    pairs = (
        valid_pairs
        if valid_pairs is not None
        else {(sym, thr) for sym in symbols for thr in thresholds}
    )

    loaded = 0
    loaded_pairs: set[tuple[str, int]] = set()
    for symbol, threshold in sorted(pairs):
        # Issue #96: Skip checkpoint when gap-fill already wrote bars
        key = f"{symbol}@{threshold}"
        gf = (gap_fill_results or {}).get(key)
        if gf is not None and getattr(gf, "bars_written", 0) > 0:
            logger.info(
                "skipping checkpoint for %s@%d: gap-fill wrote %d bars",
                symbol,
                threshold,
                gf.bars_written,
            )
            continue

        cp = _load_checkpoint(symbol, threshold)
        if not (cp and cp.get("processor_checkpoint")):
            continue

        # Guard: reject stale checkpoints that point beyond CH's last bar.
        # This happens when bars are manually deleted but checkpoints aren't.
        # The checkpoint's forming bar references trades that no longer have
        # completed bars in CH — injecting it would create a massive gap.
        cp_bar = cp["processor_checkpoint"].get("incomplete_bar", {})
        cp_first_tid = cp_bar.get("first_agg_trade_id", 0)
        try:
            from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

            with OpenDeviationBarCache() as cache:
                _, ch_last_tid = cache.get_latest_bar_with_trade_id(
                    symbol, threshold
                )
            if ch_last_tid and cp_first_tid > 0 and cp_first_tid > ch_last_tid + 1:
                logger.warning(
                    "REJECTING stale checkpoint for %s@%d: checkpoint first_tid=%d "
                    "is beyond CH last_tid=%d — skipping (file preserved for next restart)",
                    symbol,
                    threshold,
                    cp_first_tid,
                    ch_last_tid,
                )
                # Do NOT auto-delete checkpoint files (#286 fix). Deletion causes
                # cascading failures: next restart has zero checkpoints → fresh
                # processor → overlapping bars with CH. The file is preserved; on
                # shutdown, the current session writes fresh checkpoints that will
                # be valid on the next restart.
                continue

            # Note: do NOT delete overlapping bars here — individual bar
            # deletion cascades and creates more problems. The checkpoint's
            # forming bar will produce a bar with the correct first_agg_trade_id
            # and ReplacingMergeTree deduplicates on merge. Any remaining
            # overlaps from multi-restart storms are resolved by kintsugi.
        except Exception:
            pass  # CH unavailable — inject checkpoint anyway (better than nothing)

        try:
            engine.set_checkpoint(  # type: ignore[attr-defined]
                symbol,
                threshold,
                cp["processor_checkpoint"],
            )
            loaded += 1
            loaded_pairs.add((symbol, threshold))
            cp_age = time.time() - cp.get("updated_at", time.time())
            logger.info(
                "injected checkpoint for %s@%d (age=%.0fs)",
                symbol,
                threshold,
                cp_age,
            )
            log_checkpoint_event(
                "sidecar_checkpoint_inject",
                symbol,
                trace_id,
                threshold=threshold,
                checkpoint_age_sec=round(cp_age),
                has_incomplete_bar=bool(
                    cp["processor_checkpoint"].get("has_incomplete_bar")
                ),
            )
            emit_hook(
                HookEvent.SIDECAR_CHECKPOINT_RESTORED,
                symbol=symbol,
                trace_id=trace_id,
                threshold_dbps=threshold,
                has_incomplete_bar=bool(
                    cp["processor_checkpoint"].get("has_incomplete_bar")
                ),
                checkpoint_age_sec=round(cp_age),
            )
        except Exception:
            logger.exception(
                "failed to inject checkpoint for %s@%d",
                symbol,
                threshold,
            )
            log_checkpoint_event(
                "sidecar_checkpoint_inject_failed",
                symbol,
                trace_id,
                threshold=threshold,
            )
    return loaded, loaded_pairs


def _extract_checkpoints(
    engine: object,
    trace_id: str,
    bars_written: int,
) -> None:
    """Extract and save processor checkpoints on shutdown."""
    from opendeviationbar.hooks import HookEvent, emit_hook
    from opendeviationbar.logging import log_checkpoint_event

    try:
        all_checkpoints = engine.collect_checkpoints(timeout_ms=5000)  # type: ignore[attr-defined]
        for key, cp_dict in all_checkpoints.items():
            parts = key.split(":")
            if len(parts) == _EXPECTED_KEY_PARTS:
                sym, thr_str = parts
                thr_int = int(thr_str)
                _save_checkpoint(
                    sym,
                    thr_int,
                    {},
                    processor_checkpoint=cp_dict,
                )
                log_checkpoint_event(
                    "sidecar_checkpoint_extract",
                    sym,
                    trace_id,
                    threshold=thr_int,
                    has_incomplete_bar=bool(cp_dict.get("has_incomplete_bar")),
                )
                emit_hook(
                    HookEvent.SIDECAR_CHECKPOINT_SAVED,
                    symbol=sym,
                    trace_id=trace_id,
                    threshold_dbps=thr_int,
                    has_incomplete_bar=bool(cp_dict.get("has_incomplete_bar")),
                    bars_written_session=bars_written,
                )
        logger.info(
            "saved %d processor checkpoints on shutdown",
            len(all_checkpoints),
        )
    except Exception:
        logger.exception("failed to collect/save processor checkpoints")


# =============================================================================
# Engine creation helper (Issue #107: watchdog restart)
# =============================================================================


def _resilience_cfg() -> ResilienceConfig:
    from opendeviationbar.config.resilience import ResilienceConfig
    return ResilienceConfig()


def _sidecar_cfg() -> SidecarConfig:
    from opendeviationbar.config.sidecar import SidecarConfig
    return SidecarConfig.from_env()


def _backfill_cfg() -> BackfillConfig:
    from opendeviationbar.config.backfill import BackfillConfig
    return BackfillConfig()


def _seed_trade_ids_from_clickhouse(
    engine: object,
    symbols: list[str],
    thresholds: list[int],
) -> None:
    """Seed StreamManager trade-ID tracker from ClickHouse last_agg_trade_id.

    Issue #161: Queries ClickHouse for the most recent last_agg_trade_id per
    (symbol, threshold) and injects it into StreamManager before start().
    This enables gap detection from the very first WS trade after startup.

    Issue #261: Retries up to 3 times with 5s backoff. Without seeding, the
    Rust gap detector has no baseline trade ID and cannot detect gaps on
    the first WS trade after restart.
    """
    _rcfg = _resilience_cfg()
    for attempt in range(_rcfg.seed_max_attempts):
        try:
            from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

            cache = OpenDeviationBarCache()
            seeded = 0
            for symbol in symbols:
                for threshold in thresholds:
                    try:
                        _ts, last_tid = cache.get_latest_bar_with_trade_id(
                            symbol, threshold
                        )
                        if last_tid is not None and last_tid > 0:
                            engine.seed_trade_id(symbol, threshold, last_tid)  # type: ignore[attr-defined]
                            seeded += 1
                    except (OSError, RuntimeError):
                        logger.debug(
                            "trade-ID seed query failed for %s@%d", symbol, threshold
                        )
        except (OSError, RuntimeError, ImportError):
            if attempt < _rcfg.seed_max_attempts - 1:
                logger.warning(
                    "trade-ID seeding attempt %d/%d failed, retrying in %ds",
                    attempt + 1,
                    _rcfg.seed_max_attempts,
                    _rcfg.seed_retry_delay_s,
                )
                time.sleep(_rcfg.seed_retry_delay_s)
            else:
                logger.warning(
                    "trade-ID seeding FAILED after %d attempts — "
                    "Rust gap detector will start unseeded",
                    _rcfg.seed_max_attempts,
                )
                _notify_seeding_failure()
        else:
            if seeded > 0:
                logger.info(
                    "seeded trade-ID continuity for %d pairs from ClickHouse", seeded
                )
            return  # success


def _notify_seeding_failure() -> None:
    """Send Telegram alert when trade-ID seeding fails after all retries."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram

        msg = (
            "<b>⚠ Trade-ID Seeding Failed</b>\n"
            f"All {_resilience_cfg().seed_max_attempts} attempts failed. "
            "Rust gap detector starting unseeded — gaps after restart may not be detected."
            + _build_context_block("sidecar")
        )
        send_telegram(msg, disable_notification=False)
    except Exception:
        logger.exception("failed to send seeding failure notification")


def _inline_startup_backfill(
    _config: SidecarConfig,
    valid_pairs: set[tuple[str, int]],
    _trace_id: str,
) -> None:
    """Fill trailing edge for pairs with missing/stale checkpoints BEFORE engine start.

    Runs BEFORE start_ws() — no WS buffer, no concurrent writers, no ordering
    mistakes. Uses backfill_gap(overlap_strategy="skip") so NO bars are deleted.

    This is the autonomous version of the manual "Stop → Delete → Backfill → Start"
    playbook. See sidecar-restart-zero-overlap-zero-gap-recovery-playbook for context.

    History of failed alternatives (DO NOT regress):
      - reconstruct_today(): OOMs at 2.5G with 18 symbols (580f34b, 04c4e4d)
      - Asclepius fill_gap() on live engine: cascading overlaps (023e947)
      - _gap_fill() on startup: fresh processor boundary mismatch (a20e477)
      - Clean-slate DELETE: midnight-to-now gap (17490c1 oscillation)
    """
    # Step 1: Identify pairs that need backfill (no valid checkpoint)
    needs_backfill: list[tuple[str, int]] = []
    try:
        from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            for symbol, threshold in sorted(valid_pairs):
                cp = _load_checkpoint(symbol, threshold)
                if cp and cp.get("processor_checkpoint"):
                    # Checkpoint exists — check if stale
                    cp_bar = cp["processor_checkpoint"].get("incomplete_bar", {})
                    cp_first_tid = cp_bar.get("first_agg_trade_id", 0)
                    try:
                        _, ch_last_tid = cache.get_latest_bar_with_trade_id(
                            symbol, threshold
                        )
                        if (
                            ch_last_tid
                            and cp_first_tid > 0
                            and cp_first_tid > ch_last_tid + 1
                        ):
                            needs_backfill.append((symbol, threshold))
                        # else: checkpoint is valid, no backfill needed
                    except (OSError, RuntimeError):
                        pass  # Can't check — skip, let _inject_checkpoints handle it
                else:
                    # No checkpoint file or no processor state
                    needs_backfill.append((symbol, threshold))
    except (ImportError, OSError, RuntimeError):
        logger.warning("inline backfill: CH unavailable — skipping (Puck will fill gaps)")
        return

    if not needs_backfill:
        logger.info("inline backfill: all %d pairs have valid checkpoints — skipping", len(valid_pairs))
        return

    # Step 2: Run backfill_gap for each pair that needs it
    logger.info(
        "inline backfill: %d/%d pairs need trailing-edge fill (no valid checkpoint): %s",
        len(needs_backfill),
        len(valid_pairs),
        ", ".join(f"{s}@{t}" for s, t in needs_backfill),
    )

    from opendeviationbar.backfill import backfill_gap

    total_bars = 0
    for symbol, threshold in needs_backfill:
        try:
            result = backfill_gap(
                symbol=symbol,
                threshold_decimal_bps=threshold,
                # "skip" — NEVER delete existing bars. If there are overlaps from a
                # previous session, RMT dedup handles them. Constraint A: no individual
                # bar deletion (8121ee5 cascade). Constraint E: no DELETE operations.
                overlap_strategy="skip",
                force=True,
            )
            bars = getattr(result, "bars_written", 0)
            total_bars += bars
            if bars > 0:
                logger.info("inline backfill: %s@%d → %d bars", symbol, threshold, bars)
        except Exception:
            logger.exception("inline backfill: %s@%d FAILED (non-fatal, Puck will fill)", symbol, threshold)

    if total_bars > 0:
        logger.info("inline backfill: completed — %d total bars across %d pairs", total_bars, len(needs_backfill))
    else:
        logger.info("inline backfill: no trailing edge to fill (CH already current)")


def _heal_day_boundary_gaps(
    engine: object,
    valid_pairs: set[tuple[str, int]],
) -> None:
    """Deterministic day-boundary gap heal on startup (#286).

    Checks ClickHouse for Stathera gaps in today's bars. For each gap found,
    submits fill_gap() to the live engine. Covers ALL edge cases: manual bar
    deletions, stale checkpoints, multi-restart scenarios, and any situation
    where today's data has trade-ID discontinuities.

    Uses the live engine's fill_gap() — rate-limited parallel REST fetch,
    checkpoint-restored processor state, correct bar boundaries.
    """
    try:
        from opendeviationbar.clickhouse.cache import OpenDeviationBarCache
        from opendeviationbar.clickhouse.stathera_query import (
            FULL_TABLE,
            per_pair_filter,
            per_pair_window,
        )
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode
        from opendeviationbar.symbol_registry import filter_pairs_by_registry

        ouroboros_mode = get_operational_ouroboros_mode()
        cache = OpenDeviationBarCache()
        pairs = filter_pairs_by_registry(valid_pairs)
        total_healed = 0

        for symbol, threshold in sorted(pairs):
            try:
                # Query for gaps touching today (including yesterday→today boundary)
                result = cache.client.query(
                    f"""
                    SELECT prev_last_tid, first_agg_trade_id, tid_delta
                    FROM (
                        SELECT
                            lagInFrame(last_agg_trade_id, 1) OVER w AS prev_last_tid,
                            first_agg_trade_id,
                            first_agg_trade_id
                              - lagInFrame(last_agg_trade_id, 1) OVER w - 1
                              AS tid_delta,
                            row_number() OVER w AS rn
                        FROM {FULL_TABLE} FINAL
                        {per_pair_filter()}
                        {per_pair_window()}
                    )
                    WHERE tid_delta > 0 AND rn > 1
                    ORDER BY first_agg_trade_id
                    """,
                    parameters={
                        "symbol": symbol,
                        "threshold": int(threshold),
                        "ouroboros": ouroboros_mode,
                    },
                )
                rows = result.result_rows if result.result_rows else []
                for row in rows:
                    anchor_tid, resume_tid, delta = int(row[0]), int(row[1]), int(row[2])
                    if delta > 0:
                        # Gap: fill missing trades via live engine
                        logger.info(
                            "day-boundary heal: %s@%d gap delta=%d — submitting fill_gap",
                            symbol, threshold, delta,
                        )
                        try:
                            engine.fill_gap(symbol, anchor_tid + 1, resume_tid)  # type: ignore[attr-defined]
                            total_healed += 1
                        except Exception:
                            logger.warning(
                                "day-boundary heal: fill_gap failed for %s@%d (non-fatal)",
                                symbol, threshold, exc_info=True,
                            )
                    # Overlaps (td < 0): do NOT attempt individual bar deletion —
                    # it cascades and makes things worse. Legacy overlaps from
                    # multi-restart storms are resolved by kintsugi day-recompute.
            except Exception:
                logger.debug(
                    "day-boundary heal: query failed for %s@%d", symbol, threshold
                )

        if total_healed > 0:
            logger.info("day-boundary heal: submitted %d fill_gap commands", total_healed)
        else:
            logger.info("day-boundary heal: no gaps found — continuity clean")

    except Exception:
        logger.warning("day-boundary heal failed (non-fatal)", exc_info=True)


def _startup_tid_reconciliation(
    engine: object,
    symbols: list[str],
    thresholds: list[int],
) -> None:
    """Puck: reconcile CH last committed TID with checkpoint TID on startup.

    If the checkpoint file's ``last_agg_trade_id`` exceeds CH's (meaning bars
    were computed but CH write failed — exactly the CH-blip scenario), submit
    ``fill_gap()`` to re-fetch and reprocess those trades.
    """
    try:
        from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

        cache = OpenDeviationBarCache()
        for symbol in symbols:
            for threshold in thresholds:
                try:
                    cp = _load_checkpoint(symbol, threshold)
                    cp_tid = (cp or {}).get("last_agg_trade_id")
                    if not cp_tid or cp_tid <= 0:
                        continue

                    _ts, ch_last_tid = cache.get_latest_bar_with_trade_id(
                        symbol, threshold
                    )
                    if ch_last_tid is None or ch_last_tid <= 0:
                        continue

                    if cp_tid > ch_last_tid:
                        gap_size = cp_tid - ch_last_tid - 1
                        if gap_size > 0:
                            logger.warning(
                                "puck: startup TID reconciliation — "
                                "checkpoint_tid=%d > CH_tid=%d for %s@%d "
                                "(gap=%d trades), submitting fill_gap",
                                cp_tid,
                                ch_last_tid,
                                symbol,
                                threshold,
                                gap_size,
                            )
                            engine.fill_gap(symbol, ch_last_tid + 1, cp_tid)  # type: ignore[attr-defined]
                except (OSError, RuntimeError, AttributeError):
                    logger.debug(
                        "puck: TID reconciliation skipped for %s@%d",
                        symbol,
                        threshold,
                        exc_info=True,
                    )
    except (OSError, RuntimeError, ImportError):
        logger.debug("puck: startup TID reconciliation skipped (CH unavailable)")


def _create_engine(
    config: SidecarConfig,
    trace_id: str,
    gap_fill_results: dict | None,
    valid_pairs: set[tuple[str, int]] | None = None,
) -> tuple[object, int]:
    """Create and configure a streaming engine, returning (engine, checkpoints_loaded).

    Issue #161: Replaced LiveBarEngine with StreamManager for:
    - Adaptive rate limiting on REST gap-fill (Issue #162)
    - Trade-ID continuity tracking seeded from ClickHouse
    - Shadow validation diagnostics

    Issue #178: Feature-flagged OdbEngine (unified hot-path/cold-path engine).
    Set OPENDEVIATIONBAR_ENGINE=unified to opt in. Default: legacy (StreamManager).
    """
    import os

    # Issue #128: Wire per-feature computation toggles from Settings
    from opendeviationbar.config import Settings

    pop = Settings.get().population
    engine_mode = os.environ.get("OPENDEVIATIONBAR_ENGINE", "legacy")

    if engine_mode == "unified":
        from opendeviationbar._core import OdbEngine

        engine = OdbEngine(
            config.symbols,
            config.thresholds,
            config.include_microstructure,
            compute_tier2=pop.compute_tier2,
            compute_tier3=pop.compute_tier3,
            compute_hurst=pop.compute_hurst or None,
            compute_permutation_entropy=pop.compute_permutation_entropy or None,
        )
        logger.info("Using unified OdbEngine (Issue #178)")
    else:
        from opendeviationbar._core import StreamManager

        engine = StreamManager(
            config.symbols,
            config.thresholds,
            config.include_microstructure,
            compute_tier2=pop.compute_tier2,
            compute_tier3=pop.compute_tier3,
            compute_hurst=pop.compute_hurst or None,
            compute_permutation_entropy=pop.compute_permutation_entropy or None,
        )

    # Issue #286: Start WS connections FIRST — buffer trades immediately.
    # This captures trades from the first millisecond of startup while we
    # load checkpoints and seed gap detectors. Zero-gap transition.
    if hasattr(engine, "start_ws"):
        engine.start_ws()
        logger.info("WS connections started (buffered mode) — loading checkpoints next")

    # Issue #161: Seed trade-ID continuity from ClickHouse before start.
    # This enables gap detection from the very first buffered WS trade.
    _seed_trade_ids_from_clickhouse(engine, config.symbols, config.thresholds)

    loaded, loaded_pairs = _inject_checkpoints(
        engine,
        config.symbols,
        config.thresholds,
        trace_id,
        gap_fill_results=gap_fill_results,
        valid_pairs=valid_pairs,
    )

    # (#286 v13.28+) Rust-native historical fill: when checkpoints are missing,
    # fill_from_rest() creates processors, feeds REST trades through them, and
    # preserves the SAME processors for start(). Zero transition gap because the
    # processor that processed the REST trades is the SAME one that processes WS.
    #
    # This replaces the Python _inline_startup_backfill() which used a SEPARATE
    # fresh processor via backfill_gap() — causing a forming-bar abandonment gap.
    #
    # Architecture: WS buffers in parallel while REST catches up. At the junction,
    # trade-ID dedup skips WS trades already processed by REST. Puck fills any
    # micro-gap at the boundary.
    # (#286 v13.29+) Rust fill_from_rest() — zero-gap restart via same-processor
    # REST→WS transition. Now uses chunked REST pagination (50K trades/chunk)
    # instead of loading everything into a Vec. Same processors handle both REST
    # and WS trades — zero forming-bar loss, zero transition gap.
    if hasattr(engine, "fill_from_rest") and loaded == 0:
        try:
            rest_bars = engine.fill_from_rest()
            logger.info(
                "fill_from_rest: %d bars produced (processors preserved for WS)",
                rest_bars,
            )
        except (RuntimeError, OSError) as e:
            logger.warning("fill_from_rest failed: %s (falling back to WS-only start)", e)

    # (#286 fix) Build minimum-TID guard for pairs that started without a
    # checkpoint. A fresh processor produces bars from the first WS trade,
    # which may have a first_agg_trade_id WITHIN CH's existing range →
    # overlapping bars. Guard: drop any bar whose first_agg_trade_id is
    # ≤ the CH seed for that pair (those trades are already covered).
    min_tid_guard: dict[tuple[str, int], int] = {}
    all_pairs = (
        valid_pairs
        if valid_pairs is not None
        else {(sym, thr) for sym in config.symbols for thr in config.thresholds}
    )
    unloaded_pairs = all_pairs - loaded_pairs
    if unloaded_pairs:
        try:
            from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

            with OpenDeviationBarCache() as cache:
                for symbol, threshold in unloaded_pairs:
                    try:
                        _, ch_last_tid = cache.get_latest_bar_with_trade_id(
                            symbol, threshold
                        )
                        if ch_last_tid and ch_last_tid > 0:
                            min_tid_guard[(symbol, threshold)] = ch_last_tid
                    except (OSError, RuntimeError):
                        pass
        except (OSError, RuntimeError, ImportError):
            pass
        if min_tid_guard:
            logger.info(
                "min-TID guard active for %d pairs (no checkpoint loaded): %s",
                len(min_tid_guard),
                ", ".join(f"{s}@{t}" for s, t in sorted(min_tid_guard)),
            )

    # start() now drains the WS buffer seamlessly — trades buffered since
    # start_ws() are processed through checkpoint-restored processors.
    engine.start()
    return engine, loaded, min_tid_guard


# =============================================================================
# Watchdog notification (Issue #107)
# =============================================================================


def _notify_watchdog_trigger(
    stale_seconds: float,
    trades_received: int,
    restart_count: int,
) -> None:
    """Send Telegram alert when watchdog detects stale trade flow."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        logger.warning("telegram module not available, skipping watchdog notification")
        return

    try:
        msg = (
            "<b>WATCHDOG TRIGGER</b>\n"
            f"<b>Stale:</b> {stale_seconds:.0f}s (no trade increment)\n"
            f"<b>Total trades:</b> {trades_received:,}\n"
            f"<b>Restart:</b> #{restart_count + 1}\n"
            "<b>Likely cause:</b> WS silently disconnected or exchange maintenance\n"
            "<b>Next:</b> Sidecar will restart engine + gap-fill from last known trade"
            + _build_context_block("sidecar")
        )
        send_telegram(msg, disable_notification=False)
    except Exception:
        logger.exception("failed to send watchdog Telegram notification")


# =============================================================================
# Lethe / Argus / Charon notification helpers
# =============================================================================


def _notify_lethe_overflow(new_drops: int, total_drops: int) -> None:
    """Lethe: alert when ring buffer drops bars (silent data loss)."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        return
    try:
        msg = (
            "<b>LETHE: RING BUFFER OVERFLOW</b>\n"
            f"<b>New drops:</b> {new_drops:,}\n"
            f"<b>Total drops:</b> {total_drops:,}\n"
            "<b>Cause:</b> Backpressure from slow ClickHouse writes\n"
            "<b>Impact:</b> Bars lost until kintsugi reconciles\n"
            "<b>If recurring:</b> Check CH write latency or increase buffer size"
            + _build_context_block("sidecar")
        )
        send_telegram(msg, disable_notification=False)
    except Exception:
        logger.exception("Lethe: failed to send overflow alert")


def _notify_argus_stale(stale_symbols: list[str], timeout_s: int) -> None:
    """Argus: alert when symbols go silent beyond timeout."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        return
    try:
        symbols_str = ", ".join(sorted(stale_symbols))
        msg = (
            "<b>ARGUS: SYMBOL SILENCE</b>\n"
            f"<b>Symbols:</b> {symbols_str}\n"
            f"<b>Silent for:</b> &gt;{timeout_s}s\n"
            "<b>Possible causes:</b> (1) WS feed dropped for subset, "
            "(2) exchange delisted symbol, (3) extreme low volume period\n"
            "<b>Check:</b> WebSocket connectivity, exchange status"
            + _build_context_block("sidecar")
        )
        send_telegram(msg, disable_notification=False)
    except Exception:
        logger.exception("Argus: failed to send stale symbol alert")


def _notify_charon_replay(bars_replayed: int, trigger: str) -> None:
    """Charon: informational alert when dead-letter bars are replayed."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        return
    try:
        msg = (
            "<b>CHARON: DEAD-LETTER REPLAY</b>\n"
            f"<b>Bars replayed:</b> {bars_replayed:,}\n"
            f"<b>Trigger:</b> {trigger}\n"
            "<b>Context:</b> Bars were dead-lettered due to prior CH write failures. "
            "Replay succeeded — data integrity restored."
            + _build_context_block("sidecar")
        )
        send_telegram(msg, disable_notification=True)
    except Exception:
        logger.exception("Charon: failed to send replay notification")


# =============================================================================
# Watchdog error recovery (Issue #107, TGI #2)
# =============================================================================


def _restart_engine_with_recovery(
    engine: object,
    config: SidecarConfig,
    trace_id: str,
    bars_written: int,
    restart_attempt: int,
    valid_pairs: set[tuple[str, int]] | None = None,
) -> tuple[object | None, bool, dict[tuple[str, int], int]]:
    """Restart engine with timeout guards and error recovery.

    Each shutdown step has isolated error handling to prevent cascading failures.

    Returns:
        (new_engine, success_bool, min_tid_guard) - engine is None if restart failed
    """
    try:
        # Step 1: Graceful shutdown attempt (may fail or hang)
        try:
            engine.stop()
            logger.info("engine.stop() completed")
        except (RuntimeError, OSError, TimeoutError, ValueError) as e:
            logger.warning("engine.stop() raised exception: %s", e)

        # Step 2: Extract checkpoints (critical for state recovery)
        try:
            _extract_checkpoints(engine, trace_id, bars_written)
            logger.info("checkpoint extraction succeeded")
        except (OSError, ValueError, RuntimeError) as e:
            logger.warning("checkpoint extraction failed: %s", e)

        # Step 3: Force shutdown
        try:
            engine.shutdown()
            logger.info("engine.shutdown() completed")
        except (RuntimeError, OSError, TimeoutError, ValueError) as e:
            logger.warning("engine.shutdown() raised: %s", e)

        # Step 4: Fresh gap-fill before creating new engine (Issue #115 Phase 1b)
        # Issue #261: force=True bypasses trailing-edge guard so gaps < 30 min
        # (caused by brief CH blips) are filled on restart.
        logger.info(
            "running fresh gap-fill after watchdog restart #%d",
            restart_attempt,
        )
        fresh_gap_fill = _gap_fill(
            config.symbols,
            config.thresholds,
            valid_pairs=valid_pairs,
            force=True,
        )

        # Step 5: Create new engine with fresh gap-fill results
        new_engine, _, new_min_tid_guard = _create_engine(
            config,
            trace_id,
            fresh_gap_fill,
            valid_pairs=valid_pairs,
        )
        logger.info("engine restart #%d succeeded", restart_attempt)
    except Exception:
        logger.exception("WATCHDOG RESTART FAILED (attempt #%d)", restart_attempt)
        return None, False, {}
    else:
        return new_engine, True, new_min_tid_guard


def _notify_restart_failure(
    restart_number: int,
    consecutive_timeouts: int,
    stale_seconds: float,
) -> None:
    """Alert ops when watchdog restart fails via Telegram."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        logger.warning("telegram module not available for restart failure alert")
        return

    try:
        msg = (
            "<b>WATCHDOG RESTART FAILED</b>\n"
            f"<b>Attempt:</b> {restart_number}\n"
            f"<b>Consecutive timeouts:</b> {consecutive_timeouts}\n"
            f"<b>Stale:</b> {stale_seconds:.0f}s\n"
            "<b>Action:</b> Sidecar will exit. systemd should auto-restart.\n"
            "<b>Check:</b> <code>journalctl --user "
            "-u opendeviationbar-sidecar -n 50</code>" + _build_context_block("sidecar")
        )
        send_telegram(msg, disable_notification=False)
        logger.info("restart failure alert sent via Telegram")
    except Exception:
        logger.exception("failed to send restart failure alert")


# =============================================================================
# Issue #109: Health check HTTP endpoint
# =============================================================================

# Mutable container for cached engine metrics, updated by the main sidecar loop.
# The health handler reads this dict — avoids PyO3 borrow conflicts with next_bar().
_health_state: dict[str, object] = {"metrics": {}}

# Issue #216/#217: Rate limiting to prevent Binance API budget exhaustion.


class _TokenBucket:
    """Token bucket rate limiter with burst support (#222).

    Inspired by PyrateLimiter's leaky bucket design but deliberately minimal
    (20 lines vs 300+). Supports weighted requests and monotonic clock-jump
    safety. See ~/fork-tools/PyrateLimiter for the full-featured alternative.
    """

    __slots__ = ("_burst", "_last_refill", "_lock", "_name", "_rate", "_tokens")

    def __init__(self, rate: float, burst: int, *, name: str = "unnamed") -> None:
        self._rate = rate
        self._burst = burst
        self._tokens = float(burst)
        self._last_refill = time.monotonic()
        self._lock = threading.Lock()
        self._name = name

    def try_acquire(self, weight: int = 1) -> bool:
        with self._lock:
            now = time.monotonic()
            # Guard against monotonic clock backward jump (macOS sleep/wake)
            elapsed = max(0.0, now - self._last_refill)
            self._tokens = min(
                self._burst,
                self._tokens + elapsed * self._rate,
            )
            self._last_refill = now
            if self._tokens >= weight:
                self._tokens -= weight
                return True
            logger.debug(
                "rate limited [%s]: tokens=%.1f, weight=%d",
                self._name,
                self._tokens,
                weight,
            )
            return False


_gap_fill_bucket = _TokenBucket(rate=1.0, burst=5, name="gap-fill")
# Ariadne uses burst=2 (single lookups, not paginated like gap-fill)
_ariadne_bucket = _TokenBucket(rate=2.0, burst=2, name="ariadne")

_catchup_bucket = _TokenBucket(rate=0.2, burst=1, name="catchup")

# Issue #163: SSE broadcast infrastructure.
# Each connected SSE client gets a queue. Bars are pushed via non-blocking put_nowait().
# Clients that can't keep up get events dropped (backpressure via bounded queue).
_sse_clients: list[queue.Queue[str | None]] = []
_sse_clients_lock = threading.Lock()
_SSE_CLIENT_QUEUE_SIZE = 256  # Max buffered events per client before drop


def _register_sse_client() -> queue.Queue[str | None]:
    """Register a new SSE client and return its event queue."""
    q: queue.Queue[str | None] = queue.Queue(maxsize=_SSE_CLIENT_QUEUE_SIZE)
    with _sse_clients_lock:
        _sse_clients.append(q)
    return q


def _unregister_sse_client(q: queue.Queue[str | None]) -> None:
    """Remove a disconnected SSE client."""
    with _sse_clients_lock, contextlib.suppress(ValueError):
        _sse_clients.remove(q)


def _broadcast_bar_event(symbol: str, threshold: int, bar: dict[str, Any]) -> None:
    """Broadcast a completed bar to all connected SSE clients.

    Non-blocking: clients that can't keep up have events silently dropped.
    """
    event_id = f"{symbol}:{threshold}:{bar.get('close_time_ms', 0)}"
    data = json.dumps({"symbol": symbol, "threshold": threshold, **bar})
    message = f"event: bar\nid: {event_id}\ndata: {data}\n\n"

    with _sse_clients_lock:
        for client_queue in _sse_clients:
            with contextlib.suppress(queue.Full):
                client_queue.put_nowait(message)


def _broadcast_forming_event(symbol: str, threshold: int, bar: dict[str, Any]) -> None:
    """Broadcast a forming (incomplete) bar to all connected SSE clients (Issue #214).

    Uses event type 'forming' (distinct from 'bar') so consumers can route by type.
    Stale forming bars (no recent trades) are filtered by the daemon thread before
    calling this function.
    """
    event_id = f"{symbol}:{threshold}:forming:{bar.get('close_time_ms', 0)}"
    data = json.dumps(
        {"symbol": symbol, "threshold": threshold, "is_forming": True, **bar}
    )
    message = f"event: forming\nid: {event_id}\ndata: {data}\n\n"

    with _sse_clients_lock:
        for client_queue in _sse_clients:
            with contextlib.suppress(queue.Full):
                client_queue.put_nowait(message)


def _broadcast_checkpoint_event(checkpoints: dict[str, Any]) -> None:
    """Broadcast processor checkpoints to all connected SSE clients (Issue #215).

    Sent periodically (default 5min) and on graceful shutdown for crash resilience.
    """
    data = json.dumps({"checkpoints": checkpoints})
    message = f"event: checkpoint\nid: checkpoint:{int(time.time())}\ndata: {data}\n\n"

    with _sse_clients_lock:
        for client_queue in _sse_clients:
            with contextlib.suppress(queue.Full):
                client_queue.put_nowait(message)


def _broadcast_gap_event(event: dict[str, Any]) -> None:
    """Broadcast a gap detection event to all connected SSE clients (Phase 1c).

    Non-blocking: clients that can't keep up have events silently dropped.
    """
    event_id = f"gap:{event.get('symbol', '')}:{event.get('detected_at_ms', 0)}"
    data = json.dumps(event)
    message = f"event: gap_detected\nid: {event_id}\ndata: {data}\n\n"

    with _sse_clients_lock:
        for client_queue in _sse_clients:
            with contextlib.suppress(queue.Full):
                client_queue.put_nowait(message)


def _start_forming_bar_broadcaster(
    watcher: object,
    valid_pairs: set[tuple[str, int]],
    interval_s: float = 1.0,
    stale_s: float = 60.0,
) -> threading.Event:
    """Start a daemon thread that broadcasts forming bars at regular intervals (Issue #214).

    Reads forming bar snapshots from the engine's watch channels at `interval_s` Hz.
    Skips broadcasting if no SSE clients are connected or if the forming bar's last
    trade timestamp is older than `stale_s` seconds.

    Returns a stop event that can be set to terminate the thread.
    """
    stop_event = threading.Event()

    def _loop() -> None:
        consecutive_errors = 0
        while not stop_event.is_set():
            try:
                # Skip if no SSE clients connected
                with _sse_clients_lock:
                    has_clients = bool(_sse_clients)
                if not has_clients:
                    stop_event.wait(interval_s)
                    continue

                forming_bars = watcher.get_forming_bars()
                now_us = int(time.time() * 1_000_000)
                stale_us = int(stale_s * 1_000_000)

                for fb in forming_bars:
                    sym = fb.pop("_symbol", "UNKNOWN")
                    thr = fb.pop("_threshold", 0)
                    last_trade_us = fb.pop("_last_trade_timestamp_us", 0)

                    # Skip invalid pairs
                    if (sym, thr) not in valid_pairs:
                        continue

                    # Skip stale forming bars (no recent trades)
                    if last_trade_us > 0 and (now_us - last_trade_us) > stale_us:
                        continue

                    _broadcast_forming_event(sym, thr, fb)

                consecutive_errors = 0

            except Exception:
                consecutive_errors += 1
                if consecutive_errors >= 10:  # noqa: PLR2004
                    logger.exception(
                        "forming bar broadcaster: %d consecutive failures, backing off",
                        consecutive_errors,
                    )
                    stop_event.wait(min(interval_s * consecutive_errors, 30.0))
                else:
                    logger.warning(
                        "forming bar broadcast error (non-fatal)", exc_info=True
                    )

            stop_event.wait(interval_s)

    thread = threading.Thread(target=_loop, daemon=True, name="forming-bar-broadcaster")
    thread.start()
    logger.info(
        "forming bar broadcaster started (interval=%.1fs, stale=%.0fs)",
        interval_s,
        stale_s,
    )
    return stop_event


def _update_health_metrics(metrics: dict[str, int]) -> None:
    """Cache engine metrics for the /health endpoint (called from main loop)."""
    _health_state["metrics"] = metrics


def _start_health_server(port: int, sse_enabled: bool = True) -> HTTPServer | None:
    """Start a lightweight HTTP server for health checks and SSE bar events.

    Endpoints:
        GET /health          — JSON health status (200/503/500)
        GET /events/bars     — SSE stream of completed bars (Issue #163)
            ?symbols=BTCUSDT,ETHUSDT  (optional filter)
            ?thresholds=250,500       (optional filter)

    Returns the server instance (for shutdown) or None if port=0 (disabled).
    """
    if port == 0:
        return None

    class SidecarHandler(BaseHTTPRequestHandler):
        def do_GET(self) -> None:
            parsed = urlparse(self.path)
            if parsed.path == "/health":
                self._handle_health()
            elif parsed.path == "/events/bars" and sse_enabled:
                self._handle_sse(parsed.query)
            elif parsed.path.startswith("/ariadne/"):
                self._handle_ariadne(parsed.path, parsed.query)
            elif parsed.path == "/trades/gap-fill":
                self._handle_gap_fill(parsed.query)
            elif parsed.path.startswith("/catchup/"):
                self._handle_catchup(parsed.path, parsed.query)
            else:
                self.send_error(404)

        def _handle_health(self) -> None:
            try:
                from opendeviationbar.health_checks import run_all_checks

                results = run_all_checks()
                bool_checks = {k: v for k, v in results.items() if isinstance(v, bool)}
                all_pass = all(bool_checks.values())

                engine_metrics = _health_state.get("metrics", {})

                resp: dict[str, object] = {
                    "status": "healthy" if all_pass else "unhealthy",
                    "checks": results,
                    "engine": engine_metrics,
                }
                body = json.dumps(resp).encode()
                code = 200 if all_pass else 503
            except (ImportError, OSError, RuntimeError, ValueError):
                body = json.dumps(
                    {"status": "error", "checks": {}},
                ).encode()
                code = 500

            self.send_response(code)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)

        def _handle_sse(self, query_string: str) -> None:  # noqa: PLR0915
            """Stream completed and forming bars as Server-Sent Events (Issue #163, #214)."""
            params = parse_qs(query_string)
            from opendeviationbar.symbol_registry import normalize_symbol

            symbol_filter = set()
            threshold_filter = set()
            if "symbols" in params:
                symbol_filter = {
                    normalize_symbol(s)
                    for s in params["symbols"][0].split(",")
                    if s.strip()
                }
            if "thresholds" in params:
                threshold_filter = {
                    int(t.strip())
                    for t in params["thresholds"][0].split(",")
                    if t.strip()
                }

            self.send_response(200)
            self.send_header("Content-Type", "text/event-stream")
            self.send_header("Cache-Control", "no-cache")
            self.send_header("Connection", "keep-alive")
            self.send_header("X-Accel-Buffering", "no")
            self.end_headers()

            client_queue = _register_sse_client()
            heartbeat_interval = 15.0
            last_heartbeat = time.monotonic()

            try:
                while True:
                    try:
                        message = client_queue.get(timeout=1.0)
                    except queue.Empty:
                        message = None

                    if message is not None:
                        # Apply client-side filters
                        if symbol_filter or threshold_filter:
                            try:
                                # Extract symbol/threshold from event data line
                                data_line = ""
                                for line in message.split("\n"):
                                    if line.startswith("data: "):
                                        data_line = line[6:]
                                        break
                                if data_line:
                                    evt = json.loads(data_line)
                                    if (
                                        symbol_filter
                                        and evt.get("symbol") not in symbol_filter
                                    ):
                                        continue
                                    if (
                                        threshold_filter
                                        and evt.get("threshold") not in threshold_filter
                                    ):
                                        continue
                            except (json.JSONDecodeError, KeyError):
                                pass  # Send unfiltered on parse error

                        self.wfile.write(message.encode())
                        self.wfile.flush()

                    # Heartbeat to detect dead connections
                    now = time.monotonic()
                    if now - last_heartbeat >= heartbeat_interval:
                        self.wfile.write(b": heartbeat\n\n")
                        self.wfile.flush()
                        last_heartbeat = now

            except (BrokenPipeError, ConnectionResetError, OSError):
                pass  # Client disconnected
            finally:
                _unregister_sse_client(client_queue)

        def _parse_symbol_threshold(
            self, path: str, endpoint: str
        ) -> tuple[str, int] | None:
            """Parse /{endpoint}/{symbol}/{threshold} path. Returns (symbol, threshold) or None on error."""
            parts = path.strip("/").split("/")
            if len(parts) != 3:  # noqa: PLR2004
                self._send_json(
                    400, {"error": f"expected /{endpoint}/{{symbol}}/{{threshold}}"}
                )
                return None
            from opendeviationbar.symbol_registry import normalize_symbol

            symbol = normalize_symbol(parts[1])
            try:
                threshold = int(parts[2])
            except ValueError:
                self._send_json(400, {"error": "threshold must be an integer"})
                return None
            return (symbol, threshold)

        def _handle_ariadne(self, path: str, query_string: str = "") -> None:
            """GET /ariadne/{symbol}/{threshold} — cascading trade-ID lookup (Issue #217)."""
            if not _ariadne_bucket.try_acquire():
                self._send_json(429, {"error": "rate limited, retry after 0.5s"})
                return

            parsed = self._parse_symbol_threshold(path, "ariadne")
            if parsed is None:
                return
            symbol, threshold = parsed

            params = parse_qs(query_string)
            committed_only = params.get("committed_only", ["false"])[0].lower() in (
                "true",
                "1",
            )

            try:
                # Pass engine reference from module-level _health_state
                engine_ref = _health_state.get("engine")
                result = ariadne_lookup(
                    symbol, threshold, engine=engine_ref, committed_only=committed_only
                )
                self._send_json(200, result)
            except ValueError as e:
                self._send_json(400, {"error": str(e)})
            except Exception:
                logger.exception("ariadne endpoint failed for %s@%d", symbol, threshold)
                self._send_json(500, {"error": "internal server error"})

        def _handle_gap_fill(self, query_string: str) -> None:
            """GET /trades/gap-fill?symbol=X&from_agg_id=N&limit=M (Issue #216)."""
            if not _gap_fill_bucket.try_acquire():
                self._send_json(429, {"error": "rate limited, retry after 1s"})
                return

            params = parse_qs(query_string)
            symbol = params.get("symbol", [None])[0]
            from_agg_id_str = params.get("from_agg_id", [None])[0]
            limit_str = params.get("limit", ["1000"])[0]

            if not symbol:
                self._send_json(400, {"error": "missing 'symbol' parameter"})
                return
            if not from_agg_id_str:
                self._send_json(400, {"error": "missing 'from_agg_id' parameter"})
                return

            try:
                from_agg_id = int(from_agg_id_str)
                limit = max(1, min(int(limit_str), 5000))
            except ValueError:
                self._send_json(
                    400, {"error": "from_agg_id and limit must be integers"}
                )
                return

            try:
                from opendeviationbar.symbol_registry import normalize_symbol

                result = gap_fill_trades(normalize_symbol(symbol), from_agg_id, limit)
                self._send_json(200, result)
            except ValueError as e:
                self._send_json(400, {"error": str(e)})
            except Exception:
                logger.exception("gap-fill endpoint failed for %s", symbol)
                self._send_json(502, {"error": "upstream data fetch failed"})

        def _handle_catchup(self, path: str, query_string: str = "") -> None:
            """GET /catchup/{symbol}/{threshold} — server-side gap-fill (#228)."""
            if not _catchup_bucket.try_acquire():
                self._send_json(429, {"error": "rate limited, retry after 5s"})
                return

            parsed = self._parse_symbol_threshold(path, "catchup")
            if parsed is None:
                return
            symbol, threshold = parsed

            params = parse_qs(query_string)
            try:
                max_trades = max(
                    1,
                    min(
                        int(params.get("max_trades", ["50000"])[0]),
                        _backfill_cfg().paginated_max_trades,
                    ),
                )
            except ValueError:
                self._send_json(400, {"error": "max_trades must be an integer"})
                return

            try:
                engine_ref = _health_state.get("engine")
                result = catchup_lookup(
                    symbol, threshold, max_trades=max_trades, engine=engine_ref
                )
                self._send_json(200, result)
            except ValueError as e:
                self._send_json(400, {"error": str(e)})
            except Exception:
                logger.exception("catchup endpoint failed for %s@%d", symbol, threshold)
                self._send_json(500, {"error": "internal server error"})

        def _send_json(self, code: int, data: object) -> None:
            """Send a JSON response."""
            body = json.dumps(data).encode()
            self.send_response(code)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)

        def log_message(self, format: str, *args: object) -> None:  # noqa: A002
            # Suppress per-request access logs (too noisy for health probes)
            pass

    try:
        from http.server import ThreadingHTTPServer

        server = ThreadingHTTPServer(("0.0.0.0", port), SidecarHandler)
        thread = threading.Thread(target=server.serve_forever, daemon=True)
        thread.start()
        endpoints = (
            "/health, /ariadne/{sym}/{thr}, /trades/gap-fill, /catchup/{sym}/{thr}"
        )
        if sse_enabled:
            endpoints += ", /events/bars (SSE)"
        logger.info("sidecar HTTP server listening on port %d [%s]", port, endpoints)
    except OSError as e:
        logger.warning(
            "failed to start health server on port %d: %s",
            port,
            e,
        )
        return None
    else:
        return server


# =============================================================================
# Dead-letter + retry helpers (Issue #115: Kintsugi sidecar hardening)
# =============================================================================

_DEAD_LETTER_DIR = Path("/tmp/opendeviationbar-dead-letter")
_STARTUP_SYMBOL_PREVIEW = 5  # Max symbols shown in startup Telegram
_ARGUS_TRADE_RECENCY_S = 60  # Trades within this age = "quiet" (not "dead")
_BURST_DRAIN_MAX = 256  # Issue #273: max bars to drain atomically from ring buffer


def _niffler_dead_letter(
    symbol: str,
    threshold: int,
    bar_dicts: list[dict[str, Any]],
) -> Path | None:
    """Niffler: serialize failed bars to a dead-letter Parquet file.

    Extracts TID range for targeted kintsugi repair if Niffler + Charon both fail.
    Returns the path written, or None on failure.
    """
    # Phase 5: Extract TID range for diagnostics
    min_first_tid = None
    max_last_tid = None
    for bd in bar_dicts:
        ftid = bd.get("first_agg_trade_id")
        ltid = bd.get("last_agg_trade_id")
        if ftid is not None:
            min_first_tid = ftid if min_first_tid is None else min(min_first_tid, ftid)
        if ltid is not None:
            max_last_tid = ltid if max_last_tid is None else max(max_last_tid, ltid)

    try:
        import polars as pl

        _DEAD_LETTER_DIR.mkdir(parents=True, exist_ok=True)
        ts = int(time.time())
        path = _DEAD_LETTER_DIR / f"{symbol}_{threshold}_{ts}.parquet"
        # Metadata keys (_symbol, _threshold) already .pop()'d at call site
        pl.DataFrame(bar_dicts).write_parquet(path)
    except (OSError, RuntimeError, ValueError):
        logger.exception(
            "niffler: failed to write dead-letter for %s@%d",
            symbol,
            threshold,
        )
        return None
    else:
        logger.warning(
            "niffler: dead-lettered %d bars to %s "
            "(tid_range=[%s, %s])",
            len(bar_dicts),
            path,
            min_first_tid,
            max_last_tid,
        )
        return path


def _niffler_notify_dead_letter(symbol: str, threshold: int, n_bars: int, path: Path) -> None:
    """Niffler: send Telegram alert when bars are dead-lettered."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        return
    try:
        msg = (
            "<b>NIFFLER: DEAD LETTER</b>\n"
            f"<b>Symbol:</b> {symbol}@{threshold}\n"
            f"<b>Bars lost:</b> {n_bars:,}\n"
            f"<b>Path:</b> <code>{path}</code>\n"
            "<b>Cause:</b> ClickHouse write failed after 3 retries\n"
            "<b>Replay:</b> Niffler immediate (sub-5s) + Charon backup (5 min)\n"
            '<b>Manual:</b> <code>python -c "'
            "from opendeviationbar.kintsugi import replay_dead_letters; "
            'replay_dead_letters()"</code>' + _build_context_block("sidecar")
        )
        send_telegram(msg, disable_notification=False)
    except Exception:
        logger.exception("niffler: failed to send Telegram alert")


def _niffler_immediate_replay() -> None:
    """Niffler: immediately replay dead-letter files after writing them.

    Spawned in a daemon thread so the main sidecar loop is not blocked.
    Redis write lock prevents concurrent writes. Charon remains as 5-min backup
    in case this immediate replay also fails.
    """
    try:
        from opendeviationbar.kintsugi import replay_dead_letters

        replayed = replay_dead_letters()
        if replayed > 0:
            logger.info("niffler: replayed %d dead-letter bars (immediate)", replayed)
            _notify_charon_replay(replayed, "niffler-immediate")
    except (ImportError, OSError, RuntimeError, ConnectionError):
        logger.warning(
            "niffler: immediate replay failed (Charon will retry in 5 min)",
            exc_info=True,
        )


def _write_batch_with_retry(
    symbol: str,
    threshold: int,
    bar_dicts: list[dict[str, Any]],
    *,
    cache: object | None = None,
) -> bool:
    """Write a batch to ClickHouse with stamina retry + dead-letter fallback.

    Returns True if write succeeded, False if all retries exhausted (dead-lettered).
    Issue #117: Accepts optional cached OpenDeviationBarCache.
    Charon: Uses stamina for production-grade retry (replaces hand-rolled loop).
    """
    import stamina

    try:

        @stamina.retry(
            on=(OSError, RuntimeError, ConnectionError, ValueError),
            attempts=_resilience_cfg().flush_max_retries,
            timeout=30.0,
        )
        def _stamina_write() -> None:
            _write_bars_batch_to_clickhouse(symbol, threshold, bar_dicts, cache=cache)

        _stamina_write()
    except (OSError, RuntimeError, ConnectionError, ValueError):
        # All retries exhausted — dead-letter
        logger.exception(
            "flush FAILED after %d retries for %s@%d (%d bars) — writing dead-letter",
            _resilience_cfg().flush_max_retries,
            symbol,
            threshold,
            len(bar_dicts),
        )
        dl_path = _niffler_dead_letter(symbol, threshold, bar_dicts)
        if dl_path is not None:
            _niffler_notify_dead_letter(symbol, threshold, len(bar_dicts), dl_path)
            # Niffler: spawn immediate replay in daemon thread (non-blocking)
            threading.Thread(
                target=_niffler_immediate_replay,
                name="niffler-replay",
                daemon=True,
            ).start()
        return False
    else:
        return True


# =============================================================================
# Main sidecar loop
# =============================================================================


def run_sidecar(config: SidecarConfig) -> None:  # noqa: PLR0915
    """Main sidecar entry point. Blocks until interrupted.

    Parameters
    ----------
    config
        Sidecar configuration. Use ``SidecarConfig.from_env()`` for
        env-var-driven setup, or construct directly.

    Issue #96 Task #144 Phase 3: Streaming latency optimization
    - Batches bar writes to reduce ClickHouse roundtrips
    - Accumulates bars by (symbol, threshold) pair
    - Flushes on timeout or batch size threshold
    """
    # Issue #126: SIGHUP reloads config (enables runtime ouroboros_mode switching)

    def _sighup_handler(*_args: object) -> None:
        from opendeviationbar.config import Settings

        Settings.reload_and_clear()
        new_mode = Settings.get().population.ouroboros_mode
        logger.info("SIGHUP: config reloaded (ouroboros_mode=%s)", new_mode)
        # Issue #134: Notify operators of runtime config change
        try:
            from opendeviationbar.notify.telegram import send_telegram

            send_telegram(
                f"<b>SIGHUP: sidecar config reloaded</b>\n"
                f"<b>ouroboros_mode:</b> {new_mode}\n"
                f"<b>PID:</b> {os.getpid()}",
                disable_notification=True,
            )
        except (ImportError, OSError):
            logger.debug("SIGHUP: Telegram notification failed (non-fatal)")

    signal.signal(signal.SIGHUP, _sighup_handler)

    from opendeviationbar.logging import generate_trace_id

    trace_id = generate_trace_id("sdc")
    sidecar_start_time = time.monotonic()

    if not config.symbols:
        msg = (
            "No symbols configured. "
            "Enable symbols in python/opendeviationbar/data/symbols.toml "
            "(set enabled=true) or pass symbols list directly. "
            "OPENDEVIATIONBAR_STREAMING_SYMBOLS env var overrides the registry."
        )
        raise ValueError(msg)

    # Issue #120: Build valid (symbol, threshold) pairs, skip invalid with log.
    # Use get_min_threshold_for_symbol() directly to avoid firing NDJSON telemetry
    # + Pushover via resolve_and_validate_threshold() on sidecar startup.
    from opendeviationbar.threshold import get_min_threshold_for_symbol

    valid_pairs: set[tuple[str, int]] = set()
    for sym in config.symbols:
        min_thr = get_min_threshold_for_symbol(sym)
        for thr in config.thresholds:
            if thr >= min_thr:
                valid_pairs.add((sym, thr))
            else:
                logger.info(
                    "skipping %s@%d (below min_threshold %d)", sym, thr, min_thr
                )

    if not valid_pairs:
        msg = "No valid (symbol, threshold) pairs after filtering"
        raise ValueError(msg)

    # Issue #79: Symbol registry gate — drop disabled symbols before streaming.
    # enabled=false in symbols.toml means intentionally inactive; skip silently.
    try:
        from opendeviationbar.symbol_registry import filter_pairs_by_registry

        filtered = filter_pairs_by_registry(valid_pairs)
        dropped = valid_pairs - set(filtered)
        if dropped:
            for sym, thr in sorted(dropped):
                logger.info("skipping %s@%d (disabled in symbol registry)", sym, thr)
        valid_pairs = set(filtered)
    except ImportError:
        pass

    if not valid_pairs:
        msg = "No valid pairs after symbol registry gate (all symbols disabled?)"
        raise ValueError(msg)

    # Log per-symbol threshold map
    sym_thresholds: dict[str, list[int]] = defaultdict(list)
    for sym, thr in valid_pairs:
        sym_thresholds[sym].append(thr)
    for _sym, thrs in sym_thresholds.items():
        thrs.sort()

    logger.info(
        "valid pairs: %d symbols, %d total (symbol, threshold) pairs",
        len(sym_thresholds),
        len(valid_pairs),
    )
    for sym, thrs in sorted(sym_thresholds.items()):
        logger.info("  %s: %s", sym, thrs)

    # Issue #119: Symbol coverage health check
    env_symbols_str = os.environ.get("OPENDEVIATIONBAR_STREAMING_SYMBOLS", "")
    from opendeviationbar.symbol_registry import normalize_symbol

    env_symbols = [normalize_symbol(s) for s in env_symbols_str.split(",") if s.strip()]
    config_upper = {normalize_symbol(s) for s in config.symbols}
    if env_symbols and config_upper != set(env_symbols):
        missing = sorted(set(env_symbols) - config_upper)
        coverage_msg = (
            f"Symbol mismatch: {len(config.symbols)}/{len(env_symbols)} symbols. "
            f"Missing: {missing}. Use --from-env."
        )
        logger.warning(coverage_msg)
        try:
            from opendeviationbar.hooks import HookEvent, emit_hook

            emit_hook(
                HookEvent.SYMBOL_COVERAGE_MISMATCH,
                symbol="*",
                active=len(config.symbols),
                expected=len(env_symbols),
                missing=missing,
            )
        except (ImportError, OSError, RuntimeError, ValueError):
            logger.debug("failed to emit SYMBOL_COVERAGE_MISMATCH hook", exc_info=True)
        try:
            import sys

            from opendeviationbar.notify.telegram import (
                _build_context_block,
                send_telegram,
            )

            msg = (
                "<b>SIDECAR PARTIAL COVERAGE</b>\n"
                f"<b>Active:</b> {len(config.symbols)}/{len(env_symbols)} symbols\n"
                f"<b>Missing:</b> {', '.join(missing)}\n"
                "<b>Env var:</b> OPENDEVIATIONBAR_STREAMING_SYMBOLS\n"
                "<b>Fix:</b> use <code>--from-env</code> flag\n"
                f"<b>Cmdline:</b> <code>{' '.join(sys.argv)}</code>"
                + _build_context_block("sidecar")
            )
            send_telegram(msg, disable_notification=False)
        except (ImportError, OSError, RuntimeError):
            logger.debug(
                "failed to send coverage mismatch alert",
                exc_info=True,
            )

    logger.info(
        "starting sidecar: symbols=%s, thresholds=%s, microstructure=%s, "
        "watchdog_timeout_s=%d, max_watchdog_restarts=%d",
        config.symbols,
        config.thresholds,
        config.include_microstructure,
        config.watchdog_timeout_s,
        config.max_watchdog_restarts,
    )

    # Issue #117: Cache OpenDeviationBarCache at startup
    # to avoid per-flush schema.sql reads
    from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

    ch_cache = OpenDeviationBarCache()

    # Issue #109: Start health check HTTP server (daemon thread)
    health_server = _start_health_server(
        config.health_port, sse_enabled=config.sse_enabled
    )

    # Issue #117-119: Set start time for uptime tracking + send startup notification
    try:
        from opendeviationbar.notify.telegram import set_start_time

        set_start_time(sidecar_start_time)
    except ImportError:
        pass
    try:
        from opendeviationbar.hooks import HookEvent, emit_hook

        emit_hook(
            HookEvent.SIDECAR_STARTED,
            symbol="*",
            symbols=config.symbols,
            thresholds=config.thresholds,
            health_port=config.health_port,
            pid=os.getpid(),
        )
    except (ImportError, OSError, RuntimeError, ValueError):
        logger.debug("failed to emit SIDECAR_STARTED hook", exc_info=True)
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram

        preview = ", ".join(config.symbols[:_STARTUP_SYMBOL_PREVIEW])
        ellipsis = "..." if len(config.symbols) > _STARTUP_SYMBOL_PREVIEW else ""
        # Issue #120: Show per-symbol threshold coverage in startup notification
        thresholds_summary = "; ".join(
            f"{sym}:{thrs}" for sym, thrs in sorted(sym_thresholds.items())
        )
        startup_msg = (
            "<b>SIDECAR STARTED</b>\n"
            f"<b>Symbols:</b> {len(config.symbols)} "
            f"({preview}{ellipsis})\n"
            f"<b>Coverage:</b> {thresholds_summary}\n"
            f"<b>Health:</b> :{config.health_port}/health\n"
            f"<b>PID:</b> {os.getpid()}" + _build_context_block("sidecar")
        )
        send_telegram(startup_msg, disable_notification=True)
    except (ImportError, OSError, RuntimeError):
        logger.debug("failed to send sidecar startup Telegram alert", exc_info=True)

    # (#286) No startup DELETE needed. Checkpoint-restored processor produces
    # bars with the same first_agg_trade_id as the previous instance's forming
    # bar. ReplacingMergeTree naturally overwrites by ORDER BY key. Buffered WS
    # + unlimited Puck ensures zero trade-level gaps. Future restarts produce
    # perfectly continuous bars — no vacuum, no overlaps, no gaps.
    gap_fill_results = None

    # Charon: replay dead-letter files from previous sidecar runs before engine start
    if _DEAD_LETTER_DIR.exists() and any(_DEAD_LETTER_DIR.glob("*.parquet")):
        try:
            from opendeviationbar.kintsugi import replay_dead_letters

            charon_replayed = replay_dead_letters()
            if charon_replayed > 0:
                _notify_charon_replay(charon_replayed, "startup")
                logger.info(
                    "Charon: replayed %d dead-letter bars on startup", charon_replayed
                )
        except (ImportError, OSError, RuntimeError, ConnectionError):
            logger.exception("Charon: startup replay failed (non-fatal)")

    # (#286) Python inline backfill: fills trailing edge for pairs with missing
    # checkpoints. The Rust fill_from_rest() is the long-term replacement but
    # currently OOMs on large gaps (loads all trades into Vec — needs chunking).
    # Python backfill uses chunked REST + bounded memory (MEM-014).
    _inline_startup_backfill(config, valid_pairs, trace_id)

    # Start Rust engine (Issue #107: extracted to helper for watchdog restart)
    engine, checkpoints_loaded, min_tid_guard = _create_engine(
        config,
        trace_id,
        gap_fill_results,
        valid_pairs=valid_pairs,
    )
    logger.info(
        "live bar engine started (checkpoints_loaded=%d)",
        checkpoints_loaded,
    )

    # Issue #217: Expose engine reference for /ariadne endpoint
    _health_state["engine"] = engine

    # Puck: startup CH→engine TID reconciliation
    # If CH has committed bars beyond the engine's checkpoint, submit fill_gap()
    # commands to recover the delta (covers CH-blip gaps from prior runs).
    if hasattr(engine, "fill_gap"):
        _startup_tid_reconciliation(engine, config.symbols, config.thresholds)

    # (#286) Deterministic day-boundary gap heal: check CH for gaps between
    # yesterday's last bar and today's first bar (or engine seed). Covers ALL
    # edge cases including manual bar deletions, stale checkpoints, multi-restart
    # scenarios, and any situation where today's data has a gap from yesterday.
    # Uses the live engine's fill_gap() — rate-limited, correct processor state.
    if hasattr(engine, "fill_gap"):
        _heal_day_boundary_gaps(engine, valid_pairs)

    # Issue #214: Extract forming bar watcher (separate PyO3 object to avoid borrow conflict)
    forming_bar_watcher = engine.take_forming_bar_watcher()
    forming_bar_stop = None
    if config.sse_enabled and config.forming_bar_interval_s > 0:
        if forming_bar_watcher is not None:
            forming_bar_stop = _start_forming_bar_broadcaster(
                forming_bar_watcher,
                valid_pairs,
                interval_s=config.forming_bar_interval_s,
                stale_s=config.forming_bar_stale_s,
            )
        else:
            logger.warning("forming bar watcher not available — broadcaster disabled")

    # Phase 1c: Extract gap event watcher (same pattern as forming bar watcher)
    gap_watcher = (
        engine.take_gap_event_watcher()
        if hasattr(engine, "take_gap_event_watcher")
        else None
    )
    if gap_watcher is not None:
        logger.info("gap event watcher attached — SSE gap_detected events enabled")
    else:
        logger.debug("gap event watcher not available — gap SSE events disabled")

    # Signal handling for graceful shutdown
    running = True

    def _handle_signal(signum: int, _frame: object) -> None:
        nonlocal running
        logger.info("received signal %d, shutting down", signum)
        running = False

    signal.signal(signal.SIGINT, _handle_signal)
    signal.signal(signal.SIGTERM, _handle_signal)

    bars_written = 0

    # Issue #107: Watchdog state — detect half-open TCP via trade flow staleness
    last_trades_received = 0
    last_trade_increment_time = time.monotonic()
    watchdog_restarts = 0
    consecutive_timeouts = 0  # Track None returns for heartbeat logging
    last_restart_mono = 0.0  # Issue #261: cooldown guard against cascading restarts
    timeout_heartbeat_interval = 60  # Log every 60 timeouts ≈ 5 min at 5s timeout

    # Lethe: ring buffer overflow detection
    lethe_last_dropped = 0
    lethe_drop_accumulator = 0
    lethe_threshold = config.lethe_alert_threshold

    # Argus: per-symbol liveness tracking (phi accrual + trade-flow awareness)
    argus_last_bar_time: dict[str, float] = {}
    argus_last_check = time.monotonic()
    # Phi accrual: adaptive per-symbol failure detection.
    # first_heartbeat_estimate_ms=60_000 (1 min) — typical bar emission interval.
    # threshold=8.0 — standard phi threshold (high confidence dead).
    argus_phi = PhiAccrualDetector(
        threshold=8.0,
        first_heartbeat_estimate_ms=60_000,
        min_std_deviation_ms=5_000.0,
    )
    if config.argus_timeout_s > 0:
        now_mono = time.monotonic()
        for sym, _ in valid_pairs:
            argus_last_bar_time.setdefault(sym, now_mono)
            argus_phi.heartbeat(sym, now_mono * 1000)

    # Charon: dead-letter self-replay
    charon_last_replay = time.monotonic()

    # Issue #215: Periodic checkpoint push to SSE clients
    last_checkpoint_push = time.monotonic()

    # Issue #96 Task #144 Phase 3: Batch accumulation for streaming optimization
    bar_batch_buffer = defaultdict(list)  # Keyed by (symbol, threshold)
    batch_max_size = 10  # Flush batch when reaching this size
    batch_timeout_s = 1.0  # Flush batch every N seconds
    last_batch_flush_time = time.monotonic()

    def _update_data_freshness(
        symbol: str, threshold: int, bars: list[dict[str, Any]]
    ) -> None:
        """Update data_freshness table with latest bar timestamp (Issue #163)."""
        if not bars or ch_cache is None:
            return
        last_bar = bars[-1]
        close_time_ms = last_bar.get("close_time_ms", 0)
        if not close_time_ms:
            return
        with contextlib.suppress(OSError, RuntimeError):
            ch_cache.client.command(
                "INSERT INTO opendeviationbar_cache.data_freshness "
                "(symbol, threshold_decimal_bps, last_close_time_ms, last_bar_count) "
                f"VALUES ('{symbol}', {threshold}, {close_time_ms}, {bars_written})"
            )

    def _flush_all_batches() -> int:
        """Flush all accumulated batches to ClickHouse with retry + dead-letter.

        Issue #115 (Kintsugi Phase 1a): Uses _write_batch_with_retry() for
        exponential backoff (1s, 2s, 4s) and dead-letter on final failure.
        Issue #117: Threads cached ch_cache to avoid per-flush schema.sql reads.
        """
        nonlocal last_batch_flush_time
        written = 0
        for (symbol, threshold), bars in bar_batch_buffer.items():
            if not bars:
                continue
            if _write_batch_with_retry(
                symbol,
                threshold,
                bars,
                cache=ch_cache,
            ):
                written += len(bars)
                _update_data_freshness(symbol, threshold, bars)
        bar_batch_buffer.clear()
        last_batch_flush_time = time.monotonic()
        return written

    def _drain_gap_events() -> None:
        """Drain all pending gap events (non-blocking) and broadcast via SSE."""
        nonlocal gap_watcher
        if gap_watcher is None or not config.sse_enabled:
            return
        while True:
            try:
                gap_event = gap_watcher.next_event(timeout_ms=0)
            except (RuntimeError, AttributeError):
                gap_watcher = None
                return
            if gap_event is None:
                return
            _broadcast_gap_event(gap_event)
            logger.info(
                "gap detected: %s expected=%d actual=%d gap=%d filled=%s",
                gap_event["symbol"],
                gap_event["expected_tid"],
                gap_event["actual_tid"],
                gap_event["gap_size"],
                gap_event["filled_inline"],
            )

    try:
        while running:
            # Check if batch timeout occurred (1 second since last flush)
            now = time.monotonic()
            if (now - last_batch_flush_time) >= batch_timeout_s and bar_batch_buffer:
                written = _flush_all_batches()
                if config.verbose:
                    logger.debug("batch timeout flush: %d bars written", written)

            bar = engine.next_bar(timeout_ms=config.timeout_ms)

            if bar is None:
                consecutive_timeouts += 1

                # Periodic heartbeat (show sidecar is responsive)
                if consecutive_timeouts % timeout_heartbeat_interval == 0:
                    has_metrics = hasattr(engine, "get_metrics")
                    metrics = engine.get_metrics() if has_metrics else {}
                    trades_received = (
                        metrics.get("trades_received", 0) if metrics else 0
                    )
                    stale_s = time.monotonic() - last_trade_increment_time
                    logger.info(
                        "watchdog heartbeat: %d consecutive timeouts, "
                        "last_trades_received=%d, stale=%.0fs",
                        consecutive_timeouts,
                        trades_received,
                        stale_s,
                    )

                # Check trade flow via metrics (Issue #107)
                if config.watchdog_timeout_s > 0:
                    metrics = engine.get_metrics()
                    _update_health_metrics(metrics)
                    current_trades = metrics.get("trades_received", 0)

                    if current_trades > last_trades_received:
                        # Reset counter on fresh trades
                        consecutive_timeouts = 0
                        last_trades_received = current_trades
                        last_trade_increment_time = time.monotonic()
                    else:
                        stale_s = time.monotonic() - last_trade_increment_time
                        if stale_s >= config.watchdog_timeout_s:
                            logger.warning(
                                "WATCHDOG: no trades %.0fs, %d timeouts",
                                stale_s,
                                consecutive_timeouts,
                            )
                            _notify_watchdog_trigger(
                                stale_s,
                                current_trades,
                                watchdog_restarts,
                            )

                            if watchdog_restarts >= config.max_watchdog_restarts:
                                logger.error(
                                    "WATCHDOG: max restarts (%d) exceeded, exiting",
                                    config.max_watchdog_restarts,
                                )
                                break

                            # Issue #261: Cooldown guard against cascading restarts
                            since_last = time.monotonic() - last_restart_mono
                            if (
                                last_restart_mono > 0
                                and since_last < _sidecar_cfg().min_restart_interval_s
                            ):
                                logger.warning(
                                    "WATCHDOG: restart cooldown active (%.0fs since last, need %ds), skipping",
                                    since_last,
                                    _sidecar_cfg().min_restart_interval_s,
                                )
                                consecutive_timeouts += 1
                                continue

                            # Flush pending bars before restart
                            written = _flush_all_batches()
                            if written > 0:
                                logger.info("flushed %d bars before restart", written)

                            # Use error-tolerant restart (Issue #107, TGI #2)
                            new_engine, success, new_min_tid = _restart_engine_with_recovery(
                                engine,
                                config,
                                trace_id,
                                bars_written,
                                watchdog_restarts + 1,
                                valid_pairs=valid_pairs,
                            )

                            if success:
                                engine = new_engine
                                min_tid_guard = new_min_tid
                                # Issue #214: Update engine reference for /ariadne endpoint
                                _health_state["engine"] = engine
                                # Issue #214: Restart forming bar broadcaster with new engine's watcher
                                if forming_bar_stop is not None:
                                    forming_bar_stop.set()
                                forming_bar_watcher = engine.take_forming_bar_watcher()
                                if (
                                    config.sse_enabled
                                    and config.forming_bar_interval_s > 0
                                    and forming_bar_watcher is not None
                                ):
                                    forming_bar_stop = _start_forming_bar_broadcaster(
                                        forming_bar_watcher,
                                        valid_pairs,
                                        interval_s=config.forming_bar_interval_s,
                                        stale_s=config.forming_bar_stale_s,
                                    )
                                # Phase 1c: Re-acquire gap event watcher from new engine
                                gap_watcher = (
                                    engine.take_gap_event_watcher()
                                    if hasattr(engine, "take_gap_event_watcher")
                                    else None
                                )
                                watchdog_restarts += 1
                                last_restart_mono = time.monotonic()
                                consecutive_timeouts = 0
                                last_trades_received = 0
                                last_trade_increment_time = time.monotonic()
                                # Reset Argus liveness timestamps + phi detector on restart
                                restart_mono = time.monotonic()
                                argus_last_bar_time = {
                                    sym: restart_mono for sym, _ in valid_pairs
                                }
                                argus_phi = PhiAccrualDetector(
                                    threshold=8.0,
                                    first_heartbeat_estimate_ms=60_000,
                                    min_std_deviation_ms=5_000.0,
                                )
                                for sym, _ in valid_pairs:
                                    argus_phi.heartbeat(sym, restart_mono * 1000)
                                logger.info("watchdog restart succeeded")
                            else:
                                # Restart failed — escalate
                                logger.error(
                                    "ESCALATION: restart failed after %d timeouts",
                                    consecutive_timeouts,
                                )
                                _notify_restart_failure(
                                    watchdog_restarts + 1,
                                    consecutive_timeouts,
                                    stale_s,
                                )
                                # Exit to be restarted by systemd/supervisor
                                break

                # Phase 1c: Drain pending gap events (non-blocking)
                _drain_gap_events()

                # Lethe: detect ring buffer overflow
                if consecutive_timeouts % timeout_heartbeat_interval == 0:
                    lethe_metrics = (
                        engine.get_metrics() if hasattr(engine, "get_metrics") else {}
                    )
                    current_dropped = lethe_metrics.get("dropped_bars", 0)
                    if current_dropped > lethe_last_dropped:
                        new_drops = current_dropped - lethe_last_dropped
                        lethe_drop_accumulator += new_drops
                        lethe_last_dropped = current_dropped
                        if lethe_drop_accumulator >= lethe_threshold:
                            _notify_lethe_overflow(new_drops, current_dropped)
                            lethe_drop_accumulator = 0
                            # Re-arm at higher threshold after first alert
                            lethe_threshold = _sidecar_cfg().lethe_rearm_threshold

                # Argus: per-symbol liveness check (phi accrual + trade awareness)
                if config.argus_timeout_s > 0:
                    now_argus = time.monotonic()
                    if now_argus - argus_last_check >= _sidecar_cfg().argus_check_interval_s:
                        argus_last_check = now_argus

                        per_sym_last_trade_us: dict[str, int] = {}
                        try:
                            _fb_source = (
                                forming_bar_watcher
                                if forming_bar_watcher is not None
                                else engine
                            )
                            forming_bars = _fb_source.get_forming_bars()
                            for fb in forming_bars:
                                sym = fb.get("_symbol")
                                ts = fb.get("_last_trade_timestamp_us", 0)
                                if sym and ts > per_sym_last_trade_us.get(sym, 0):
                                    per_sym_last_trade_us[sym] = ts
                        except (RuntimeError, AttributeError, KeyError):
                            logger.debug(
                                "Argus: get_forming_bars() unavailable, falling back to bar-only tracking"
                            )

                        now_us = int(time.time() * 1_000_000)
                        now_ms = now_argus * 1000  # monotonic ms for phi
                        dead_syms = []
                        quiet_syms = []

                        for sym in argus_last_bar_time:
                            phi_val = argus_phi.phi(sym, now_ms)
                            if phi_val < argus_phi.threshold:
                                continue  # Phi says symbol is alive
                            # Phi says suspicious/dead — check trade flow
                            last_trade_us = per_sym_last_trade_us.get(sym, 0)
                            if last_trade_us > 0:
                                trade_age_s = (now_us - last_trade_us) / 1_000_000
                                if trade_age_s < _ARGUS_TRADE_RECENCY_S:
                                    quiet_syms.append(sym)
                                    continue
                            dead_syms.append(sym)

                        if dead_syms:
                            _notify_argus_stale(dead_syms, config.argus_timeout_s)
                        if quiet_syms:
                            logger.debug(
                                "Argus: %d symbols receiving trades but no bar breach in %ds: %s",
                                len(quiet_syms),
                                config.argus_timeout_s,
                                ", ".join(sorted(quiet_syms)),
                            )

                # Issue #215: Periodic checkpoint push to SSE clients
                if config.sse_enabled and config.checkpoint_push_interval_s > 0:
                    now_cp = time.monotonic()
                    if (
                        now_cp - last_checkpoint_push
                    ) >= config.checkpoint_push_interval_s:
                        last_checkpoint_push = now_cp
                        with _sse_clients_lock:
                            has_cp_clients = bool(_sse_clients)
                        if has_cp_clients:
                            try:
                                cp_data = {}
                                for sym, thr in valid_pairs:
                                    cp = _load_checkpoint(sym, thr)
                                    if cp:
                                        cp_data[f"{sym}:{thr}"] = cp
                                if cp_data:
                                    _broadcast_checkpoint_event(cp_data)
                            except Exception:
                                logger.warning(
                                    "checkpoint push failed (non-fatal)",
                                    exc_info=True,
                                )

                # Charon: periodic dead-letter replay
                now_charon = time.monotonic()
                if now_charon - charon_last_replay >= config.charon_interval_s:
                    charon_last_replay = now_charon
                    if _DEAD_LETTER_DIR.exists() and any(
                        _DEAD_LETTER_DIR.glob("*.parquet")
                    ):
                        try:
                            from opendeviationbar.kintsugi import replay_dead_letters

                            replayed = replay_dead_letters()
                            if replayed > 0:
                                _notify_charon_replay(replayed, "periodic")
                        except (ImportError, OSError, RuntimeError, ConnectionError):
                            logger.exception(
                                "Charon: periodic replay failed (non-fatal)"
                            )

                continue

            # Issue #273: Burst-atomic bar draining.
            #
            # After receiving the first bar, immediately drain all additional
            # bars available in the ring buffer (0ms timeout). This ensures
            # that when the Rust engine emits a burst of bars from same-ms
            # trades, the Python side processes ALL of them together before
            # any batch flush or checkpoint I/O. Without this, a batch flush
            # could occur between bar N and bar N+1 of a burst, potentially
            # leaving bar N+1 unwritten if the sidecar restarts.
            burst_bars = [bar]
            for _ in range(_BURST_DRAIN_MAX):
                extra = engine.next_bar(timeout_ms=0)
                if extra is None:
                    break
                burst_bars.append(extra)

            if len(burst_bars) > 1:
                logger.debug(
                    "burst-atomic drain: processing %d bars atomically",
                    len(burst_bars),
                )

            # Bar(s) received — reset watchdog
            metrics = engine.get_metrics()
            last_trades_received = metrics.get("trades_received", 0)
            last_trade_increment_time = time.monotonic()

            # Process each bar in the burst
            for burst_bar in burst_bars:
                symbol = burst_bar.pop("_symbol", "UNKNOWN")
                threshold = burst_bar.pop("_threshold", 0)

                # Argus: update per-symbol liveness (phi heartbeat + timestamp)
                if config.argus_timeout_s > 0:
                    argus_last_bar_time[symbol] = time.monotonic()
                    argus_phi.heartbeat(symbol)

                # Issue #120: Skip bars for invalid (symbol, threshold) pairs
                if (symbol, threshold) not in valid_pairs:
                    continue

                # (#264) L1: Filter cross-midnight bars before batch buffer.
                open_ms = burst_bar.get("open_time_ms", 0)
                close_ms = burst_bar.get("close_time_ms", 0)
                if open_ms > 0 and close_ms > 0 and (open_ms // 86_400_000) != (close_ms // 86_400_000):
                    logger.warning(
                        "cross-midnight bar discarded: %s@%d open=%d close=%d "
                        "(kintsugi will heal the gap)",
                        symbol,
                        threshold,
                        open_ms,
                        close_ms,
                    )
                    continue

                # (#286 fix) L5: Min-TID guard — drop bars from fresh processors
                # that overlap with CH's existing data. When no checkpoint loaded,
                # the processor starts from scratch and may produce bars whose
                # first_agg_trade_id falls within CH's committed range.
                bar_first_tid = burst_bar.get("first_agg_trade_id", 0)
                ch_floor = min_tid_guard.get((symbol, threshold), 0)
                if ch_floor > 0 and bar_first_tid > 0 and bar_first_tid <= ch_floor:
                    logger.debug(
                        "min-TID guard: dropping %s@%d bar first_tid=%d "
                        "(≤ CH floor %d)",
                        symbol,
                        threshold,
                        bar_first_tid,
                        ch_floor,
                    )
                    continue
                # Once a bar passes the guard, remove it — Puck has bridged the gap
                # and the processor is now producing bars beyond CH's range.
                if ch_floor > 0:
                    del min_tid_guard[(symbol, threshold)]

                # Issue #163: Broadcast to SSE clients before batching
                if config.sse_enabled:
                    _broadcast_bar_event(symbol, threshold, burst_bar)

                # Accumulate bar in batch buffer
                batch_key = (symbol, threshold)
                bar_batch_buffer[batch_key].append(burst_bar)
                bars_written += 1

                # Issue #96: Batch checkpoint saves — every 10 bars or on flush
                if (
                    bars_written % batch_max_size == 0
                    or len(bar_batch_buffer[batch_key]) >= batch_max_size
                ):
                    _save_checkpoint(symbol, threshold, burst_bar)

                # Flush batch if size threshold reached
                if len(bar_batch_buffer[batch_key]) >= batch_max_size:
                    batch = bar_batch_buffer[batch_key]
                    wrote = _write_batch_with_retry(
                        symbol,
                        threshold,
                        batch,
                        cache=ch_cache,
                    )
                    if wrote:
                        _update_data_freshness(symbol, threshold, batch)
                        if config.verbose or bars_written % 100 == 0:
                            logger.info(
                                "%s@%d: batch written (%d bars, total=%d, close=%s)",
                                symbol,
                                threshold,
                                len(batch),
                                bars_written,
                                burst_bar.get("close"),
                            )
                    bar_batch_buffer[batch_key] = []

            # Phase 1c: Drain pending gap events after processing burst
            _drain_gap_events()

    finally:
        exit_reason = "clean shutdown" if not running else "exception"

        # Issue #214: Stop forming bar broadcaster
        if forming_bar_stop is not None:
            forming_bar_stop.set()

        # Issue #215: Final checkpoint push to SSE clients before shutdown
        if config.sse_enabled:
            with _sse_clients_lock:
                has_shutdown_clients = bool(_sse_clients)
            if has_shutdown_clients:
                try:
                    cp_data = {}
                    for sym, thr in valid_pairs:
                        cp = _load_checkpoint(sym, thr)
                        if cp:
                            cp_data[f"{sym}:{thr}"] = cp
                    if cp_data:
                        _broadcast_checkpoint_event(cp_data)
                except Exception:
                    logger.warning(
                        "shutdown checkpoint push failed (non-fatal)", exc_info=True
                    )

        # Flush all pending bars on shutdown
        written = _flush_all_batches()
        if written > 0:
            logger.info("flushed %d pending bars on shutdown", written)

        engine.stop()

        # Issue #97: Extract and save processor checkpoints for next startup
        _extract_checkpoints(engine, trace_id, bars_written)

        engine.shutdown()

        # Issue #109: Shut down health server
        if health_server is not None:
            health_server.shutdown()

        # Issue #117: Close cached OpenDeviationBarCache
        try:
            ch_cache.close()
        except (OSError, RuntimeError):
            logger.debug("failed to close cached OpenDeviationBarCache", exc_info=True)

        metrics = engine.get_metrics()
        logger.info(
            "sidecar stopped: bars_written=%d, engine_metrics=%s",
            bars_written,
            metrics,
        )

        # Issue #117-119: Shutdown notification + hook
        try:
            from opendeviationbar.hooks import HookEvent, emit_hook

            emit_hook(
                HookEvent.SIDECAR_STOPPED,
                symbol="*",
                bars_written=bars_written,
                exit_reason=exit_reason,
                pid=os.getpid(),
            )
        except (ImportError, OSError, RuntimeError, ValueError):
            logger.debug("failed to emit SIDECAR_STOPPED hook", exc_info=True)
        try:
            from opendeviationbar.notify.telegram import (
                _build_context_block,
                send_telegram,
            )

            shutdown_msg = (
                "<b>SIDECAR STOPPED</b>\n"
                f"<b>Bars written:</b> {bars_written:,}\n"
                f"<b>Reason:</b> {exit_reason}\n"
                f"<b>PID:</b> {os.getpid()}\n"
                "<b>Next:</b> systemd should auto-restart within 10s\n"
                "<b>If not:</b> <code>journalctl --user -u opendeviationbar-sidecar -n 20</code>"
                + _build_context_block("sidecar")
            )
            send_telegram(shutdown_msg, disable_notification=False)
        except (ImportError, OSError, RuntimeError):
            logger.debug(
                "failed to send shutdown Telegram alert",
                exc_info=True,
            )
