from __future__ import annotations

import logging
import time
from collections import defaultdict
from datetime import UTC

import polars as pl

from .clickhouse_write import _write_batch_with_retry
from .http_server import _junction_state

logger = logging.getLogger(__name__)


def _get_midnight_crossover_tid(symbol: str) -> int | None:
    """Determine the first aggregated trade ID at or after UTC midnight.

    First-principles design — zero arbitrary parameters:

    1. **CH-seeded fromId** (1 REST call, 20 weight):
       CH yesterday MAX(last_agg_trade_id) → fromId=max+1 → verify timestamp
       >= midnight. Fast path when CH has yesterday's data.

    2. **API-max post-midnight query** (1 REST call, 20 weight):
       ``startTime=midnight, endTime=midnight+1h`` → take result[0].
       The 1-hour window is not our parameter — it is the Binance aggTrades
       API maximum allowed time range. We use 100% of what the API permits.
       No symbol has zero trades in 1 hour. Parameter-free by construction.

    Returns the agg_trade_id of the first trade of today, or None on failure.
    """
    from datetime import datetime

    midnight_ms = int(
        datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0).timestamp()
        * 1000
    )

    # --- Tier 1: CH hint + REST verification (1 REST call) ---
    seed_tid = _ch_yesterday_last_tid(symbol)
    if seed_tid is not None:
        crossover = _verify_crossover_from_seed(symbol, seed_tid, midnight_ms)
        if crossover is not None:
            _self_validate_crossover(symbol, crossover, midnight_ms, source="CH-seed")
            return crossover

    # --- Tier 2: API-max post-midnight (1 REST call, parameter-free) ---
    # Binance aggTrades API constraint: startTime-to-endTime < 1 hour.
    # We use the FULL hour — not a guess, but the API's own limit.
    crossover = _api_max_post_midnight(symbol, midnight_ms)
    if crossover is not None:
        _self_validate_crossover(symbol, crossover, midnight_ms, source="API-max")
        return crossover

    logger.warning("midnight crossover for %s: both tiers failed", symbol)
    return None


def _self_validate_crossover(
    symbol: str, crossover_tid: int, midnight_ms: int, *, source: str
) -> None:
    """Self-validate the crossover TID by proving 3 properties from Binance REST.

    Fetches fromId=crossover_tid-1, limit=2 (1 REST call) to get both
    the predecessor trade and the crossover trade. Then verifies:

    1. predecessor.timestamp < midnight  (belongs to yesterday)
    2. crossover.timestamp >= midnight   (belongs to today)
    3. crossover.agg_trade_id == predecessor.agg_trade_id + 1  (Stathera continuity)

    Logs INFO on pass, WARNING on any violation. Never raises — validation
    is advisory (the crossover is used regardless, but violations are loud).
    """
    from opendeviationbar._core import fetch_aggtrades_by_id

    try:
        # Single REST call: fetch predecessor + crossover in one batch
        pair_size = 2  # predecessor + crossover
        trades = fetch_aggtrades_by_id(symbol, crossover_tid - 1, pair_size)
        if not trades or len(trades) < pair_size:
            logger.info(
                "midnight crossover for %s: tid=%d (%s) — "
                "self-validation skipped (< 2 trades returned, likely first trade ever)",
                symbol,
                crossover_tid,
                source,
            )
            return

        pred = trades[0]
        cross = trades[1]

        pred_tid = pred.get("agg_trade_id", pred.get("a"))
        pred_ts = pred.get("timestamp", pred.get("T", 0))
        cross_tid = cross.get("agg_trade_id", cross.get("a"))
        cross_ts = cross.get("timestamp", cross.get("T", 0))

        # Property 1: predecessor is yesterday
        prop1 = pred_ts < midnight_ms
        # Property 2: crossover is today
        prop2 = cross_ts >= midnight_ms
        # Property 3: consecutive trade IDs (Stathera at day boundary)
        prop3 = cross_tid == pred_tid + 1

        if prop1 and prop2 and prop3:
            logger.info(
                "midnight crossover for %s: tid=%d (%s) — "
                "VALIDATED: pred_tid=%d ts=%d < midnight, cross_tid=%d ts=%d >= midnight, consecutive",
                symbol,
                crossover_tid,
                source,
                pred_tid,
                pred_ts,
                cross_tid,
                cross_ts,
            )
        else:
            violations = []
            if not prop1:
                violations.append(f"pred.ts={pred_ts} >= midnight (NOT yesterday)")
            if not prop2:
                violations.append(f"cross.ts={cross_ts} < midnight (NOT today)")
            if not prop3:
                violations.append(
                    f"cross_tid={cross_tid} != pred_tid+1={pred_tid + 1} (GAP)"
                )
            logger.warning(
                "midnight crossover for %s: tid=%d (%s) — "
                "VALIDATION FAILED: %s",
                symbol,
                crossover_tid,
                source,
                "; ".join(violations),
            )
    except Exception:
        logger.debug(
            "midnight crossover self-validation failed for %s (tid=%d)",
            symbol,
            crossover_tid,
            exc_info=True,
        )


def _ch_yesterday_last_tid(symbol: str) -> int | None:
    """Query CH for yesterday's MAX(last_agg_trade_id) across all thresholds.

    Returns None if no yesterday data exists (fresh symbol, post-OPTIMIZE, etc).
    """
    try:
        from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            result = cache.client.query(
                "SELECT max(last_agg_trade_id) "
                "FROM opendeviationbar_cache.open_deviation_bars FINAL "
                "WHERE symbol = %(symbol)s "
                "  AND toDate(close_time_ms / 1000) = yesterday() "
                "  AND last_agg_trade_id > 0",
                parameters={"symbol": symbol},
            )
            rows = result.result_rows
            if rows and rows[0][0] and rows[0][0] > 0:
                return int(rows[0][0])
    except Exception:
        logger.debug("CH yesterday query failed for %s", symbol, exc_info=True)
    return None


def _verify_crossover_from_seed(
    symbol: str, seed_tid: int, midnight_ms: int
) -> int | None:
    """Verify a CH-seeded TID by fetching fromId=seed+1 from Binance REST.

    If the seed is yesterday's exact last TID, the returned trade IS the
    crossover (timestamp >= midnight). If the seed is slightly early (e.g.,
    orphan bar missing), the first trade might still be before midnight —
    in that case we return None and let Tier 2 handle it.

    Exactly 1 REST call. Source of truth: Binance.
    """
    from opendeviationbar._core import fetch_aggtrades_by_id

    try:
        trades = fetch_aggtrades_by_id(symbol, seed_tid + 1, 10)
        if not trades:
            return None

        first = trades[0]
        ts = first.get("timestamp", first.get("T", 0))
        if ts >= midnight_ms:
            return first.get("agg_trade_id", first.get("a"))

        # Seed was slightly early — trades are still yesterday. Tier 2 will fix.
        return None  # noqa: TRY300
    except Exception:
        logger.debug(
            "fromId verify failed for %s (seed=%d)", symbol, seed_tid, exc_info=True
        )
        return None


def _api_max_post_midnight(symbol: str, midnight_ms: int) -> int | None:
    """Query the first trade after midnight using the API's maximum time range.

    Binance aggTrades API constraint: startTime-to-endTime must be < 1 hour.
    We use exactly that limit: [midnight, midnight + 3_600_000). The first
    element of the result is the first trade of today.

    This is parameter-free: the 1-hour window is the API's own maximum,
    not a value we chose. Any actively traded Binance symbol has trades
    within 1 hour of midnight. One call, deterministic.
    """
    from opendeviationbar._core import fetch_aggtrades_rest

    # 3_600_000 ms = 1 hour = Binance aggTrades API max time range
    binance_api_max_range_ms = 3_600_000

    try:
        trades = fetch_aggtrades_rest(
            symbol, midnight_ms, midnight_ms + binance_api_max_range_ms
        )
        if trades:
            return trades[0].get("agg_trade_id", trades[0].get("a"))
    except Exception:
        logger.debug("API-max post-midnight query failed for %s", symbol, exc_info=True)

    return None


def _verify_yesterday_orphan(symbol: str, crossover_tid: int) -> bool:
    """Verify yesterday's orphan bar closes at exactly crossover_tid - 1.

    PREFLIGHT-02: Queries ClickHouse for yesterday's orphan bar and checks
    that orphan.last_agg_trade_id == crossover_tid - 1 (perfect Stathera continuity).

    Returns True if verification passes, False if mismatch or missing.
    Never raises -- logs WARNING on any issue and returns False.
    """
    try:
        from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            orphan_tid = cache.get_yesterday_orphan_tid(symbol)

        if orphan_tid is None:
            logger.info(
                "no orphan bar for %s yesterday -- fresh deploy or gap, deferring to kintsugi",
                symbol,
            )
            return False

        if orphan_tid == crossover_tid - 1:
            logger.info(
                "midnight continuity verified for %s: orphan_last_tid=%d, crossover_tid=%d",
                symbol,
                orphan_tid,
                crossover_tid,
            )
            return True

        logger.warning(
            "midnight continuity MISMATCH for %s: orphan_last_tid=%d, expected=%d, crossover_tid=%d"
            " -- deferring to kintsugi",
            symbol,
            orphan_tid,
            crossover_tid - 1,
            crossover_tid,
        )
        return False  # noqa: TRY300
    except Exception:
        logger.warning(
            "midnight orphan verification failed for %s", symbol, exc_info=True
        )
        return False


def _check_yesterday_integrity(
    symbols: list[str],  # noqa: ARG001
    thresholds: list[int],  # noqa: ARG001
    preflight_verified: dict[str, int],
) -> dict[str, str]:
    """Check yesterday's data integrity without repairing (STARTUP-01).

    Fast check (~3s for 16 symbols) that replaces the blocking
    ``_repair_yesterday_if_needed()`` in the sidecar startup path.

    For each symbol with a verified crossover TID:
    1. Orphan check: yesterday's orphan bar last_agg_trade_id == crossover_tid - 1
    2. Stathera audit: zero gaps AND zero overlaps for yesterday (all thresholds)

    Returns
    -------
    dict[str, str]
        Mapping of {symbol: reason} for broken symbols. Empty dict means all clean.
        Reasons: "orphan_misaligned", "orphan_missing", "stathera_gap",
        "stathera_overlap", "ch_unavailable".
    """
    if not preflight_verified:
        logger.info("yesterday integrity: no preflight data, skipping check")
        return {}

    from opendeviationbar.clickhouse.cache import OpenDeviationBarCache
    from opendeviationbar.clickhouse.stathera_query import (
        FULL_TABLE,
        STATHERA_CORE_COLUMNS,
        VALID_TID_FILTER,
    )
    from opendeviationbar.ouroboros import get_operational_ouroboros_mode

    flagged: dict[str, str] = {}

    try:
        ouroboros_mode = get_operational_ouroboros_mode()

        with OpenDeviationBarCache() as cache:
            for symbol, crossover_tid in sorted(preflight_verified.items()):
                # --- Orphan check ---
                orphan_tid = cache.get_yesterday_orphan_tid(symbol)
                expected_tid = crossover_tid - 1

                if orphan_tid is None:
                    flagged[symbol] = "orphan_missing"
                    logger.warning(
                        "yesterday integrity: %s orphan missing (expected last_tid=%d)",
                        symbol,
                        expected_tid,
                    )
                    continue
                if orphan_tid != expected_tid:
                    flagged[symbol] = "orphan_misaligned"
                    logger.warning(
                        "yesterday integrity: %s orphan misaligned "
                        "(orphan_last_tid=%d, expected=%d)",
                        symbol,
                        orphan_tid,
                        expected_tid,
                    )
                    continue

                # --- Stathera audit for yesterday (all thresholds in one query) ---
                try:
                    result = cache.client.query(
                        f"""
                        SELECT threshold_decimal_bps, tid_delta
                        FROM (
                            SELECT threshold_decimal_bps, {STATHERA_CORE_COLUMNS}
                            FROM {FULL_TABLE} FINAL
                            WHERE symbol = {{symbol:String}}
                              AND ouroboros_mode = {{ouroboros:String}}
                              AND toDate(open_time_ms / 1000, 'UTC') = yesterday()
                              AND {VALID_TID_FILTER}
                            WINDOW w AS (
                                PARTITION BY threshold_decimal_bps
                                ORDER BY first_agg_trade_id, close_time_ms
                            )
                        )
                        WHERE rn > 1 AND tid_delta != 0
                        LIMIT 1
                        """,
                        parameters={
                            "symbol": symbol,
                            "ouroboros": ouroboros_mode,
                        },
                    )
                    if result.result_rows:
                        tid_delta = result.result_rows[0][1]
                        if tid_delta > 0:
                            flagged[symbol] = "stathera_gap"
                            logger.warning(
                                "yesterday integrity: %s Stathera gap (tid_delta=%d)",
                                symbol,
                                tid_delta,
                            )
                        else:
                            flagged[symbol] = "stathera_overlap"
                            logger.warning(
                                "yesterday integrity: %s Stathera overlap (tid_delta=%d)",
                                symbol,
                                tid_delta,
                            )
                except Exception:
                    flagged[symbol] = "ch_unavailable"
                    logger.warning(
                        "yesterday integrity: Stathera query failed for %s",
                        symbol,
                        exc_info=True,
                    )

    except (ConnectionError, OSError, RuntimeError) as exc:
        # CH completely unavailable — flag all symbols
        logger.warning(
            "yesterday integrity: CH unavailable (%s), flagging all %d symbols",
            exc,
            len(preflight_verified),
        )
        return {sym: "ch_unavailable" for sym in preflight_verified}

    # Summary log
    if flagged:
        logger.warning(
            "yesterday integrity: %d/%d symbols broken: %s",
            len(flagged),
            len(preflight_verified),
            ", ".join(f"{s}={r}" for s, r in sorted(flagged.items())),
        )
    else:
        logger.info(
            "yesterday integrity: all %d symbols clean", len(preflight_verified)
        )

    return flagged


def _repair_yesterday_if_needed(
    symbols: list[str],
    thresholds: list[int],
    preflight_verified: dict[str, int],
    valid_pairs: set[tuple[str, int]] | None,
) -> None:
    """Verify yesterday is clean. If orphan bar missing/misaligned, repair.

    Chronological invariant: yesterday MUST be correct before today's
    fill_from_rest, so today's first bar connects seamlessly to yesterday's
    last bar. Without this, restarts after midnight leave a gap at the
    day boundary that persists until kintsugi repairs tomorrow — too late.

    For each symbol with a verified crossover TID, checks if yesterday's
    last bar ends at crossover_tid - 1. If not, invokes kintsugi's
    day-recompute for yesterday (synchronous, blocks until complete).
    """
    import datetime as dt

    if not preflight_verified:
        logger.info("yesterday repair: no preflight data, skipping")
        return

    from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

    yesterday = dt.datetime.now(dt.UTC).date() - dt.timedelta(days=1)
    yesterday_start_ms = int(
        dt.datetime.combine(yesterday, dt.time.min, tzinfo=dt.UTC).timestamp() * 1000
    )
    yesterday_end_ms = yesterday_start_ms + 86_400_000

    # Collect symbols that need yesterday repair
    symbols_to_repair: list[str] = []
    with OpenDeviationBarCache() as cache:
        for symbol, crossover_tid in sorted(preflight_verified.items()):
            orphan_tid = cache.get_yesterday_orphan_tid(symbol)
            expected_tid = crossover_tid - 1

            if orphan_tid == expected_tid:
                continue  # yesterday is clean for this symbol

            if orphan_tid is None:
                logger.warning(
                    "yesterday repair: %s missing orphan bar (expected last_tid=%d)",
                    symbol,
                    expected_tid,
                )
            else:
                logger.warning(
                    "yesterday repair: %s orphan misaligned "
                    "(orphan_last_tid=%d, expected=%d, gap=%d trades)",
                    symbol,
                    orphan_tid,
                    expected_tid,
                    expected_tid - orphan_tid,
                )
            symbols_to_repair.append(symbol)

    if not symbols_to_repair:
        logger.info("yesterday repair: all %d symbols clean", len(preflight_verified))
        return

    logger.info(
        "yesterday repair: %d/%d symbols need repair: %s",
        len(symbols_to_repair),
        len(preflight_verified),
        symbols_to_repair,
    )

    # Repair each symbol's yesterday via kintsugi day-recompute
    from opendeviationbar.clickhouse.write_lock import cache_write_lock
    from opendeviationbar.kintsugi import Shard, _repair_under_lock

    registered_pairs = valid_pairs or {
        (sym, thr) for sym in symbols for thr in thresholds
    }
    repaired = 0
    for symbol in symbols_to_repair:
        # Repair all thresholds for this symbol
        symbol_thresholds = sorted(
            thr for sym, thr in registered_pairs if sym == symbol
        )
        for threshold in symbol_thresholds:
            shard = Shard(
                symbol=symbol,
                threshold_decimal_bps=threshold,
                gap_start_ms=yesterday_start_ms,
                gap_end_ms=yesterday_end_ms,
                gap_hours=24.0,
                anchor_last_tid=0,
                resume_first_tid=0,
            )
            try:
                t0 = time.monotonic()
                with cache_write_lock(symbol, threshold, timeout=300):
                    written = _repair_under_lock(
                        shard=shard,
                        day_start_ms=yesterday_start_ms,
                        day_end_ms=yesterday_end_ms,
                        repair_date=yesterday,
                        ouroboros_mode="day",
                        include_microstructure=True,
                        verbose=False,
                    )
                elapsed_s = time.monotonic() - t0
                logger.info(
                    "yesterday repair: %s@%d — %d bars in %.1fs",
                    symbol,
                    threshold,
                    written,
                    elapsed_s,
                )
                repaired += 1
            except Exception:
                logger.warning(
                    "yesterday repair: %s@%d failed (non-fatal, kintsugi will retry)",
                    symbol,
                    threshold,
                    exc_info=True,
                )

    logger.info(
        "yesterday repair: completed %d pairs for %d symbols",
        repaired,
        len(symbols_to_repair),
    )


def _heal_day_boundary_gaps(
    engine: object,
    valid_pairs: set[tuple[str, int]],
) -> None:
    """Deterministic day-boundary gap heal on startup (#286).

    Checks ClickHouse for Stathera gaps in today's bars. For each gap found,
    submits fill_gap() to the live engine. Covers ALL edge cases: manual bar
    deletions, stale checkpoints, multi-restart scenarios, and any situation
    where today's data has trade-ID discontinuities.

    Uses the live engine's fill_gap() — rate-limited parallel REST fetch,
    checkpoint-restored processor state, correct bar boundaries.
    """
    try:
        from opendeviationbar.clickhouse.cache import OpenDeviationBarCache
        from opendeviationbar.clickhouse.stathera_query import (
            FULL_TABLE,
            per_pair_filter,
            per_pair_window,
        )
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode
        from opendeviationbar.symbol_registry import filter_pairs_by_registry

        ouroboros_mode = get_operational_ouroboros_mode()
        cache = OpenDeviationBarCache()
        pairs = filter_pairs_by_registry(valid_pairs)
        total_healed = 0

        for symbol, threshold in sorted(pairs):
            try:
                # Query for gaps touching today (including yesterday→today boundary)
                result = cache.client.query(
                    f"""
                    SELECT prev_last_tid, first_agg_trade_id, tid_delta
                    FROM (
                        SELECT
                            lagInFrame(last_agg_trade_id, 1) OVER w AS prev_last_tid,
                            first_agg_trade_id,
                            first_agg_trade_id
                              - lagInFrame(last_agg_trade_id, 1) OVER w - 1
                              AS tid_delta,
                            row_number() OVER w AS rn
                        FROM {FULL_TABLE} FINAL
                        {per_pair_filter()}
                        AND toDate(open_time_ms / 1000) = today()
                        {per_pair_window()}
                    )
                    WHERE tid_delta > 0 AND rn > 1
                    ORDER BY first_agg_trade_id
                    """,
                    parameters={
                        "symbol": symbol,
                        "threshold": int(threshold),
                        "ouroboros": ouroboros_mode,
                    },
                )
                rows = result.result_rows if result.result_rows else []
                for row in rows:
                    anchor_tid, resume_tid, delta = (
                        int(row[0]),
                        int(row[1]),
                        int(row[2]),
                    )
                    if delta > 0:
                        # Gap: fill missing trades via live engine
                        logger.info(
                            "day-boundary heal: %s@%d gap delta=%d — submitting fill_gap",
                            symbol,
                            threshold,
                            delta,
                        )
                        try:
                            engine.fill_gap(symbol, anchor_tid + 1, resume_tid)  # type: ignore[attr-defined]
                            total_healed += 1
                        except Exception:
                            logger.warning(
                                "day-boundary heal: fill_gap failed for %s@%d (non-fatal)",
                                symbol,
                                threshold,
                                exc_info=True,
                            )
            except Exception:
                logger.debug(
                    "day-boundary heal: query failed for %s@%d", symbol, threshold
                )

        if total_healed > 0:
            logger.info(
                "day-boundary heal: submitted %d fill_gap commands", total_healed
            )
        else:
            logger.info("day-boundary heal: no gaps found — continuity clean")

    except Exception:
        logger.warning("day-boundary heal failed (non-fatal)", exc_info=True)


def _sync_flush_rest_bars(  # noqa: PLR0915
    engine: object,
    valid_pairs: set[tuple[str, int]] | None,
) -> int:
    """Drain REST bars from ring buffer and write to CH synchronously (#296).

    Called between fill_from_rest() and engine.start(). At this point the
    ring buffer contains ONLY bars produced by fill_from_rest() — WS bars
    don't enter until start() is called. This guarantees an exclusive drain
    window with no race conditions.

    Writing to CH before _heal_day_boundary_gaps() ensures the heal query
    sees accurate state and doesn't report false gaps for ranges already
    covered by fill_from_rest().

    Uses drain_bars() (synchronous, no block_on) instead of polling next_bar()
    which fails when called via PyO3 block_on before engine.start().

    Returns the number of bars written.
    """
    day_ms = 86_400_000

    # Drain all REST bars from ring buffer — synchronous, no async polling.
    if hasattr(engine, "drain_bars"):
        all_bars: list[dict] = engine.drain_bars()  # type: ignore[attr-defined]
    else:
        # Fallback for OdbEngine or older builds without drain_bars
        all_bars = []
        while True:
            bar = engine.next_bar(timeout_ms=500)  # type: ignore[attr-defined]
            if bar is None:
                break
            all_bars.append(bar)

    raw_count = len(all_bars)
    batches: dict[tuple[str, int], list[dict]] = defaultdict(list)
    drained = 0
    cross_midnight_count = 0
    _seen_rejected: set[str] = set()

    if valid_pairs is not None:
        vp_sample = sorted(valid_pairs)[:5]
        vp_types = {(type(s).__name__, type(t).__name__) for s, t in list(valid_pairs)[:3]}
        logger.info(
            "sync_flush: valid_pairs=%d, sample=%s, types=%s",
            len(valid_pairs), vp_sample, vp_types,
        )

    for bar in all_bars:
        symbol = bar.pop("_symbol", "UNKNOWN")
        threshold = bar.pop("_threshold", 0)

        # Skip pairs not in valid_pairs
        if valid_pairs is not None and (symbol, threshold) not in valid_pairs:
            _pair_key = f"{symbol}@{threshold}"
            if _pair_key not in _seen_rejected:
                _seen_rejected.add(_pair_key)
                logger.info(
                    "sync_flush: REJECTED %s (sym_type=%s, thr_type=%s, thr_repr=%r)",
                    _pair_key,
                    type(symbol).__name__,
                    type(threshold).__name__,
                    threshold,
                )
            continue

        # L1: cross-midnight filter (#264)
        open_ms = bar.get("open_time_ms", 0)
        close_ms = bar.get("close_time_ms", 0)
        if open_ms > 0 and close_ms > 0 and (open_ms // day_ms) != (close_ms // day_ms):
            cross_midnight_count += 1
            logger.warning(
                "sync_flush: cross-midnight bar discarded: %s@%d open=%d close=%d",
                symbol,
                threshold,
                open_ms,
                close_ms,
            )
            continue

        batches[(symbol, threshold)].append(bar)
        drained += 1

    filtered_invalid_pairs = raw_count - drained - cross_midnight_count
    logger.info(
        "sync_flush: drain diagnostics: raw=%d, valid=%d, cross_midnight=%d, invalid_pair=%d",
        raw_count,
        drained,
        cross_midnight_count,
        filtered_invalid_pairs,
    )

    if drained == 0:
        logger.info(
            "sync_flush: ring buffer empty — nothing to flush (raw drain: %d bars)",
            len(all_bars),
        )
        return 0

    from opendeviationbar import __version__
    from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

    source_version = f"{__version__}+sidecar"
    written = 0

    for (symbol, threshold), bars in batches.items():
        try:
            bars_df = pl.DataFrame(bars)
            with OpenDeviationBarCache() as cache:
                inserted = cache.store_bars_batch(
                    symbol=symbol,
                    threshold_decimal_bps=threshold,
                    bars=bars_df,
                    version=source_version,
                    overlap_strategy="off",
                    _skip_lock=False,
                )
            written += inserted
        except Exception:
            logger.info(
                "sync_flush: store_bars_batch failed for %s@%d, "
                "falling back to _write_batch_with_retry (non-fatal)",
                symbol,
                threshold,
                exc_info=True,
            )
            if _write_batch_with_retry(symbol, threshold, bars):
                written += len(bars)

    # Populate junction telemetry for /status endpoint
    for (symbol, _threshold), bars in batches.items():
        if symbol not in _junction_state:
            _junction_state[symbol] = {}
        last_tid = max(
            (b.get("last_agg_trade_id", 0) for b in bars),
            default=0,
        )
        if last_tid > 0:
            _junction_state[symbol]["rest_last_tid"] = last_tid

    # Diagnostic: log which pairs were written and which had zero bars
    pairs_with_bars = sorted(
        ((s, t, len(bars)) for (s, t), bars in batches.items()),
        key=lambda x: (-x[2], x[0], x[1]),
    )
    logger.info(
        "sync_flush: %d/%d REST bars written to CH before start() (%d pairs)",
        written,
        drained,
        len(batches),
    )
    _max_pairs_to_log = 50
    if len(pairs_with_bars) <= _max_pairs_to_log:
        for sym, thr, cnt in pairs_with_bars:
            logger.debug("sync_flush: wrote %s@%d = %d bars", sym, thr, cnt)

    return written


def _run_midnight_preflight(
    engine: object,
    symbols: list[str],
    thresholds: list[int],
) -> dict[str, int]:
    """Run midnight preflight verification for all streaming symbols.

    PREFLIGHT-03: For each symbol, queries the crossover trade ID (first trade
    after UTC midnight) and verifies yesterday's orphan bar. If verified,
    overrides the engine's trade-ID seed for ALL thresholds of that symbol
    with crossover_tid - 1, so fill_from_rest starts from the exact right point.

    Returns dict mapping symbol -> crossover_tid for symbols that passed preflight.
    Symbols that fail preflight keep their existing CH-seeded values (graceful fallback).
    """
    verified: dict[str, int] = {}
    for symbol in symbols:
        crossover_tid = _get_midnight_crossover_tid(symbol)
        if crossover_tid is None:
            logger.warning(
                "preflight skipped for %s: could not determine crossover trade ID",
                symbol,
            )
            continue

        # Verification is log-only — does not gate seed override.
        _verify_yesterday_orphan(symbol, crossover_tid)

        # Override CH seed with verified crossover TID for all thresholds.
        for threshold in thresholds:
            engine.seed_trade_id(symbol, threshold, crossover_tid - 1)

        verified[symbol] = crossover_tid

    logger.info(
        "midnight preflight: verified %d/%d symbols, overrode seeds for %d pairs",
        len(verified),
        len(symbols),
        len(verified) * len(thresholds),
    )
    return verified
