# FILE-SIZE-OK: Backfill logic (REST, fromId, adaptive loop)
"""Gap backfill for open deviation bar cache (Issue #92).

Bridges the gap between the latest Binance Vision archive data and the current
time by fetching recent trades via REST API and computing open deviation bars.

Architecture:
    Vision first → REST fallback → ClickHouse

The adaptive polling loop self-tunes its frequency based on observed gap size,
configured via OPENDEVIATIONBAR_GAP_* env vars in .mise.toml (SSoT).

Usage
-----
Single-shot backfill:

>>> from opendeviationbar.backfill import backfill_gap
>>> result = backfill_gap("BTCUSDT", threshold_decimal_bps=250)
>>> print(f"Filled {result.bars_written} bars, gap was {result.gap_seconds}s")

Backfill all cached symbols:

>>> from opendeviationbar.backfill import backfill_all_gaps
>>> results = backfill_all_gaps()

Trailing-edge fill is now handled by kintsugi daemon (#238 Phase 2).
The adaptive loop has been removed — use ``kintsugi --daemon`` instead.
"""

from __future__ import annotations

import gc
import logging
import os
import time
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from opendeviationbar.config.backfill import BackfillConfig
    from opendeviationbar.config.resilience import ResilienceConfig

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Lazy config accessors (avoid import-time pydantic-settings resolution)
# ---------------------------------------------------------------------------


def _backfill_cfg() -> BackfillConfig:
    from opendeviationbar.config.backfill import BackfillConfig

    return BackfillConfig()


def _resilience_cfg() -> ResilienceConfig:
    from opendeviationbar.config.resilience import ResilienceConfig

    return ResilienceConfig()


# =============================================================================
# Vision Availability Memory (Redis-backed Negative Cache)
# =============================================================================


def _is_vision_cached_unavailable(symbol: str, date_str: str) -> bool:
    """Check if Vision 404 was recorded recently (via Redis)."""
    from opendeviationbar.clickhouse.write_lock import _get_redis_client

    client = _get_redis_client()
    if client is None:
        return False  # No Redis = no cache = always try
    try:
        import redis

        return client.exists(f"odb:vision:404:{symbol}:{date_str}") > 0
    except (OSError, redis.exceptions.RedisError):
        logger.debug("Redis vision cache read failed", exc_info=True)
        return False


def _mark_vision_unavailable(symbol: str, date_str: str) -> None:
    """Record Vision 404 in Redis (persistent, cross-daemon).

    Uses a shorter TTL (15 min) for dates within 2 days of today since
    Vision archives for recent dates may not yet be published (PYEF-02).
    """
    from opendeviationbar.clickhouse.write_lock import _get_redis_client

    client = _get_redis_client()
    if client is None:
        return  # Graceful degradation — no negative cache

    # Dynamic TTL: recent dates get shorter cache (PYEF-02)
    cfg = _backfill_cfg()
    try:
        parsed = datetime.strptime(date_str, "%Y-%m-%d").date()
        days_ago = (datetime.now(tz=UTC).date() - parsed).days
        ttl = cfg.vision_probe_ttl_recent_s if days_ago <= 2 else cfg.vision_probe_ttl_s
    except (ValueError, TypeError):
        ttl = cfg.vision_probe_ttl_s  # fallback to standard 1h

    try:
        import redis

        client.set(
            f"odb:vision:404:{symbol}:{date_str}",
            "1",
            ex=ttl,
        )
    except (OSError, redis.exceptions.RedisError):
        logger.debug("Redis vision cache write failed", exc_info=True)


# =============================================================================
# Fixed-Interval Polling (Stathera-only: no time-based gap detection)
# =============================================================================


_WATERMARK_GC_RATIO = 0.65
_WATERMARK_ABORT_RATIO = 0.90

# Per-pair recovery sleep when watermark GC fires between symbol pairs.
# Gives mimalloc PURGE_DELAY time to return pages to OS before the next pair.
_INTER_PAIR_RECOVERY_SLEEP_S = 30


# =============================================================================
# MEM-014: Memory Watermark for Chunked Gap Repair
# =============================================================================


def _force_allocator_purge() -> None:
    """Force mimalloc/glibc to return freed pages to the OS immediately.

    On Linux with mimalloc (LD_PRELOAD), this calls mi_collect(true) which
    forces immediate page decommit — more aggressive than MIMALLOC_PURGE_DELAY.
    Falls back to glibc malloc_trim(0) if mimalloc is not loaded.
    No-op on macOS (not needed for development).
    """
    import ctypes
    import sys

    if sys.platform != "linux":
        return

    # Try mimalloc first (loaded via LD_PRELOAD in systemd service)
    try:
        libmi = ctypes.CDLL("libmimalloc.so.2")
        libmi.mi_collect(ctypes.c_bool(True))  # force=true: immediate purge
    except OSError:
        pass
    else:
        return

    # Fallback: glibc malloc_trim (returns free pages at top of heap)
    try:
        libc = ctypes.CDLL("libc.so.6")
        libc.malloc_trim(0)
    except OSError:
        pass


def _check_memory_watermark(memory_max_bytes: int) -> str:
    """Dask-style tiered watermark for chunked gap repair (MEM-014).

    Compares current process RSS against the configured memory ceiling
    (typically from systemd MemoryMax) and returns an action tier.

    Parameters
    ----------
    memory_max_bytes : int
        Memory ceiling in bytes (e.g., from systemd MemoryMax).

    Returns
    -------
    str
        ``"ok"`` — RSS < 65% of ceiling, continue normally.
        ``"gc"`` — RSS 65-90% of ceiling, run gc.collect() then continue.
        ``"abort"`` — RSS >= 90% of ceiling, stop processing to avoid OOM.
    """
    from opendeviationbar.resource_guard import _get_process_rss_mb

    rss_bytes = _get_process_rss_mb() * 1024 * 1024
    ratio = rss_bytes / memory_max_bytes if memory_max_bytes > 0 else 0.0

    if ratio >= _WATERMARK_ABORT_RATIO:
        return "abort"
    if ratio >= _WATERMARK_GC_RATIO:
        return "gc"
    return "ok"


# =============================================================================
# Result Types
# =============================================================================


@dataclass
class BackfillResult:
    """Result of a single backfill operation."""

    symbol: str
    threshold_decimal_bps: int
    bars_written: int
    gap_seconds: float
    latest_ts_before: int | None
    latest_ts_after: int | None
    duration_seconds: float = 0.0
    error: str | None = None


# =============================================================================
# Core Backfill Logic
# =============================================================================

# Issue #164: BTCUSDT always backfilled first (bandwidth priority)
_PRIORITY_SYMBOLS = ("BTCUSDT",)


def _pair_sort_key(pair: tuple[str, int]) -> tuple[int, str, int]:
    """Sort key: priority symbols first, then alphabetical, then threshold."""
    symbol, threshold = pair
    symbol_rank = 0 if symbol in _PRIORITY_SYMBOLS else 1
    return (symbol_rank, symbol, threshold)


def _get_cached_symbol_threshold_pairs() -> list[tuple[str, int]]:
    """Query ClickHouse for all distinct (symbol, threshold) pairs in cache.

    Issue #126: Filters by operational ouroboros_mode to avoid returning
    pairs that only exist in a different mode.

    Issue #164: Results sorted with BTCUSDT first (bandwidth priority).

    Returns
    -------
    list[tuple[str, int]]
        List of (symbol, threshold_decimal_bps) pairs, BTCUSDT first.
    """
    from opendeviationbar.clickhouse import OpenDeviationBarCache
    from opendeviationbar.ouroboros import get_operational_ouroboros_mode

    mode = get_operational_ouroboros_mode()
    with OpenDeviationBarCache() as cache:
        result = cache.client.query(
            "SELECT DISTINCT symbol, threshold_decimal_bps "
            "FROM opendeviationbar_cache.open_deviation_bars FINAL "
            "WHERE ouroboros_mode = {mode:String} "
            "ORDER BY symbol, threshold_decimal_bps",
            parameters={"mode": mode},
        )
        pairs = [(row[0], row[1]) for row in result.result_rows]

    # Filter to enabled symbols only (respect Symbol Registry gate).
    from opendeviationbar.symbol_registry import filter_pairs_by_registry

    pairs = filter_pairs_by_registry(pairs)

    pairs.sort(key=_pair_sort_key)
    return pairs


def _try_vision_for_complete_days(
    symbol: str,
    threshold_decimal_bps: int,
    gap_start_ms: int,
    include_microstructure: bool = False,
    verbose: bool = False,
    market: str | None = None,
) -> int:
    """Try Vision archives for complete past days in the gap.

    Issue #158 Phase 9: Vision-first data source priority.  Binance Vision
    publishes complete daily aggTrades ZIP files with NO rate limit — always
    try Vision before falling back to REST API.

    Uses yesterday as cutoff (T-1) instead of T-2.  The negative cache
    handles empirical unavailability per (symbol, date) without an
    overly conservative static cutoff.

    Returns the number of bars written.  Failures are non-fatal (caller
    falls back to REST).
    """
    from datetime import timedelta

    from opendeviationbar.checkpoint import populate_cache_resumable

    # Compute date range of complete past days in the gap
    gap_start_date = datetime.fromtimestamp(gap_start_ms / 1000, tz=UTC).date()
    today_utc = datetime.now(UTC).date()
    # T-7: Vision archives may not be published for the last 7 days.
    # Only fall back to REST for data this recent; Vision is authoritative for older dates.
    vision_cutoff = today_utc - timedelta(days=7)

    # Start from the day AFTER the gap start (gap_start is already covered)
    vision_start = gap_start_date + timedelta(days=1)

    if vision_start > vision_cutoff:
        logger.debug(
            "%s: no complete days for Vision (start=%s, cutoff=%s)",
            symbol,
            vision_start,
            vision_cutoff,
        )
        return 0

    # Check negative cache — if ALL days are cached-unavailable, skip entirely
    all_cached = True
    check_date = vision_start
    while check_date <= vision_cutoff:
        if not _is_vision_cached_unavailable(symbol, check_date.strftime("%Y-%m-%d")):
            all_cached = False
            break
        check_date += timedelta(days=1)

    if all_cached:
        logger.debug(
            "%s @ %d dbps: all Vision days cached-unavailable (%s to %s)",
            symbol,
            threshold_decimal_bps,
            vision_start,
            vision_cutoff,
        )
        return 0

    vision_start_str = vision_start.strftime("%Y-%m-%d")
    vision_end_str = vision_cutoff.strftime("%Y-%m-%d")

    logger.info(
        "%s @ %d dbps: trying Vision first for %s to %s",
        symbol,
        threshold_decimal_bps,
        vision_start_str,
        vision_end_str,
    )

    try:
        bars = populate_cache_resumable(
            symbol,
            vision_start_str,
            vision_end_str,
            threshold_decimal_bps=threshold_decimal_bps,
            include_microstructure=include_microstructure,
            verbose=verbose,
            notify=False,
            max_days=None,  # bypass guard — this is an authorized internal call
            skip_missing_days=True,  # Vision-first: skip 404s gracefully
            market=market,
        )
        if bars > 0:
            logger.info(
                "%s @ %d dbps: Vision filled %d bars (%s to %s)",
                symbol,
                threshold_decimal_bps,
                bars,
                vision_start_str,
                vision_end_str,
            )
    except (RuntimeError, OSError, ConnectionError, ValueError) as exc:
        # ALL failures mark negative cache — not just 404s.
        # Network timeouts, DNS errors, etc. mean Vision is unavailable now.
        # Dynamic TTL: 15min for recent dates, 1h otherwise (PYEF-02).
        _mark_vision_unavailable(symbol, vision_end_str)
        # Compute display TTL for log message
        try:
            _parsed = datetime.strptime(vision_end_str, "%Y-%m-%d").date()
            _days_ago = (datetime.now(tz=UTC).date() - _parsed).days
            _ttl_label = "15m" if _days_ago <= 2 else "1h"
        except (ValueError, TypeError):
            _ttl_label = "1h"
        logger.warning(
            "%s @ %d dbps: Vision fill failed (cached %s, will use REST): %s",
            symbol,
            threshold_decimal_bps,
            _ttl_label,
            exc,
        )
        return 0
    else:
        return bars


def backfill_gap(
    symbol: str,
    threshold_decimal_bps: int = 250,
    *,
    include_microstructure: bool = True,
    verbose: bool = False,
    overlap_strategy: str = "skip",
    force: bool = False,
) -> BackfillResult:
    """Backfill trailing edge: latest cached bar → now.

    Fills the gap between the most recent bar in ClickHouse and the current
    time via Ariadne fromId pagination (REST API). No time-based gap detection
    — all gap detection is Stathera-only (via kintsugi). Interior gaps are
    handled exclusively by kintsugi's discover→repair cycle.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT").
    threshold_decimal_bps : int
        Threshold in decimal basis points.
    include_microstructure : bool
        Include all 26 microstructure features (default True).
    verbose : bool
        Enable detailed logging.
    overlap_strategy : str
        Stathera overlap strategy for writes ("skip", "replace", "warn", "off").
    force : bool
        Issue #261: Bypass the trailing-edge guard (``_TRAILING_EDGE_MIN_GAP_S``).
        Used by sidecar watchdog restart to fill gaps < 30 min that would
        otherwise be skipped.

    Returns
    -------
    BackfillResult
        Summary of the backfill operation.
    """
    from opendeviationbar.clickhouse import OpenDeviationBarCache
    from opendeviationbar.symbol_registry import validate_symbol_registered
    from opendeviationbar.threshold import get_min_threshold_for_symbol

    validate_symbol_registered(symbol, operation="backfill_gap")
    # Use get_min_threshold_for_symbol() instead of resolve_and_validate_threshold()
    # to avoid firing NDJSON telemetry + Pushover on every call. backfill_gap() is
    # called in loops by backfill_all_gaps() and sidecar _gap_fill() — telemetry
    # spam for @250 pairs with min_threshold=500 was flooding Telegram.
    min_thr = get_min_threshold_for_symbol(symbol)
    if threshold_decimal_bps < min_thr:
        msg = f"Threshold {threshold_decimal_bps} dbps below minimum {min_thr} dbps for {symbol}"
        raise ValueError(msg)

    t0 = time.monotonic()
    now_ms = int(datetime.now(UTC).timestamp() * 1000)

    # Issue #158 (Stathera Phase 4): Combined query for timestamp + trade ID
    with OpenDeviationBarCache() as cache:
        latest_ts, latest_tid = cache.get_latest_bar_with_trade_id(
            symbol,
            threshold_decimal_bps,
        )

    if latest_ts is None:
        logger.warning(
            "No cached bars for %s @ %d dbps — cannot backfill "
            "(run populate_cache_resumable first)",
            symbol,
            threshold_decimal_bps,
        )
        return BackfillResult(
            symbol=symbol,
            threshold_decimal_bps=threshold_decimal_bps,
            bars_written=0,
            gap_seconds=0,
            latest_ts_before=None,
            latest_ts_after=None,
            duration_seconds=time.monotonic() - t0,
            error="no_cached_bars",
        )

    gap_ms = now_ms - latest_ts
    gap_seconds = gap_ms / 1000.0

    # Skip trailing-edge fill when gap is small (< 30 min). The sidecar is
    # likely actively processing these trades via WebSocket. Backfilling via
    # REST would race with the sidecar, producing overlapping bars from
    # different starting trade IDs. Only backfill when the gap is large enough
    # to indicate the sidecar missed this pair (down, reconnecting, or stale).
    # Issue #261: force=True bypasses this guard (used by sidecar watchdog restart).
    if not force and gap_seconds < _backfill_cfg().trailing_edge_min_gap_s:
        logger.debug(
            "%s @ %d dbps: trailing edge %.1f min — too fresh, skipping "
            "(sidecar likely active, avoids overlap)",
            symbol,
            threshold_decimal_bps,
            gap_seconds / 60,
        )
        return BackfillResult(
            symbol=symbol,
            threshold_decimal_bps=threshold_decimal_bps,
            bars_written=0,
            gap_seconds=gap_seconds,
            latest_ts_before=latest_ts,
            latest_ts_after=latest_ts,
            duration_seconds=time.monotonic() - t0,
        )

    logger.info(
        "%s @ %d dbps: trailing edge %.1f min — backfilling via REST API",
        symbol,
        threshold_decimal_bps,
        gap_seconds / 60,
    )

    # Fetch trades via REST API with Ariadne (fromId pagination)
    # Issue #158: Pass trade-ID anchor directly (avoids re-querying ClickHouse)
    rest_bars = 0
    try:
        rest_bars = _fetch_and_process_gap(
            symbol=symbol,
            threshold_decimal_bps=threshold_decimal_bps,
            start_ms=latest_ts + 1,
            end_ms=now_ms,
            include_microstructure=include_microstructure,
            verbose=verbose,
            ariadne_anchor_override=latest_tid,
            overlap_strategy=overlap_strategy,
        )
    except (RuntimeError, OSError, ConnectionError) as e:
        logger.warning(
            "%s @ %d dbps: REST backfill failed: %s",
            symbol,
            threshold_decimal_bps,
            e,
        )
        return BackfillResult(
            symbol=symbol,
            threshold_decimal_bps=threshold_decimal_bps,
            bars_written=0,
            gap_seconds=gap_seconds,
            latest_ts_before=latest_ts,
            latest_ts_after=latest_ts,
            duration_seconds=time.monotonic() - t0,
            error=str(e),
        )

    # Query new latest timestamp
    with OpenDeviationBarCache() as cache:
        new_latest_ts = cache.get_latest_bar_timestamp(symbol, threshold_decimal_bps)

    elapsed = time.monotonic() - t0
    logger.info(
        "%s @ %d dbps: backfilled %d bars in %.1fs (gap was %.1f min)",
        symbol,
        threshold_decimal_bps,
        rest_bars,
        elapsed,
        gap_seconds / 60,
    )

    return BackfillResult(
        symbol=symbol,
        threshold_decimal_bps=threshold_decimal_bps,
        bars_written=rest_bars,
        gap_seconds=gap_seconds,
        latest_ts_before=latest_ts,
        latest_ts_after=new_latest_ts,
        duration_seconds=elapsed,
    )


def _fetch_aggtrades_rest(
    symbol: str,
    start_ms: int,
    end_ms: int,
) -> list[dict]:
    """Fetch aggregated trades from Binance REST API via Rust (native performance).

    Delegates to Rust `fetch_aggtrades_rest()` which uses reqwest to paginate
    through /api/v3/aggTrades (max 1000 per request) with async I/O.

    Returns list of trade dicts compatible with
    OpenDeviationBarProcessor.process_trades().
    """
    from opendeviationbar._core import fetch_aggtrades_rest

    try:
        return fetch_aggtrades_rest(symbol, start_ms, end_ms)
    except Exception as e:
        msg = f"REST API fetch failed for {symbol} [{start_ms}-{end_ms}]: {e}"
        raise RuntimeError(msg) from e


def _fetch_aggtrades_by_id(
    symbol: str,
    from_id: int,
    limit: int = 1000,
) -> list[dict]:
    """Fetch aggregated trades by fromId (zero-gap pagination via Rust).

    Uses ``fromId`` parameter instead of ``startTime`` to avoid dropping
    trades at millisecond boundaries.  Delegates to Rust
    ``fetch_aggtrades_by_id()`` which includes 429 backoff.

    Returns list of trade dicts compatible with
    OpenDeviationBarProcessor.process_trades().
    """
    from opendeviationbar._core import fetch_aggtrades_by_id

    try:
        return fetch_aggtrades_by_id(symbol, from_id, limit)
    except Exception as e:
        msg = (
            f"REST API fetch (fromId) failed for {symbol} "
            f"[fromId={from_id}, limit={limit}]: {e}"
        )
        raise RuntimeError(msg) from e


def _fetch_aggtrades_parallel(
    symbol: str,
    from_id: int,
    to_id: int,
    concurrency: int = 5,
) -> list[dict]:
    """Fetch trades in parallel via Rust parallel_fetch (#291).

    5x faster than sequential for large ranges. Uses the shared
    per-process rate limiter (OnceLock singleton in Rust).
    """
    from opendeviationbar._core import fetch_aggtrades_parallel

    return fetch_aggtrades_parallel(symbol, from_id, to_id, concurrency)


def _fetch_latest_aggtrade(symbol: str) -> dict:
    """Fetch the latest aggregated trade for a symbol via Rust.

    Single REST call with ``limit=1``.  Returns single trade dict as
    anchor point for cursor-based ``fromId`` pagination.
    """
    from opendeviationbar._core import fetch_latest_aggtrade

    try:
        return fetch_latest_aggtrade(symbol)
    except Exception as e:
        msg = f"REST API fetch (latest) failed for {symbol}: {e}"
        raise RuntimeError(msg) from e


def _process_gap_fallback_starttime(
    *,
    symbol: str,
    threshold_decimal_bps: int,
    start_ms: int,
    end_ms: int,
    include_microstructure: bool = True,
    overlap_strategy: str = "skip",
    _skip_lock: bool = False,
    source_tag: str = "kintsugi",
) -> int:
    """Process gap using startTime-based REST fetch (no Ariadne anchor needed).

    In day-mode Ouroboros, each day is independently processable — just fetch
    ALL trades by time range and process through a fresh processor. No trade-ID
    anchor needed. This makes parallel day repairs truly independent.

    Args:
        overlap_strategy: "skip" for first-ever backfill, "replace" for kintsugi.
        _skip_lock: True when caller already holds cache_write_lock.
        source_tag: Source label for Stathera audit trail ("backfill" or "kintsugi").

    Returns number of bars written.
    """
    import polars as pl

    from opendeviationbar.clickhouse import OpenDeviationBarCache
    from opendeviationbar.orchestration.helpers import (
        _create_processor,
    )

    trades = _fetch_aggtrades_rest(symbol, start_ms, end_ms)
    logger.debug(
        "%s: startTime fallback fetched %d trades (%d-%d ms)",
        symbol,
        len(trades),
        start_ms,
        end_ms,
    )
    if not trades:
        return 0

    processor = _create_processor(
        threshold_decimal_bps,
        include_microstructure,
        symbol=symbol,
    )

    # Day-mode: split trades at midnight boundaries (#264)
    from opendeviationbar.ouroboros import get_operational_ouroboros_mode
    from opendeviationbar.validation.day_mode import split_trades_by_day

    all_bars: list[dict] = []
    if get_operational_ouroboros_mode() == "day":
        for i, day_trades in enumerate(split_trades_by_day(trades)):
            if i > 0:
                # Flush orphan from previous day, then reset
                orphan = processor.reset_at_ouroboros()
                if orphan:
                    orphan["is_orphan"] = True
                    all_bars.append(orphan)
            all_bars.extend(processor.process_trades(day_trades))
        # Flush final orphan — critical for days where no bar breaches threshold.
        # Without this, low-volatility days at high thresholds (e.g., 1000 dbps)
        # produce 0 bars, causing kintsugi to DELETE existing orphans without
        # replacement → gap count increases instead of decreasing.
        orphan = processor.reset_at_ouroboros()
        if orphan:
            orphan["is_orphan"] = True
            all_bars.append(orphan)
    else:
        all_bars = processor.process_trades(trades)

    if not all_bars:
        return 0

    bars_df = pl.DataFrame(all_bars)
    from opendeviationbar import __version__

    source_version = f"{__version__}+{source_tag}"
    with OpenDeviationBarCache() as cache:
        return cache.store_bars_batch(
            symbol=symbol,
            threshold_decimal_bps=threshold_decimal_bps,
            bars=bars_df,
            version=source_version,
            overlap_strategy=overlap_strategy,
            _skip_lock=_skip_lock,
        )


def _fetch_and_process_gap(
    *,
    symbol: str,
    threshold_decimal_bps: int,
    start_ms: int,
    end_ms: int,
    include_microstructure: bool = True,
    verbose: bool = False,
    ariadne_anchor_override: int | None = None,
    overlap_strategy: str = "skip",
    _skip_lock: bool = False,
) -> int:
    """Fetch trades and process into open deviation bars using Ariadne (fromId).

    MEM-014: Processes trades in 7-day chunks to bound memory usage.
    Uses ``process_trades_streaming()`` to maintain processor state
    (incomplete bar) across chunk boundaries.

    Issue #115 (Kintsugi Phase 2a): Uses Ariadne fromId pagination when
    a last_agg_trade_id anchor exists in ClickHouse. Falls back to
    startTime only for first-ever backfill (no bars at all).

    Issue #158 (Stathera Phase 4): Accepts ariadne_anchor_override to
    avoid re-querying ClickHouse when caller already has the trade ID.

    Returns number of bars written.
    """
    import polars as pl

    from opendeviationbar.clickhouse import OpenDeviationBarCache
    from opendeviationbar.orchestration.helpers import (
        _create_processor,
    )

    # Issue #158: Use caller-provided anchor, fall back to ClickHouse query
    ariadne_anchor: int | None = ariadne_anchor_override
    if ariadne_anchor is None:
        with OpenDeviationBarCache() as cache:
            ariadne_anchor = cache.get_ariadne_high_water_mark(
                symbol,
                threshold_decimal_bps,
            )

    use_ariadne = ariadne_anchor is not None

    if not use_ariadne:
        # Fallback to startTime (first-ever backfill only) — delegate to helper
        return _process_gap_fallback_starttime(
            symbol=symbol,
            threshold_decimal_bps=threshold_decimal_bps,
            start_ms=start_ms,
            end_ms=end_ms,
            include_microstructure=include_microstructure,
            source_tag="backfill",
        )

    # --- MEM-014: Chunked Ariadne path ---
    assert ariadne_anchor is not None  # type narrowing
    current_from_id = ariadne_anchor + 1
    total_written = 0

    # Memory ceiling for watermark checks (default 4G like kintsugi MemoryMax)
    memory_max_bytes = _backfill_cfg().memory_max_bytes

    # Single streaming processor across all chunks — preserves incomplete bar
    processor = _create_processor(
        threshold_decimal_bps,
        include_microstructure,
        symbol=symbol,
    )

    # Build chunk time windows (used only to bound _fetch_trades_ariadne end_ms)
    chunk_start_ms = start_ms
    while chunk_start_ms < end_ms:
        chunk_end_ms = min(chunk_start_ms + _backfill_cfg().chunk_duration_ms, end_ms)

        # Fetch trades for this chunk via Ariadne
        trades = _fetch_trades_ariadne(
            symbol,
            current_from_id,
            chunk_end_ms,
            verbose=verbose,
        )

        if not trades:
            logger.debug(
                "%s: no trades in chunk [from_id=%d, end_ms=%d]",
                symbol,
                current_from_id,
                chunk_end_ms,
            )
            chunk_start_ms = chunk_end_ms
            continue

        logger.debug(
            "%s: Ariadne chunk fetched %d trades (from_id=%d, end_ms=%d)",
            symbol,
            len(trades),
            current_from_id,
            chunk_end_ms,
        )

        # Advance Ariadne cursor from raw trades (before any filtering)
        last_trade_raw = trades[-1]
        last_trade_id = last_trade_raw.get("agg_trade_id", last_trade_raw.get("a", 0))
        current_from_id = last_trade_id + 1

        # (#264) Enforce start_ms boundary — prevents cross-midnight bars
        # when Ariadne anchor is previous day's last bar
        trades = [
            t for t in trades
            if (t.get("timestamp") or t.get("time", 0)) >= start_ms
        ]
        if not trades:
            chunk_start_ms = chunk_end_ms
            continue

        # (#264) Day-mode: reset processor at UTC midnight crossings
        # within this chunk. Matches checkpoint.py:1170 pattern.
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode
        from opendeviationbar.validation.day_mode import split_trades_by_day

        bars: list[dict] = []
        if get_operational_ouroboros_mode() == "day":
            day_groups = split_trades_by_day(trades)
            for i, day_trades in enumerate(day_groups):
                if i > 0:
                    orphan = processor.reset_at_ouroboros()
                    if orphan:
                        orphan["is_orphan"] = True
                        bars.append(orphan)
                bars.extend(processor.process_trades_streaming(day_trades))
        else:
            bars = processor.process_trades_streaming(trades)

        # Write completed bars to ClickHouse
        if bars:
            bars_df = pl.DataFrame(bars)
            # Issue #158: Stathera source tagging
            from opendeviationbar import __version__

            source_version = f"{__version__}+backfill"
            with OpenDeviationBarCache() as cache:
                written = cache.store_bars_batch(
                    symbol=symbol,
                    threshold_decimal_bps=threshold_decimal_bps,
                    bars=bars_df,
                    version=source_version,
                    overlap_strategy=overlap_strategy,
                    _skip_lock=_skip_lock,
                )
            total_written += written
            logger.debug(
                "%s: chunk wrote %d bars (total %d)",
                symbol,
                written,
                total_written,
            )

        # MEM-014: Free chunk memory and check watermark
        del trades, bars
        gc.collect()

        watermark = _check_memory_watermark(memory_max_bytes)
        if watermark == "abort":
            logger.warning(
                "%s: memory watermark ABORT (RSS >= 90%% of %d MB) — "
                "stopping gap repair early after %d bars written",
                symbol,
                memory_max_bytes // (1024 * 1024),
                total_written,
            )
            break
        if watermark == "gc":
            logger.info(
                "%s: memory watermark GC (RSS 65-90%% of ceiling)",
                symbol,
            )

        chunk_start_ms = chunk_end_ms

    # Flush final orphan bar — ensures the last day's incomplete bar is written.
    # Without this, low-volatility days at high thresholds produce 0 completed bars,
    # and the final incomplete bar is lost in the processor state.
    if get_operational_ouroboros_mode() == "day":
        orphan = processor.reset_at_ouroboros()
        if orphan:
            orphan["is_orphan"] = True
            orphan_df = pl.DataFrame([orphan])
            from opendeviationbar import __version__

            source_version = f"{__version__}+backfill"
            with OpenDeviationBarCache() as cache:
                written = cache.store_bars_batch(
                    symbol=symbol,
                    threshold_decimal_bps=threshold_decimal_bps,
                    bars=orphan_df,
                    version=source_version,
                    overlap_strategy=overlap_strategy,
                    _skip_lock=_skip_lock,
                )
            total_written += written

    return total_written


def _fetch_aggtrades_by_id_with_retry(
    symbol: str,
    from_id: int,
    limit: int = 1000,
    *,
    max_retries: int | None = None,
) -> list[dict]:
    """Fetch aggTrades by ID with retry for transient errors (Issue #156).

    Defense-in-depth atop the Rust-level retry in send_with_rate_limit_backoff.
    Retries on RuntimeError/ConnectionError/OSError with exponential backoff.
    """
    if max_retries is None:
        max_retries = _resilience_cfg().default_max_retries
    backoff_s = _resilience_cfg().default_backoff_s
    last_exc: Exception | None = None
    for attempt in range(max_retries):
        try:
            return _fetch_aggtrades_by_id(symbol, from_id, limit)
        except (RuntimeError, ConnectionError, OSError) as e:
            last_exc = e
            if attempt < max_retries - 1:
                logger.warning(
                    "%s: transient error fetching fromId=%d (attempt %d/%d, "
                    "retry in %.0fs): %s",
                    symbol,
                    from_id,
                    attempt + 1,
                    max_retries,
                    backoff_s,
                    e,
                )
                time.sleep(backoff_s)
                backoff_s *= 2
    raise last_exc  # type: ignore[misc]


def _fetch_trades_ariadne(
    symbol: str,
    from_id: int,
    end_ms: int,
    *,
    batch_size: int = 1000,
    verbose: bool = False,
) -> list[dict]:
    """Paginate trades using Ariadne fromId cursor until end_ms.

    Returns all trades from from_id up to end_ms (inclusive).
    """
    all_trades: list[dict] = []
    current_id = from_id

    while True:
        batch = _fetch_aggtrades_by_id_with_retry(symbol, current_id, batch_size)
        if not batch:
            break

        for trade in batch:
            # Trade timestamp: REST API uses "T", internal dicts use "timestamp"
            ts = trade.get("T") or trade.get("timestamp") or trade.get("time", 0)
            if ts > end_ms:
                return all_trades
            all_trades.append(trade)

        # Advance cursor past this batch
        last_id = batch[-1].get("agg_trade_id", batch[-1].get("a", 0))
        if last_id <= current_id:
            break  # Safety: no progress
        current_id = last_id + 1

        if verbose and len(all_trades) % 10000 == 0:
            logger.debug(
                "%s: Ariadne progress %d trades, current_id=%d",
                symbol,
                len(all_trades),
                current_id,
            )

    return all_trades


# =============================================================================
# Multi-Symbol Backfill
# =============================================================================



def _notify_sustained_failure(consecutive_errors: int, last_error: str = "") -> None:
    """Send Telegram alert after N consecutive backfill failures."""
    try:
        from opendeviationbar.notify.telegram import _build_context_block, send_telegram
    except ImportError:
        return
    try:
        lines = [
            "<b>BACKFILL SUSTAINED FAILURE</b>",
            f"<b>Consecutive errors:</b> {consecutive_errors}",
            f"<b>Threshold:</b> {_backfill_cfg().sustained_failure_threshold}",
        ]
        if last_error:
            lines.append(f"<b>Last error:</b> <code>{last_error[:200]}</code>")
        lines.append(
            "<b>Likely cause:</b> Systemic issue (not transient) — "
            "REST API down, CH unreachable, or SSH tunnel dropped"
        )
        lines.append(
            "<b>Debug:</b> <code>curl -s 'http://localhost:8123/?query=SELECT+1'</code>"
        )
        lines.append(_build_context_block("backfill"))
        send_telegram("\n".join(lines), disable_notification=False)
    except (OSError, RuntimeError, ConnectionError):
        logger.exception("failed to send sustained failure alert")


def backfill_all_gaps(
    *,
    include_microstructure: bool = True,
    verbose: bool = False,
    exclude_symbols: set[str] | None = None,
) -> list[BackfillResult]:
    """Backfill all cached symbol x threshold pairs.

    Queries ClickHouse for all distinct (symbol, threshold) pairs and runs
    backfill_gap for each.

    Issue #115 (Kintsugi Phase 2b): Sends Telegram alert after 5 consecutive
    errors across the batch (sustained failure detection).

    Parameters
    ----------
    exclude_symbols : set[str] | None
        Symbols to skip (Writer Ownership #280 — streaming symbols owned by sidecar).

    Returns
    -------
    list[BackfillResult]
        Results for each pair, including any errors.
    """
    all_pairs = _get_cached_symbol_threshold_pairs()

    # Filter out disabled symbols (respect the Symbol Registry gate).
    from opendeviationbar.symbol_registry import filter_pairs_by_registry

    all_pairs = filter_pairs_by_registry(all_pairs)

    # (#280 Spike A) Exclude streaming symbols — sidecar owns their trailing edge.
    if exclude_symbols:
        all_pairs = [(s, t) for s, t in all_pairs if s not in exclude_symbols]

    # Filter out pairs below min_threshold to avoid telemetry spam from
    # resolve_and_validate_threshold() inside backfill_gap(). ClickHouse may
    # contain @250 data for symbols whose min_threshold is 500 (historical).
    from opendeviationbar.threshold import get_min_threshold_for_symbol

    pairs = [
        (sym, thr) for sym, thr in all_pairs if thr >= get_min_threshold_for_symbol(sym)
    ]
    skipped = len(all_pairs) - len(pairs)
    if skipped > 0:
        logger.debug("backfill_all_gaps: skipped %d pairs below min_threshold", skipped)

    logger.info("Backfilling %d symbol x threshold pairs", len(pairs))

    memory_max = _backfill_cfg().memory_max_bytes

    results = []
    consecutive_errors = 0
    for pairs_processed, (symbol, threshold) in enumerate(pairs):
        # Iteration budget: cap pairs per iteration to prevent cumulative RSS creep
        if pairs_processed >= _backfill_cfg().max_pairs_per_iteration:
            logger.info(
                "Iteration budget reached (%d pairs), deferring %d remaining to next cycle",
                _backfill_cfg().max_pairs_per_iteration,
                len(pairs) - pairs_processed,
            )
            break

        try:
            result = backfill_gap(
                symbol,
                threshold,
                include_microstructure=include_microstructure,
                verbose=verbose,
            )
            results.append(result)
            if result.error:
                consecutive_errors += 1
            else:
                consecutive_errors = 0
        except (ValueError, RuntimeError, OSError, ConnectionError) as e:
            logger.warning(
                "Backfill failed for %s @ %d: %s",
                symbol,
                threshold,
                e,
            )
            results.append(
                BackfillResult(
                    symbol=symbol,
                    threshold_decimal_bps=threshold,
                    bars_written=0,
                    gap_seconds=0,
                    latest_ts_before=None,
                    latest_ts_after=None,
                    error=str(e),
                )
            )
            consecutive_errors += 1

        # Issue #180: Force GC + allocator purge between symbol pairs
        # gc.collect() frees Python objects; _force_allocator_purge() returns pages to OS
        gc.collect()
        _force_allocator_purge()

        # Per-pair watermark: catch memory pressure BETWEEN pairs, not just inside chunks.
        # Without this, pairs accumulate RSS until oomd kills the process.
        watermark = _check_memory_watermark(memory_max)
        if watermark == "abort":
            logger.warning(
                "Memory watermark ABORT after %s@%d — stopping iteration early "
                "(%d/%d pairs processed)",
                symbol,
                threshold,
                pairs_processed + 1,
                len(pairs),
            )
            break
        if watermark == "gc":
            # Skip recovery sleep if next iteration would hit budget break (#236)
            if pairs_processed + 1 >= _backfill_cfg().max_pairs_per_iteration:
                logger.info(
                    "Memory watermark GC after %s@%d — skipping sleep (iteration budget reached)",
                    symbol,
                    threshold,
                )
            else:
                logger.info(
                    "Memory watermark GC after %s@%d — recovery sleep %ds before next pair",
                    symbol,
                    threshold,
                    _INTER_PAIR_RECOVERY_SLEEP_S,
                )
                time.sleep(_INTER_PAIR_RECOVERY_SLEEP_S)
                # Re-check after sleep — mimalloc should have purged by now
                gc.collect()
                _force_allocator_purge()

        if consecutive_errors >= _backfill_cfg().sustained_failure_threshold:
            last_err = results[-1].error if results and results[-1].error else ""
            _notify_sustained_failure(consecutive_errors, last_error=last_err)
            consecutive_errors = 0  # Reset after alert

    total_bars = sum(r.bars_written for r in results)
    errors = sum(1 for r in results if r.error)
    logger.info(
        "Backfill complete: %d pairs, %d bars written, %d errors",
        len(results),
        total_bars,
        errors,
    )
    return results


# =============================================================================
# WS-2: Bar Rectification (Safety Net)
# =============================================================================


def rectify_recent_bars(
    symbol: str,
    threshold_decimal_bps: int,
    lookback_hours: float = 2.0,
    deficit_threshold: float = 0.05,
) -> int:
    """Re-fetch and overwrite bars with significant trade count deficit.

    Compares ``agg_record_count`` in ClickHouse against expected count
    derived from ``last_agg_trade_id - first_agg_trade_id + 1`` for each
    bar in the lookback window. Bars with > ``deficit_threshold`` (5%)
    trade deficit are re-fetched via REST ``fromId`` pagination and
    reprocessed through the canonical Rust processor.

    To achieve bit-exact parity with the streaming sidecar:
    - Fetches ~N lookback trades *before* each bar's ``first_agg_trade_id``
      to populate the processor's ``trade_history`` ring buffer, ensuring
      all 16 ``lookback_*`` inter-bar features are computed (not None).
    - Detects boundary divergence: if the rectified bar has a different
      ``close_time_ms`` than the original (because a previously-missing
      trade triggers a breach earlier/later), deletes the ghost bar via
      ``ALTER TABLE DELETE`` before writing the corrected bar.

    ``ReplacingMergeTree(computed_at)`` ensures the newer version wins
    for bars with matching primary keys.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT").
    threshold_decimal_bps : int
        Threshold in decimal basis points.
    lookback_hours : float
        How far back to check for deficient bars (default 2h).
    deficit_threshold : float
        Fraction of expected trades below which a bar is rectified
        (default 0.05 = 5% deficit).

    Returns
    -------
    int
        Number of bars rectified.
    """
    import polars as pl

    from opendeviationbar.clickhouse import OpenDeviationBarCache
    from opendeviationbar.orchestration.helpers import (
        _create_processor,
        _parse_microstructure_env_vars,
    )
    from opendeviationbar.ouroboros import get_operational_ouroboros_mode

    mode = get_operational_ouroboros_mode()
    cutoff_ms = int((datetime.now(UTC).timestamp() - lookback_hours * 3600) * 1000)

    # Determine lookback trade count for inter-bar feature warm-up
    _, _, _ = _parse_microstructure_env_vars(True, None, None)
    lookback_count = int(
        os.environ.get("OPENDEVIATIONBAR_INTER_BAR_LOOKBACK_COUNT", "200")
    )

    # Step 1: Query recent bars with their trade ID range and record count
    with OpenDeviationBarCache() as cache:
        result = cache.client.query(
            "SELECT close_time_ms, first_agg_trade_id, last_agg_trade_id, "
            "       agg_record_count "
            "FROM opendeviationbar_cache.open_deviation_bars FINAL "
            "WHERE symbol = {symbol:String} "
            "  AND threshold_decimal_bps = {threshold:UInt32} "
            "  AND ouroboros_mode = {mode:String} "
            "  AND close_time_ms > {cutoff_ms:Int64} "
            "ORDER BY close_time_ms",
            parameters={
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
                "mode": mode,
                "cutoff_ms": cutoff_ms,
            },
        )

    if not result.result_rows:
        return 0

    # Step 2: Identify bars with trade deficit
    deficient_bars: list[tuple[int, int, int, int]] = []
    for row in result.result_rows:
        close_time_ms, first_id, last_id, actual_count = (
            int(row[0]),
            int(row[1]),
            int(row[2]),
            int(row[3]),
        )
        expected_count = last_id - first_id + 1
        if expected_count <= 0:
            continue
        if actual_count < expected_count * (1 - deficit_threshold):
            deficient_bars.append((close_time_ms, first_id, last_id, actual_count))

    if not deficient_bars:
        return 0

    logger.info(
        "%s @ %d dbps: %d/%d bars have trade deficit in last %.1fh",
        symbol,
        threshold_decimal_bps,
        len(deficient_bars),
        len(result.result_rows),
        lookback_hours,
    )

    # Step 3: Re-fetch trades with lookback warm-up and reprocess
    rectified = 0
    for close_time_ms, first_id, last_id, old_count in deficient_bars:
        try:
            # Fetch lookback trades to warm up trade_history for inter-bar
            # features. These trades precede the bar and won't produce bars
            # themselves (the bar boundary falls within the bar trades).
            lookback_from_id = max(0, first_id - lookback_count)
            lookback_trades: list[dict] = []
            if lookback_from_id < first_id:
                lookback_trades = _fetch_trades_for_bar(
                    symbol, lookback_from_id, first_id - 1
                )

            # Fetch the bar's actual trades (full range)
            bar_trades = _fetch_trades_for_bar(symbol, first_id, last_id)
            if not bar_trades:
                continue

            # Concatenate: lookback trades warm up trade_history, then bar
            # trades produce the actual bars with correct inter-bar features
            all_trades = lookback_trades + bar_trades

            processor = _create_processor(
                threshold_decimal_bps,
                include_microstructure=True,
                symbol=symbol,
            )
            bars = processor.process_trades(all_trades)
            if not bars:
                continue

            # Filter: keep only bars whose trade range overlaps the original
            # bar's range (exclude bars formed purely from lookback trades)
            rectified_bars = [
                b
                for b in bars
                if b["last_agg_trade_id"] >= first_id
                and b["first_agg_trade_id"] <= last_id
            ]
            if not rectified_bars:
                continue

            # Detect boundary divergence: if no rectified bar has the same
            # close_time_ms as the original, the missing trades shifted the
            # breach point. Delete the ghost bar before writing corrections.
            rectified_close_times = {b["close_time_ms"] for b in rectified_bars}
            if close_time_ms not in rectified_close_times:
                _delete_ghost_bar(
                    symbol,
                    threshold_decimal_bps,
                    mode,
                    close_time_ms,
                )

            bars_df = pl.DataFrame(rectified_bars)
            from opendeviationbar import __version__

            with OpenDeviationBarCache() as cache:
                cache.store_bars_batch(
                    symbol=symbol,
                    threshold_decimal_bps=threshold_decimal_bps,
                    bars=bars_df,
                    version=f"{__version__}+rectify",
                    overlap_strategy="replace",
                )

            new_count = len(bar_trades)
            logger.info(
                "%s @ %d: rectified bar close_time_ms=%d "
                "(agg_count %d → %d, bars_produced=%d)",
                symbol,
                threshold_decimal_bps,
                close_time_ms,
                old_count,
                new_count,
                len(rectified_bars),
            )
            _emit_rectification_event(
                symbol,
                threshold_decimal_bps,
                close_time_ms,
                old_count,
                new_count,
            )
            rectified += 1

        except (RuntimeError, OSError, ConnectionError) as e:
            logger.warning(
                "%s @ %d: rectification failed for bar close_time_ms=%d: %s",
                symbol,
                threshold_decimal_bps,
                close_time_ms,
                e,
            )

    return rectified


def _delete_ghost_bar(
    symbol: str,
    threshold_decimal_bps: int,
    ouroboros_mode: str,
    close_time_ms: int,
) -> None:
    """Delete a ghost bar caused by boundary divergence during rectification.

    When rectification discovers that missing trades shift the breach point,
    the original bar's ``close_time_ms`` differs from the corrected bar's.
    Since ``ReplacingMergeTree`` deduplicates by primary key (which includes
    ``close_time_ms``), the original bar would persist as a ghost.

    This issues an ``ALTER TABLE DELETE`` to remove the stale row.
    """
    from opendeviationbar.clickhouse import OpenDeviationBarCache

    try:
        with OpenDeviationBarCache() as cache:
            cache.client.command(
                "ALTER TABLE opendeviationbar_cache.open_deviation_bars "
                "DELETE WHERE symbol = {symbol:String} "
                "  AND threshold_decimal_bps = {threshold:UInt32} "
                "  AND ouroboros_mode = {mode:String} "
                "  AND close_time_ms = {close_time_ms:Int64}",
                parameters={
                    "symbol": symbol,
                    "threshold": threshold_decimal_bps,
                    "mode": ouroboros_mode,
                    "close_time_ms": close_time_ms,
                },
            )
        logger.info(
            "%s @ %d: deleted ghost bar close_time_ms=%d (boundary divergence)",
            symbol,
            threshold_decimal_bps,
            close_time_ms,
        )
    except (RuntimeError, OSError, ConnectionError) as e:
        logger.warning(
            "%s @ %d: failed to delete ghost bar close_time_ms=%d: %s",
            symbol,
            threshold_decimal_bps,
            close_time_ms,
            e,
        )


def _fetch_trades_for_bar(
    symbol: str,
    first_agg_trade_id: int,
    last_agg_trade_id: int,
) -> list[dict]:
    """Fetch all trades for a bar's ID range via REST ``fromId`` pagination."""
    all_trades: list[dict] = []
    current_id = first_agg_trade_id

    while current_id <= last_agg_trade_id:
        limit = min(1000, last_agg_trade_id - current_id + 1)
        batch = _fetch_aggtrades_by_id_with_retry(symbol, current_id, int(limit))
        if not batch:
            break

        for trade in batch:
            trade_id = trade.get("agg_trade_id", trade.get("a", 0))
            if trade_id > last_agg_trade_id:
                return all_trades
            all_trades.append(trade)

        last_batch_id = batch[-1].get("agg_trade_id", batch[-1].get("a", 0))
        if last_batch_id <= current_id:
            break  # No progress
        current_id = last_batch_id + 1

    return all_trades


def _emit_rectification_event(
    symbol: str,
    threshold: int,
    close_time_ms: int,
    old_agg_count: int,
    new_agg_count: int,
) -> None:
    """Emit NDJSON telemetry event for a bar rectification."""
    try:
        import json

        event = {
            "ts": datetime.now(UTC).isoformat(),
            "event": "bar_rectified",
            "symbol": symbol,
            "threshold": threshold,
            "close_time_ms": close_time_ms,
            "old_agg_count": old_agg_count,
            "new_agg_count": new_agg_count,
        }
        # Append to events log (same as other telemetry)
        from pathlib import Path

        log_dir = Path(
            os.environ.get("OPENDEVIATIONBAR_LOG_DIR", "logs")
        )  # SSoT-OK: env-configurable log dir
        log_dir.mkdir(parents=True, exist_ok=True)
        with (log_dir / "events.jsonl").open("a") as f:
            f.write(json.dumps(event) + "\n")
    except (OSError, ImportError):
        pass  # Non-fatal telemetry


__all__ = [
    "BackfillResult",
    "backfill_all_gaps",
    "backfill_gap",
    "rectify_recent_bars",
]
