# FILE-SIZE-OK
"""Checkpoint system for resumable cache population (Issue #40, Issue #69).

Enables long-running cache population jobs to be resumed after interruption.
Uses atomic file writes with bar-level checkpointing for accurate resume.

Issue #69 Enhancements:
- Bar-level resumability: Preserves incomplete bar state via processor checkpoints
- Hybrid storage: Local + ClickHouse for cross-machine resume
- force_refresh parameter: Wipe cache and checkpoint to start fresh

Usage
-----
>>> from opendeviationbar.checkpoint import populate_cache_resumable
>>> bars = populate_cache_resumable(
...     symbol="BTCUSDT",
...     start_date="2024-01-01",
...     end_date="2024-06-30",
... )
>>> print(f"Populated {bars} bars")

If interrupted, simply run again - it will resume from the last checkpoint.
Use force_refresh=True to wipe cache and start over.
"""

from __future__ import annotations

import logging
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING

# Re-export all public symbols from submodules for backward compatibility
from .engine import (
    _date_range,
    _is_ouroboros_boundary,
    _process_day,
    _process_day_multi_threshold,
    _process_one_threshold_day,
    _write_bars,
)
from .models import (
    _CHECKPOINT_DIR,
    PopulationCheckpoint,
    PopulationContext,
    _get_population_config,
)
from .storage import (
    _checkpoint_provenance,
    _get_checkpoint_path,
    _load_checkpoint_from_clickhouse,
    _save_checkpoint_to_clickhouse,
)

# Explicit re-exports consumed by sidecar/checkpoint.py
__all__ = [
    "_checkpoint_provenance",
    "_process_one_threshold_day",
]

if TYPE_CHECKING:
    from opendeviationbar.processors.core import OpenDeviationBarProcessor

logger = logging.getLogger(__name__)


def populate_cache_multi_threshold(  # noqa: PLR0915
    symbol: str,
    start_date: str,
    end_date: str,
    thresholds: list[int],
    *,
    force_refresh: bool = False,
    include_microstructure: bool = False,
    ouroboros_mode: str | None = None,
    checkpoint_dir: Path | None = None,
    notify: bool = True,
    verbose: bool = True,
    inter_bar_lookback_bars: int | None = None,
    num_async_writers: int = 0,
    max_days: int | None = 366,
    skip_missing_days: bool = False,
    market: str | None = None,
) -> dict[int, int]:
    """Populate cache for multiple thresholds reading each Parquet file once.

    For a symbol with thresholds ``[250, 500, 750]``, the tick file for
    each date is read **once**, then fanned out to independent Rust
    processors running in parallel threads.  This removes redundant I/O and
    speeds up multi-threshold backfill by ~3-4x.

    Each threshold maintains its own checkpoint file and is independently
    resumable.  If threshold 500 fails on a given day, thresholds
    500/750 still advance their checkpoints.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., ``"BTCUSDT"``).
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).
    thresholds : list[int]
        List of thresholds in decimal basis points (e.g., ``[250, 500, 750]``).
    force_refresh : bool
        Wipe existing cache and checkpoints for all thresholds.
    include_microstructure : bool
        Include microstructure features.
    ouroboros_mode : str | None
        Ouroboros reset mode (default: resolved from env).
    checkpoint_dir : Path | None
        Custom checkpoint directory.
    notify : bool
        Emit hook events.
    verbose : bool
        Show progress bar.
    inter_bar_lookback_bars : int | None
        Lookback bar count for inter-bar features.
    num_async_writers : int
        Async ClickHouse writer threads per threshold (0 = disabled).
    max_days : int | None
        Maximum date range span.
    skip_missing_days : bool
        Skip days with no data.
    market : str | None
        Market type override.

    Returns
    -------
    dict[int, int]
        Map of threshold -> total bars written.
    """
    if not thresholds:
        msg = "thresholds must be a non-empty list"
        raise ValueError(msg)

    if len(thresholds) == 1:
        # Single threshold: delegate to the regular path (no fan-out overhead).
        bars = populate_cache_resumable(
            symbol,
            start_date,
            end_date,
            threshold_decimal_bps=thresholds[0],
            force_refresh=force_refresh,
            include_microstructure=include_microstructure,
            ouroboros_mode=ouroboros_mode,
            checkpoint_dir=checkpoint_dir,
            notify=notify,
            verbose=verbose,
            inter_bar_lookback_bars=inter_bar_lookback_bars,
            num_async_writers=num_async_writers,
            max_days=max_days,
            skip_missing_days=skip_missing_days,
            market=market,
        )
        return {thresholds[0]: bars}

    # --- Shared validation (performed once, not per-threshold) ---------------
    from opendeviationbar.hooks import HookEvent, emit_hook
    from opendeviationbar.logging import generate_trace_id
    from opendeviationbar.symbol_registry import (
        resolve_market,
        validate_and_clamp_start_date,
        validate_symbol_registered,
    )

    validate_symbol_registered(symbol, operation="populate_cache_multi_threshold")
    start_date = validate_and_clamp_start_date(symbol, start_date)
    market_resolved = resolve_market(symbol, market)

    if max_days is not None:
        _parsed_start = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
        _parsed_end = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)
        span = (_parsed_end - _parsed_start).days
        if span > max_days:
            msg = (
                f"Date range {start_date} to {end_date} spans {span} days "
                f"(max_days={max_days}). Use kintsugi for historical backfill "
                f"(mise run kintsugi:catchup), or pass max_days=None to override."
            )
            raise ValueError(msg)

    # T-1 guard
    today_utc = datetime.now(UTC).date()
    t1_cutoff = today_utc - timedelta(days=1)
    end_date_parsed = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC).date()
    if end_date_parsed >= today_utc:
        logger.warning(
            "end_date %s is today or future; clamping to T-1 (%s)",
            end_date_parsed,
            t1_cutoff,
        )
        end_date = t1_cutoff.strftime("%Y-%m-%d")

    if ouroboros_mode is None:
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode
        ouroboros_mode = get_operational_ouroboros_mode()

    # Validate + resolve each threshold
    from opendeviationbar.threshold import resolve_and_validate_threshold

    resolved_thresholds = [
        resolve_and_validate_threshold(symbol, t) for t in thresholds
    ]
    # Deduplicate while preserving order
    seen: set[int] = set()
    unique_thresholds: list[int] = []
    for t in resolved_thresholds:
        if t not in seen:
            seen.add(t)
            unique_thresholds.append(t)

    import os

    import opendeviationbar as _odb_mod
    from opendeviationbar.exceptions import CacheWriteError
    from opendeviationbar.orchestration.helpers import _parse_microstructure_env_vars
    from opendeviationbar.orchestration.open_deviation_bars_cache import (
        fatal_cache_write,
    )
    from opendeviationbar.processors.core import OpenDeviationBarProcessor
    from opendeviationbar.storage.parquet import TickStorage

    _mt_workers_raw = int(os.environ.get("OPENDEVIATIONBAR_MULTI_THRESHOLD_WORKERS", "0") or "0")
    _mt_workers: int | None = _mt_workers_raw if _mt_workers_raw > 0 else None

    source_version = f"{getattr(_odb_mod, '__version__', 'unknown')}+populate"
    eff_count, eff_bars, enable_intra = _parse_microstructure_env_vars(
        include_microstructure, None, inter_bar_lookback_bars
    )
    storage = TickStorage()
    trace_id = generate_trace_id("pop-multi")

    # --- Per-threshold checkpoint loading ------------------------------------
    checkpoint_paths: dict[int, Path] = {}
    checkpoints: dict[int, PopulationCheckpoint | None] = {}
    resume_dates: dict[int, str] = {}
    total_bars_per_threshold: dict[int, int] = {}
    active_processors: dict[int, OpenDeviationBarProcessor | None] = {}

    for threshold in unique_thresholds:
        cp_path = _get_checkpoint_path(
            symbol, threshold, start_date, end_date,
            checkpoint_dir, ouroboros_mode=ouroboros_mode,
        )
        checkpoint_paths[threshold] = cp_path

        if force_refresh:
            cp_path.unlink(missing_ok=True)
            checkpoints[threshold] = None
        else:
            cp = PopulationCheckpoint.load(cp_path)
            if cp is None:
                cp = _load_checkpoint_from_clickhouse(
                    symbol, threshold, start_date, end_date,
                    ouroboros_mode=ouroboros_mode,
                )
            if cp and (
                cp.symbol != symbol
                or cp.threshold_bps != threshold
                or cp.start_date != start_date
                or cp.end_date != end_date
            ):
                logger.warning(
                    "Checkpoint mismatch for %s@%d, starting fresh",
                    symbol, threshold,
                )
                cp = None
            if cp and hasattr(cp, "ouroboros_mode") and cp.ouroboros_mode and cp.ouroboros_mode != ouroboros_mode:
                msg = (
                    f"Checkpoint ouroboros_mode mismatch for {symbol}@{threshold}: "
                    f"checkpoint={cp.ouroboros_mode!r} current={ouroboros_mode!r}"
                )
                raise ValueError(msg)
            checkpoints[threshold] = cp

        cp = checkpoints[threshold]
        if cp:
            last_completed = datetime.strptime(
                cp.last_completed_date, "%Y-%m-%d"
            ).replace(tzinfo=UTC)
            resume_dates[threshold] = (last_completed + timedelta(days=1)).strftime(
                "%Y-%m-%d"
            )
            total_bars_per_threshold[threshold] = cp.bars_written
            logger.info(
                "Resuming %s@%d from %s (%d bars written)",
                symbol, threshold, resume_dates[threshold], cp.bars_written,
            )
        else:
            resume_dates[threshold] = start_date
            total_bars_per_threshold[threshold] = 0

        # Restore or init processor
        proc: OpenDeviationBarProcessor | None = None
        if cp and cp.processor_checkpoint:
            try:
                proc = OpenDeviationBarProcessor.from_checkpoint(cp.processor_checkpoint)
                if include_microstructure:
                    pop = _get_population_config()
                    proc.enable_microstructure(
                        inter_bar_lookback_count=eff_count,
                        inter_bar_lookback_bars=eff_bars,
                        include_intra_bar_features=enable_intra,
                        compute_tier2=pop.compute_tier2,
                        compute_tier3=pop.compute_tier3,
                        compute_hurst=pop.compute_hurst or None,
                        compute_permutation_entropy=pop.compute_permutation_entropy or None,
                    )
            except Exception:
                logger.exception(
                    "Failed to restore processor for %s@%d, starting fresh",
                    symbol, threshold,
                )
                proc = None
        active_processors[threshold] = proc

    # --- Build per-threshold contexts ----------------------------------------
    created_at = datetime.now(UTC).isoformat()

    def _make_ctx(threshold: int) -> PopulationContext:
        cp = checkpoints[threshold]
        return PopulationContext(
            symbol=symbol,
            threshold_decimal_bps=threshold,
            ouroboros_mode=ouroboros_mode,
            include_microstructure=include_microstructure,
            notify=notify,
            inter_bar_lookback_count=eff_count,
            inter_bar_lookback_bars=eff_bars,
            include_intra_bar_features=enable_intra,
            source_version=source_version,
            storage=storage,
            thread_pool=None,  # multi-threshold uses its own parallelism
            observer=None,
            trace_id=trace_id,
            created_at=cp.created_at if cp else created_at,
            start_date=start_date,
            end_date=end_date,
            market=market_resolved,
        )

    ctxs: dict[int, PopulationContext] = {t: _make_ctx(t) for t in unique_thresholds}

    # --- Day loop (find min resume date, iterate all dates) ------------------
    all_dates = list(_date_range(start_date, end_date))
    total_all_days = len(all_dates)

    # Set of thresholds still needing processing (not yet past end_date)
    active_thresholds = set(unique_thresholds)

    # Progress observer (based on full date x threshold work)
    observer = None
    if verbose:
        from opendeviationbar.progress import ResumableObserver

        total_work = sum(
            len(list(_date_range(resume_dates[t], end_date)))
            for t in unique_thresholds
        )
        observer = ResumableObserver(
            total=total_all_days * len(unique_thresholds),
            initial=sum(
                len(list(_date_range(start_date, end_date)))
                - len(list(_date_range(resume_dates[t], end_date)))
                for t in unique_thresholds
            ),
            desc=f"{symbol} multi-threshold",
            unit="threshold-days",
            verbose=True,
        )
        _ = total_work  # used for logging only

    for i, date in enumerate(all_dates, 1):
        # Determine which thresholds still need this date
        thresholds_for_date = {
            t for t in active_thresholds
            if date >= resume_dates[t]
        }
        if not thresholds_for_date:
            continue  # all thresholds have already passed this date

        logger.info(
            "Processing %s [%d/%d]: %s (thresholds: %s)",
            symbol, i, total_all_days, date,
            sorted(thresholds_for_date),
        )

        try:
            # Ouroboros reset for all active thresholds
            for threshold in thresholds_for_date:
                proc = active_processors[threshold]
                if proc is not None and _is_ouroboros_boundary(date, ouroboros_mode):
                    orphan_bar = proc.reset_at_ouroboros()
                    if orphan_bar is not None:
                        import polars as pl

                        orphan_bar["is_orphan"] = True
                        orphan_df = pl.DataFrame([orphan_bar])
                        written = _write_bars(orphan_df, ctxs[threshold], fatal_cache_write)
                        total_bars_per_threshold[threshold] += written

            # Fan out: single Parquet read, parallel threshold processing
            day_ctxs = {t: ctxs[t] for t in thresholds_for_date}
            day_processors = {t: active_processors[t] for t in thresholds_for_date}
            day_results = _process_day_multi_threshold(
                date, day_ctxs, day_processors, fatal_cache_write,
                n_workers=_mt_workers,
            )

            # Update state and save checkpoints per threshold
            for threshold, (bars_today, updated_proc) in day_results.items():
                total_bars_per_threshold[threshold] += bars_today
                active_processors[threshold] = updated_proc

                proc_cp = None
                if updated_proc is not None:
                    proc_cp = updated_proc.create_checkpoint(symbol)
                cp = PopulationCheckpoint.from_processor_state(
                    ctxs[threshold], date, total_bars_per_threshold[threshold], proc_cp
                )
                cp.save(checkpoint_paths[threshold])
                _save_checkpoint_to_clickhouse(cp)
                checkpoints[threshold] = cp

                logger.debug(
                    "%s@%d %s: %d bars (total: %d)",
                    symbol, threshold, date, bars_today,
                    total_bars_per_threshold[threshold],
                )
                if observer is not None:
                    observer.update(1)
                    observer.set_postfix(
                        bars=sum(total_bars_per_threshold.values())
                    )

        except (ValueError, RuntimeError, OSError, CacheWriteError) as exc:
            exc_str = str(exc)
            is_no_data = "No data available" in exc_str or "HTTP 404" in exc_str

            if is_no_data and skip_missing_days:
                logger.warning("No data for %s %s (skip_missing_days=True)", symbol, date)
                continue

            if is_no_data and date == all_dates[-1]:
                logger.warning(
                    "No data for final day %s — completing gracefully", date
                )
                break

            logger.exception("Failed to process %s for %s", symbol, date)
            if notify:
                emit_hook(
                    HookEvent.POPULATION_FAILED,
                    symbol=symbol,
                    trace_id=trace_id,
                    date=date,
                    error=str(exc),
                    total_bars=sum(total_bars_per_threshold.values()),
                )
            raise

    # --- Final orphan flush for each threshold --------------------------------
    for threshold in unique_thresholds:
        proc = active_processors[threshold]
        if proc is not None:
            import polars as pl

            orphan_bar = proc.reset_at_ouroboros()
            if orphan_bar is not None:
                orphan_bar["is_orphan"] = True
                orphan_df = pl.DataFrame([orphan_bar])
                written = _write_bars(orphan_df, ctxs[threshold], fatal_cache_write)
                total_bars_per_threshold[threshold] += written

    if observer is not None:
        observer.close()

    # --- Post-population dedup for each threshold ----------------------------
    for threshold in unique_thresholds:
        try:
            from opendeviationbar.clickhouse import OpenDeviationBarCache

            with OpenDeviationBarCache() as cache:
                cache.deduplicate_bars(symbol, threshold)
        except (ImportError, ConnectionError) as e:
            logger.debug(
                "Post-population dedup skipped for %s@%d: %s", symbol, threshold, e
            )
        except Exception as e:
            logger.warning(
                "Post-population dedup incomplete for %s@%d: %s", symbol, threshold, e
            )

    # --- Cleanup checkpoints --------------------------------------------------
    for threshold in unique_thresholds:
        checkpoint_paths[threshold].unlink(missing_ok=True)

    total_bars_all = sum(total_bars_per_threshold.values())
    logger.info(
        "Multi-threshold population complete for %s: %d total bars "
        "across thresholds %s from %s to %s",
        symbol, total_bars_all, unique_thresholds, start_date, end_date,
    )

    if notify:
        emit_hook(
            HookEvent.POPULATION_COMPLETE,
            symbol=symbol,
            trace_id=trace_id,
            start_date=start_date,
            end_date=end_date,
            total_bars=total_bars_all,
            thresholds=unique_thresholds,
        )

    return total_bars_per_threshold


def populate_cache_resumable(  # noqa: PLR0915
    symbol: str,
    start_date: str,
    end_date: str,
    *,
    threshold_decimal_bps: int | list[int] = 250,
    force_refresh: bool = False,
    include_microstructure: bool = False,
    ouroboros_mode: str | None = None,
    checkpoint_dir: Path | None = None,
    notify: bool = True,
    verbose: bool = True,
    inter_bar_lookback_bars: int | None = None,
    num_async_writers: int = 0,
    max_days: int | None = 366,
    skip_missing_days: bool = False,
    market: str | None = None,
    compute_tier2: bool = False,  # noqa: ARG001 — read from Settings
    compute_tier3: bool = False,  # noqa: ARG001 — read from Settings
    compute_hurst: bool = False,  # noqa: ARG001 — read from Settings
    compute_permutation_entropy: bool = False,  # noqa: ARG001 — read from Settings
) -> int:
    """Populate cache for a date range with automatic checkpointing.

    This function fetches tick data and computes open deviation bars day-by-day,
    saving progress after each day. If interrupted, simply run again
    with the same parameters to resume from the last checkpoint.

    Issue #69 Enhancements:
    - Bar-level resumability: Preserves incomplete bar state via processor checkpoints
    - Hybrid storage: Checkpoints saved to both local and ClickHouse
    - force_refresh: Wipe cache and checkpoint to start fresh

    Issue #126: ouroboros defaults to None, resolved
    from OPENDEVIATIONBAR_OUROBOROS_MODE.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT").
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).
    threshold_decimal_bps : int | list[int]
        Threshold(s) in decimal basis points (default: 250 = 0.25%).
        Pass a list (e.g. ``[250, 500, 750]``) to process multiple
        thresholds sharing a single Parquet read per day -- the multi-threshold
        fast path.  When a list is given the return value is the sum of bars
        written across all thresholds (for backward compatibility); use
        :func:`populate_cache_multi_threshold` directly to receive a per-threshold
        breakdown.
    force_refresh : bool
        If True, wipe existing cache and checkpoint, start fresh.
        If False (default), resume from last checkpoint.
    include_microstructure : bool
        Include microstructure features (default: False).
    ouroboros_mode : str
        Ouroboros reset mode (default: "aion").
    checkpoint_dir : Path | None
        Custom checkpoint directory. Uses default if None.
    notify : bool
        Whether to emit hook events for progress tracking.
    verbose : bool
        Show progress bar (tqdm) and structured logging (default: True).
        Set to False for batch/CI environments.
    num_async_writers : int
        Number of concurrent ClickHouse write workers (default: 0 = disabled).
        If > 0, enables async writes with connection pooling for 2-3x speedup.
        Issue #96 Phase 4: Async cache writes + connection pooling.
    max_days : int | None
        Maximum allowed date range span in days (default: 366).
        Raises ValueError if the range exceeds this limit.
        Pass ``None`` to bypass (used by kintsugi for authorized historical backfill).
        Issue #158 Phase 9: Prevents monolithic multi-year jobs.
    skip_missing_days : bool
        If True, skip days that return 404/no-data instead of failing.
        Used by Vision-first path where recent days may not yet be published.
    market : str | None
        Market type override. If None, resolved from symbol registry.

    Returns
    -------
    int
        Total number of bars written.  When ``threshold_decimal_bps`` is a
        list, this is the sum across all thresholds.

    Examples
    --------
    >>> bars = populate_cache_resumable("BTCUSDT", "2024-01-01", "2024-06-30")
    >>> print(f"Populated {bars} bars")
    14523

    >>> # Multi-threshold fast path (single Parquet read per day):
    >>> bars = populate_cache_resumable(
    ...     "BTCUSDT", "2024-01-01", "2024-06-30",
    ...     threshold_decimal_bps=[250, 500, 750],
    ... )

    >>> # If interrupted, just run again:
    >>> bars = populate_cache_resumable("BTCUSDT", "2024-01-01", "2024-06-30")
    >>> # Will resume from last checkpoint (bar-level accuracy)

    >>> # Start fresh (wipe cache and checkpoint):
    >>> bars = populate_cache_resumable(
    ...     "BTCUSDT", "2024-01-01", "2024-06-30",
    ...     force_refresh=True,
    ... )
    """
    # --- Multi-threshold delegation ------------------------------------------
    # When a list of thresholds is given, delegate to the dedicated fast path
    # that reads each Parquet tick file only once per day.
    if isinstance(threshold_decimal_bps, list):
        result = populate_cache_multi_threshold(
            symbol,
            start_date,
            end_date,
            threshold_decimal_bps,
            force_refresh=force_refresh,
            include_microstructure=include_microstructure,
            ouroboros_mode=ouroboros_mode,
            checkpoint_dir=checkpoint_dir,
            notify=notify,
            verbose=verbose,
            inter_bar_lookback_bars=inter_bar_lookback_bars,
            num_async_writers=num_async_writers,
            max_days=max_days,
            skip_missing_days=skip_missing_days,
            market=market,
        )
        return sum(result.values())

    from opendeviationbar.hooks import HookEvent, emit_hook
    from opendeviationbar.logging import generate_trace_id, log_checkpoint_event

    trace_id = generate_trace_id("pop")

    # Symbol registry gate + start date clamping (Issue #79)
    from opendeviationbar.symbol_registry import (
        validate_and_clamp_start_date,
        validate_symbol_registered,
    )

    validate_symbol_registered(symbol, operation="populate_cache_resumable")
    start_date = validate_and_clamp_start_date(symbol, start_date)

    # Resolve market from registry if not explicitly provided
    from opendeviationbar.symbol_registry import resolve_market

    market_resolved = resolve_market(symbol, market)

    # Issue #158 Phase 9: Hard guardrail against monolithic multi-year jobs.
    # Use kintsugi:catchup for historical backfill (recency-first ordering).
    if max_days is not None:  # SSoT-OK: intentional safety guard
        _parsed_start = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
        _parsed_end = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)
        span = (_parsed_end - _parsed_start).days
        if span > max_days:
            msg = (
                f"Date range {start_date} to {end_date} spans {span} days "
                f"(max_days={max_days}). Use kintsugi for historical backfill "
                f"(mise run kintsugi:catchup), or pass max_days=None to override."
            )
            raise ValueError(msg)

    # Threshold validation: enforce per-symbol minimums (Issue #115)
    from opendeviationbar.threshold import resolve_and_validate_threshold

    threshold_decimal_bps = resolve_and_validate_threshold(
        symbol,
        threshold_decimal_bps,
    )

    # Issue #126: Resolve ouroboros mode from config if not specified
    if ouroboros_mode is None:
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode

        ouroboros_mode = get_operational_ouroboros_mode()
    logger.info("populate_cache_resumable: ouroboros_mode=%s", ouroboros_mode)

    # T-1 guard: Binance Vision publishes with ~1-day lag.
    # Attempting to fetch today's data causes RuntimeError: "No data available"
    # which crashes the job after checkpoint is written but before ClickHouse
    # receives bars (gap regression scenario).
    today_utc = datetime.now(UTC).date()
    t1_cutoff = today_utc - timedelta(days=1)
    end_date_parsed = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC).date()
    if end_date_parsed >= today_utc:
        logger.warning(
            "end_date %s is today or future; clamping to T-1 (%s) "
            "(Binance Vision publishes with ~1-day lag)",
            end_date_parsed,
            t1_cutoff,
        )
        end_date = t1_cutoff.strftime("%Y-%m-%d")

    checkpoint_path = _get_checkpoint_path(
        symbol,
        threshold_decimal_bps,
        start_date,
        end_date,
        checkpoint_dir,
        ouroboros_mode=ouroboros_mode,  # Issue #126: mode in filename
    )

    # Issue #69: Handle force_refresh - wipe cache and checkpoint
    if force_refresh:
        logger.info("Force refresh: clearing cache and checkpoint for %s", symbol)
        # Delete local checkpoint (missing_ok=True prevents race with concurrent jobs)
        checkpoint_path.unlink(missing_ok=True)
        logger.debug("Deleted local checkpoint: %s", checkpoint_path)

        # Delete cached bars and ClickHouse checkpoint
        try:
            from opendeviationbar.clickhouse import OpenDeviationBarCache

            # Convert dates to timestamps for deletion
            from opendeviationbar.orchestration.helpers import (
                _datetime_to_end_ms,
                _datetime_to_start_ms,
            )

            start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
            end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)
            start_ts = _datetime_to_start_ms(start_dt)
            end_ts = _datetime_to_end_ms(end_dt)

            with OpenDeviationBarCache() as cache:
                cache.delete_bars(
                    symbol,
                    threshold_decimal_bps,
                    start_ts,
                    end_ts,
                    ouroboros_mode=ouroboros_mode,  # Issue #126
                )
                cache.delete_checkpoint(
                    symbol,
                    threshold_decimal_bps,
                    start_date,
                    end_date,
                    ouroboros_mode=ouroboros_mode,
                )
                logger.debug("Deleted cached bars and ClickHouse checkpoint")
        except (ImportError, ConnectionError) as e:
            logger.debug("ClickHouse cleanup skipped: %s", e)

        checkpoint = None
    else:
        # Check for existing checkpoint (local first, then ClickHouse)
        checkpoint = PopulationCheckpoint.load(checkpoint_path)

        # Issue #69: Try ClickHouse checkpoint for cross-machine resume
        if checkpoint is None:
            checkpoint = _load_checkpoint_from_clickhouse(
                symbol,
                threshold_decimal_bps,
                start_date,
                end_date,
                ouroboros_mode=ouroboros_mode,
            )

    resume_date = start_date
    total_bars = 0

    if checkpoint:
        # Validate checkpoint matches our parameters
        if (
            checkpoint.symbol == symbol
            and checkpoint.threshold_bps == threshold_decimal_bps
            and checkpoint.start_date == start_date
            and checkpoint.end_date == end_date
        ):
            # Issue #126: Verify ouroboros_mode consistency
            if (
                hasattr(checkpoint, "ouroboros_mode")
                and checkpoint.ouroboros_mode
                and checkpoint.ouroboros_mode != ouroboros_mode
            ):
                msg = (
                    f"Checkpoint ouroboros_mode mismatch: checkpoint has "
                    f"{checkpoint.ouroboros_mode!r} but current mode is "
                    f"{ouroboros_mode!r}. Delete the checkpoint or use matching mode."
                )
                raise ValueError(msg)

            # Resume from day after last completed
            last_completed = datetime.strptime(
                checkpoint.last_completed_date, "%Y-%m-%d"
            ).replace(tzinfo=UTC)
            resume_date = (last_completed + timedelta(days=1)).strftime("%Y-%m-%d")
            total_bars = checkpoint.bars_written

            logger.info(
                "Resuming %s population from %s (%d bars already written)",
                symbol,
                resume_date,
                total_bars,
            )

            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_SAVED,
                    symbol=symbol,
                    trace_id=trace_id,
                    action="resumed",
                    last_completed_date=checkpoint.last_completed_date,
                    bars_written=total_bars,
                )
        else:
            logger.warning("Checkpoint parameters don't match, starting fresh")
            checkpoint = None

    # Issue #96: Compute full date range once (was materialized 3 times)
    all_dates = list(_date_range(start_date, end_date))
    total_all_days = len(all_dates)

    # Process day by day
    dates = list(_date_range(resume_date, end_date))
    total_days = len(dates)

    # Issue #70: Progress tracking with ResumableObserver
    # Calculate initial count for resumed operations
    initial_days = total_all_days - total_days if checkpoint else 0

    # Initialize progress observer (lazy import to avoid circular deps)
    observer = None
    if verbose:
        from opendeviationbar.progress import ResumableObserver

        observer = ResumableObserver(
            total=total_all_days,
            initial=initial_days,
            desc=symbol,
            unit="days",
            verbose=True,
        )

    # Issue #97: Thread processor across days for bar-level continuity.
    import opendeviationbar as _odb_mod
    from opendeviationbar.exceptions import CacheWriteError
    from opendeviationbar.orchestration.helpers import _parse_microstructure_env_vars
    from opendeviationbar.orchestration.open_deviation_bars_cache import (
        fatal_cache_write,
    )
    from opendeviationbar.processors.core import OpenDeviationBarProcessor
    from opendeviationbar.storage.parquet import TickStorage

    source_version = f"{getattr(_odb_mod, '__version__', 'unknown')}+populate"

    eff_count, eff_bars, enable_intra = _parse_microstructure_env_vars(
        include_microstructure,
        None,
        inter_bar_lookback_bars,
    )
    active_processor: OpenDeviationBarProcessor | None = None

    # Restore processor from checkpoint if available
    if checkpoint and checkpoint.processor_checkpoint:
        try:
            active_processor = OpenDeviationBarProcessor.from_checkpoint(
                checkpoint.processor_checkpoint,
            )
            # Issue #128: Warn if feature config changed since checkpoint
            pop = _get_population_config()
            if (
                checkpoint.compute_tier2 != pop.compute_tier2
                or checkpoint.compute_tier3 != pop.compute_tier3
                or checkpoint.compute_hurst != pop.compute_hurst
                or checkpoint.compute_permutation_entropy
                != pop.compute_permutation_entropy
            ):
                logger.warning(
                    "Feature config changed since checkpoint "
                    "(ckpt: T2=%s T3=%s H=%s PE=%s → "
                    "now: T2=%s T3=%s H=%s PE=%s). "
                    "New bars use current config; "
                    "backfill old bars if needed.",
                    checkpoint.compute_tier2,
                    checkpoint.compute_tier3,
                    checkpoint.compute_hurst,
                    checkpoint.compute_permutation_entropy,
                    pop.compute_tier2,
                    pop.compute_tier3,
                    pop.compute_hurst,
                    pop.compute_permutation_entropy,
                )
            # Re-enable microstructure features after checkpoint restore
            if include_microstructure:
                active_processor.enable_microstructure(
                    inter_bar_lookback_count=eff_count,
                    inter_bar_lookback_bars=eff_bars,
                    include_intra_bar_features=enable_intra,
                    compute_tier2=pop.compute_tier2,
                    compute_tier3=pop.compute_tier3,
                    compute_hurst=pop.compute_hurst or None,
                    compute_permutation_entropy=pop.compute_permutation_entropy or None,
                )
            logger.info(
                "Restored processor from checkpoint "
                "(has_incomplete_bar=%s, defer_open=%s)",
                checkpoint.processor_checkpoint.get("has_incomplete_bar"),
                checkpoint.processor_checkpoint.get("defer_open"),
            )
            log_checkpoint_event(
                "checkpoint_restore",
                symbol,
                trace_id,
                threshold_dbps=threshold_decimal_bps,
                last_completed_date=checkpoint.last_completed_date,
                bars_written=checkpoint.bars_written,
                has_incomplete_bar=checkpoint.processor_checkpoint.get(
                    "has_incomplete_bar"
                ),
                last_trade_id=checkpoint.processor_checkpoint.get("last_trade_id"),
            )
            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_RESTORED,
                    symbol=symbol,
                    trace_id=trace_id,
                    threshold_dbps=threshold_decimal_bps,
                    has_incomplete_bar=checkpoint.processor_checkpoint.get(
                        "has_incomplete_bar"
                    ),
                    last_trade_id=checkpoint.processor_checkpoint.get("last_trade_id"),
                    defer_open=checkpoint.processor_checkpoint.get("defer_open"),
                )
        except Exception:
            logger.exception(
                "Failed to restore processor from checkpoint, starting fresh",
            )
            log_checkpoint_event(
                "checkpoint_restore_failed",
                symbol,
                trace_id,
                threshold_dbps=threshold_decimal_bps,
                fallback="fresh_processor",
            )
            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_RESTORE_FAILED,
                    symbol=symbol,
                    trace_id=trace_id,
                    threshold_dbps=threshold_decimal_bps,
                )
            active_processor = None

    created_at = checkpoint.created_at if checkpoint else datetime.now(UTC).isoformat()

    # Issue #96 Phase 4: Initialize thread pool for concurrent ClickHouse writes
    thread_pool = None
    if num_async_writers > 0:
        try:
            from concurrent.futures import ThreadPoolExecutor

            from opendeviationbar.clickhouse.connection_pool import get_connection_pool

            get_connection_pool(max_pool_size=num_async_writers * 2)
            logger.info(
                "Initialized ClickHouse connection pool: max=%d",
                num_async_writers * 2,
            )
            thread_pool = ThreadPoolExecutor(max_workers=num_async_writers)
            logger.info(
                "Concurrent cache writer initialized: %d workers",
                num_async_writers,
            )
        except (ImportError, ConnectionError, OSError) as e:
            logger.warning("Failed to initialize thread pool: %s", e)
            logger.info("Falling back to synchronous writes")
            thread_pool = None

    # Build loop-constant context
    ctx = PopulationContext(
        symbol=symbol,
        threshold_decimal_bps=threshold_decimal_bps,
        ouroboros_mode=ouroboros_mode,
        include_microstructure=include_microstructure,
        notify=notify,
        inter_bar_lookback_count=eff_count,
        inter_bar_lookback_bars=eff_bars,
        include_intra_bar_features=enable_intra,
        source_version=source_version,
        storage=TickStorage(),
        thread_pool=thread_pool,
        observer=observer,
        trace_id=trace_id,
        created_at=created_at,
        start_date=start_date,
        end_date=end_date,
        market=market_resolved,
    )

    for i, date in enumerate(dates, 1):
        logger.info(
            "Processing %s [%d/%d]: %s",
            symbol,
            i,
            total_days,
            date,
        )

        try:
            # Issue #97: Handle ouroboros boundary -- reset processor state
            if active_processor is not None and _is_ouroboros_boundary(
                date, ouroboros_mode
            ):
                logger.info(
                    "Ouroboros %s boundary at %s — resetting processor",
                    ouroboros_mode,
                    date,
                )
                log_checkpoint_event(
                    "ouroboros_reset",
                    symbol,
                    trace_id,
                    date=date,
                    boundary_type=ouroboros_mode,
                    had_incomplete_bar=active_processor.get_incomplete_bar()
                    is not None,
                )
                orphan_bar = active_processor.reset_at_ouroboros()
                if orphan_bar is not None:
                    import polars as pl

                    orphan_bar["is_orphan"] = True
                    orphan_df = pl.DataFrame([orphan_bar])
                    written = _write_bars(orphan_df, ctx, fatal_cache_write)
                    total_bars += written

            # Process day: fetch ticks -> stream bars -> write to ClickHouse
            bars_today, active_processor = _process_day(
                date, ctx, active_processor, fatal_cache_write,
            )
            total_bars += bars_today

            logger.debug(
                "%s %s: %d bars (total: %d)",
                symbol,
                date,
                bars_today,
                total_bars,
            )

            # Issue #70: Update progress observer
            if observer is not None:
                observer.update(1)
                observer.set_postfix(bars=total_bars)
                observer.log_event(
                    "day_complete",
                    date=date,
                    bars_today=bars_today,
                    total_bars=total_bars,
                )

            # Issue #97: Save processor checkpoint for bar-level resumability
            proc_cp = None
            if active_processor is not None:
                proc_cp = active_processor.create_checkpoint(symbol)

            checkpoint = PopulationCheckpoint.from_processor_state(
                ctx, date, total_bars, proc_cp,
            )
            checkpoint.save(checkpoint_path)
            _save_checkpoint_to_clickhouse(checkpoint)

            log_checkpoint_event(
                "checkpoint_save",
                symbol,
                trace_id,
                threshold_dbps=threshold_decimal_bps,
                date=date,
                bars_today=bars_today,
                total_bars=total_bars,
                has_incomplete_bar=proc_cp is not None
                and proc_cp.get("has_incomplete_bar", False),
                defer_open=proc_cp.get("defer_open") if proc_cp else None,
            )

            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_SAVED,
                    symbol=symbol,
                    trace_id=trace_id,
                    date=date,
                    bars_today=bars_today,
                    total_bars=total_bars,
                    progress_pct=round(i / total_days * 100, 1),
                )

        except (ValueError, RuntimeError, OSError, CacheWriteError) as exc:
            # Defense-in-depth: if the final day has no data (archive not yet
            # published on Binance Vision), complete gracefully instead of
            # crashing the entire multi-day job at 99.96% completion.
            exc_str = str(exc)
            is_no_data = "No data available" in exc_str or "HTTP 404" in exc_str

            if is_no_data and skip_missing_days:
                logger.warning(
                    "No data for %s (skipping, skip_missing_days=True) "
                    "— continuing with %d bars so far",
                    date,
                    total_bars,
                )
                continue

            if is_no_data and date == dates[-1]:
                logger.warning(
                    "No data for final day %s (archive not yet published?) "
                    "— completing gracefully with %d bars",
                    date,
                    total_bars,
                )
                break

            logger.exception(
                "Failed to process %s for %s",
                symbol,
                date,
            )
            if notify:
                emit_hook(
                    HookEvent.POPULATION_FAILED,
                    symbol=symbol,
                    trace_id=trace_id,
                    date=date,
                    error=str(exc),
                    total_bars=total_bars,
                )
            raise

    # Flush final orphan -- the last day's incomplete bar is still in the processor.
    # Without this, low-volatility days at high thresholds produce 0 completed bars
    # and the final incomplete bar is lost.
    if active_processor is not None:
        orphan_bar = active_processor.reset_at_ouroboros()
        if orphan_bar is not None:
            import polars as pl

            orphan_bar["is_orphan"] = True
            orphan_df = pl.DataFrame([orphan_bar])
            written = _write_bars(orphan_df, ctx, fatal_cache_write)
            total_bars += written

    # Issue #70: Close progress observer
    if observer is not None:
        observer.close()

    # Issue #77: Deduplicate after population completes
    # ReplacingMergeTree only deduplicates during background merges,
    # so we force immediate deduplication to ensure clean data.
    # Issue #90: Non-fatal -- dedup failure should not crash the job.
    # INSERT dedup tokens (Issue #90) prevent most duplicates at source,
    # read queries use FINAL for on-the-fly dedup, and the pipeline-level
    # OPTIMIZE TABLE FINAL job catches any stragglers.
    try:
        from opendeviationbar.clickhouse import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            logger.info(
                "Running post-population deduplication for %s @ %d dbps",
                symbol,
                threshold_decimal_bps,
            )
            cache.deduplicate_bars(symbol, threshold_decimal_bps)
            logger.debug("Deduplication complete")
    except (ImportError, ConnectionError) as e:
        logger.debug("Post-population deduplication skipped: %s", e)
    except Exception as e:
        # Non-fatal: data is already written. Common cause is ClickHouse
        # OperationalError from OPTIMIZE TABLE timeout on large partitions.
        # Background merges or the pipeline-level OPTIMIZE job handle dedup.
        logger.warning(
            "Post-population deduplication did not complete for %s @ %d dbps: %s. "
            "Data is written. Run OPTIMIZE TABLE FINAL manually or via pipeline.",
            symbol,
            threshold_decimal_bps,
            e,
        )

    # Clean up checkpoint on success (missing_ok for concurrent jobs)
    checkpoint_path.unlink(missing_ok=True)
    logger.debug("Removed checkpoint file after successful completion")

    logger.info(
        "Population complete for %s: %d bars from %s to %s",
        symbol,
        total_bars,
        start_date,
        end_date,
    )

    if notify:
        emit_hook(
            HookEvent.POPULATION_COMPLETE,
            symbol=symbol,
            trace_id=trace_id,
            start_date=start_date,
            end_date=end_date,
            total_bars=total_bars,
        )

    # Issue #96 Phase 4: Wait for pending async writes to complete
    if thread_pool is not None:
        logger.info("Waiting for pending ClickHouse writes to complete...")
        thread_pool.shutdown(wait=True)
        logger.info("All writes completed, thread pool shut down")

    return total_bars


def list_checkpoints(
    checkpoint_dir: Path | None = None,
) -> list[PopulationCheckpoint]:
    """List all existing checkpoints.

    Parameters
    ----------
    checkpoint_dir : Path | None
        Custom checkpoint directory. Uses default if None.

    Returns
    -------
    list[PopulationCheckpoint]
        List of checkpoints found.
    """
    if checkpoint_dir is None:
        checkpoint_dir = _CHECKPOINT_DIR

    if not checkpoint_dir.exists():
        return []

    checkpoints = []
    for path in checkpoint_dir.glob("*.json"):
        checkpoint = PopulationCheckpoint.load(path)
        if checkpoint:
            checkpoints.append(checkpoint)

    return checkpoints


def clear_checkpoint(
    symbol: str,
    threshold_decimal_bps: int,
    start_date: str,
    end_date: str,
    checkpoint_dir: Path | None = None,
    ouroboros_mode: str = "aion",
) -> bool:
    """Clear a specific checkpoint.

    Parameters
    ----------
    symbol : str
        Trading symbol.
    threshold_decimal_bps : int
        Threshold in decimal basis points.
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).
    checkpoint_dir : Path | None
        Custom checkpoint directory. Uses default if None.
    ouroboros_mode : str
        Ouroboros mode for checkpoint filename (Issue #126).

    Returns
    -------
    bool
        True if checkpoint was found and removed.
    """
    checkpoint_path = _get_checkpoint_path(
        symbol,
        threshold_decimal_bps,
        start_date,
        end_date,
        checkpoint_dir,
        ouroboros_mode=ouroboros_mode,
    )

    if checkpoint_path.exists():
        checkpoint_path.unlink(missing_ok=True)
        logger.info("Cleared checkpoint: %s", checkpoint_path)
        return True

    return False


__all__ = [
    "PopulationCheckpoint",
    "clear_checkpoint",
    "list_checkpoints",
    "populate_cache_multi_threshold",
    "populate_cache_resumable",
]
