# FILE-SIZE-OK
"""Checkpoint system for resumable cache population (Issue #40, Issue #69).

Enables long-running cache population jobs to be resumed after interruption.
Uses atomic file writes with bar-level checkpointing for accurate resume.

Issue #69 Enhancements:
- Bar-level resumability: Preserves incomplete bar state via processor checkpoints
- Hybrid storage: Local + ClickHouse for cross-machine resume
- force_refresh parameter: Wipe cache and checkpoint to start fresh

Usage
-----
>>> from opendeviationbar.checkpoint import populate_cache_resumable
>>> bars = populate_cache_resumable(
...     symbol="BTCUSDT",
...     start_date="2024-01-01",
...     end_date="2024-06-30",
... )
>>> print(f"Populated {bars} bars")

If interrupted, simply run again - it will resume from the last checkpoint.
Use force_refresh=True to wipe cache and start over.
"""

from __future__ import annotations

import json
import logging
import os
import socket
import tempfile
from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterator

from platformdirs import user_cache_dir

if TYPE_CHECKING:
    from concurrent.futures import ThreadPoolExecutor

    from opendeviationbar.config.backfill import BackfillConfig
    from opendeviationbar.processors.core import OpenDeviationBarProcessor
    from opendeviationbar.progress import ResumableObserver
    from opendeviationbar.storage.parquet import TickStorage

logger = logging.getLogger(__name__)

# Default checkpoint directory
_CHECKPOINT_DIR = Path(user_cache_dir("opendeviationbar", "terrylica")) / "checkpoints"


def _get_population_config():  # noqa: ANN202
    """Lazy access to PopulationConfig (avoids circular import)."""
    from opendeviationbar.config import Settings

    return Settings.get().population


@dataclass
class PopulationContext:
    """Bundle of loop-constant state for cache population.

    Groups parameters that remain constant throughout the populate loop,
    reducing parameter passing from ~15 args to a single context object.
    """

    symbol: str
    threshold_decimal_bps: int
    ouroboros_mode: str
    include_microstructure: bool
    notify: bool
    inter_bar_lookback_count: int | None
    inter_bar_lookback_bars: int | None
    include_intra_bar_features: bool
    source_version: str
    storage: TickStorage
    thread_pool: ThreadPoolExecutor | None
    observer: ResumableObserver | None
    trace_id: str
    created_at: str
    start_date: str
    end_date: str
    market: str = "spot"


@dataclass
class PopulationCheckpoint:
    """Checkpoint for resumable cache population.

    Tracks progress of a multi-day cache population job, allowing
    resumption after interruption with bar-level accuracy.

    Issue #69 Enhancements:
    - processor_checkpoint: Preserves incomplete bar state
    - last_trade_timestamp_ms: For mid-day resume filtering
    - include_microstructure: Track feature configuration
    - ouroboros_mode: Track reset mode for consistency

    Attributes
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT").
    threshold_bps : int
        Threshold in decimal basis points.
    start_date : str
        Original start date (YYYY-MM-DD).
    end_date : str
        Target end date (YYYY-MM-DD).
    last_completed_date : str
        Most recent successfully completed date.
    bars_written : int
        Total bars written so far.
    created_at : str
        ISO timestamp of checkpoint creation.
    updated_at : str
        ISO timestamp of last update.
    processor_checkpoint : dict | None
        Serialized processor state (incomplete bar, defer_open).
        Enables bar-level resumability (Issue #69).
    last_trade_timestamp_ms : int | None
        Timestamp of last processed trade for mid-day resume.
    include_microstructure : bool
        Whether microstructure features are enabled.
    ouroboros_mode : str
        Ouroboros reset mode for consistency.
    """

    symbol: str
    threshold_bps: int
    start_date: str
    end_date: str
    last_completed_date: str
    bars_written: int = 0
    created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
    updated_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
    # Issue #69: Bar-level resumability
    processor_checkpoint: dict | None = None
    last_trade_timestamp_ms: int | None = None
    include_microstructure: bool = False
    ouroboros_mode: str = "day"
    # Issue #72: Full Audit Trail - agg_trade_id range in incomplete bar
    first_agg_trade_id_in_bar: int | None = None
    last_agg_trade_id_in_bar: int | None = None
    # Issue #111 (Ariadne): High-water mark for deterministic resume via fromId
    last_processed_agg_trade_id: int | None = None
    # Issue #128: Feature computation config at checkpoint time
    compute_tier2: bool = True
    compute_tier3: bool = False
    compute_hurst: bool = False
    compute_permutation_entropy: bool = False

    def to_dict(self) -> dict[str, Any]:
        """Convert to dictionary for JSON serialization."""
        return asdict(self)

    @classmethod
    def from_processor_state(
        cls,
        ctx: PopulationContext,
        date: str,
        total_bars: int,
        proc_cp: dict | None,
    ) -> PopulationCheckpoint:
        """Factory: construct checkpoint from current processor state.

        Consolidates the ~30-line checkpoint construction that was
        duplicated in the populate loop.
        """
        pop = _get_population_config()
        return cls(
            symbol=ctx.symbol,
            threshold_bps=ctx.threshold_decimal_bps,
            start_date=ctx.start_date,
            end_date=ctx.end_date,
            last_completed_date=date,
            bars_written=total_bars,
            created_at=ctx.created_at,
            include_microstructure=ctx.include_microstructure,
            ouroboros_mode=ctx.ouroboros_mode,
            processor_checkpoint=proc_cp,
            last_trade_timestamp_ms=(
                proc_cp.get("last_timestamp_ms") if proc_cp else None
            ),
            first_agg_trade_id_in_bar=(
                (proc_cp.get("incomplete_bar_raw") or {}).get("first_agg_trade_id")
                if proc_cp
                else None
            ),
            last_agg_trade_id_in_bar=(
                (proc_cp.get("incomplete_bar_raw") or {}).get("last_agg_trade_id")
                if proc_cp
                else None
            ),
            last_processed_agg_trade_id=(
                proc_cp.get("last_trade_id") if proc_cp else None
            ),
            compute_tier2=pop.compute_tier2,
            compute_tier3=pop.compute_tier3,
            compute_hurst=pop.compute_hurst,
            compute_permutation_entropy=pop.compute_permutation_entropy,
        )

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> PopulationCheckpoint:
        """Create from dictionary, ignoring unknown keys (e.g., provenance)."""
        import dataclasses

        field_names = {f.name for f in dataclasses.fields(cls)}
        filtered = {k: v for k, v in data.items() if k in field_names}
        return cls(**filtered)

    def save(self, path: Path) -> None:
        """Save checkpoint to file with atomic write.

        Uses tempfile + fsync + rename pattern for crash safety.

        Parameters
        ----------
        path : Path
            Path to save checkpoint file.
        """
        self.updated_at = datetime.now(UTC).isoformat()
        d = self.to_dict()
        d["provenance"] = _checkpoint_provenance()
        data = json.dumps(d, indent=2)

        # Ensure parent directory exists
        path.parent.mkdir(parents=True, exist_ok=True)

        # Atomic write: write to temp file, fsync, rename
        fd, temp_path = tempfile.mkstemp(
            dir=path.parent,
            prefix=".checkpoint_",
            suffix=".tmp",
        )
        try:
            with os.fdopen(fd, "w") as f:
                f.write(data)
                f.flush()
                os.fsync(f.fileno())

            # Atomic rename (POSIX guarantees this is atomic)
            os.replace(temp_path, path)
            logger.debug("Saved checkpoint to %s", path)
        except (OSError, RuntimeError):
            # Clean up temp file on failure
            try:
                os.unlink(temp_path)
            except OSError:
                pass
            raise

    @classmethod
    def load(cls, path: Path) -> PopulationCheckpoint | None:
        """Load checkpoint from file.

        Parameters
        ----------
        path : Path
            Path to checkpoint file.

        Returns
        -------
        PopulationCheckpoint | None
            Loaded checkpoint, or None if file doesn't exist.
        """
        if not path.exists():
            return None

        try:
            data = json.loads(path.read_text())
            return cls.from_dict(data)
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            logger.warning("Failed to load checkpoint from %s: %s", path, e)
            return None


def _get_checkpoint_path(
    symbol: str,
    threshold_decimal_bps: int,
    start_date: str,
    end_date: str,
    checkpoint_dir: Path | None = None,
    ouroboros_mode: str = "day",
) -> Path:
    """Get the checkpoint file path for a population job.

    Parameters
    ----------
    symbol : str
        Trading symbol.
    threshold_decimal_bps : int
        Threshold in decimal basis points. Included in filename to prevent
        collisions when multiple thresholds run concurrently (Issue #84).
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).
    checkpoint_dir : Path | None
        Custom checkpoint directory. Uses default if None.
    ouroboros_mode : str
        Ouroboros mode included in filename to prevent cross-mode
        checkpoint pollution (Issue #126).

    Returns
    -------
    Path
        Path to the checkpoint file.
    """
    if checkpoint_dir is None:
        checkpoint_dir = _CHECKPOINT_DIR

    # Issue #126: Mode in filename prevents cross-mode resume contamination
    filename = (
        f"{symbol}_{threshold_decimal_bps}_{start_date}_{end_date}"
        f"_{ouroboros_mode}.json"
    )
    return checkpoint_dir / filename


def _load_checkpoint_from_clickhouse(
    symbol: str,
    threshold_decimal_bps: int,
    start_date: str,
    end_date: str,
    ouroboros_mode: str = "day",
) -> PopulationCheckpoint | None:
    """Load checkpoint from ClickHouse for cross-machine resume.

    Issue #69: Hybrid checkpoint storage enables resume on different machines.

    Parameters
    ----------
    symbol : str
        Trading symbol.
    threshold_decimal_bps : int
        Threshold in decimal basis points.
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).
    ouroboros_mode : str
        Ouroboros reset mode filter (default: "day").
        Prevents cross-mode checkpoint pollution.

    Returns
    -------
    PopulationCheckpoint | None
        Loaded checkpoint, or None if not found.
    """
    try:
        from opendeviationbar.clickhouse import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            data = cache.load_checkpoint(
                symbol,
                threshold_decimal_bps,
                start_date,
                end_date,
                ouroboros_mode=ouroboros_mode,
            )
            if data is None:
                return None

            # Convert ClickHouse data to PopulationCheckpoint
            return PopulationCheckpoint(
                symbol=symbol,
                threshold_bps=threshold_decimal_bps,
                start_date=start_date,
                end_date=end_date,
                last_completed_date=data["last_completed_date"],
                bars_written=data["bars_written"],
                processor_checkpoint=json.loads(data["processor_checkpoint"])
                if data.get("processor_checkpoint")
                else None,
                last_trade_timestamp_ms=data.get("last_trade_timestamp_ms"),
                include_microstructure=data.get("include_microstructure", False),
                ouroboros_mode=data.get("ouroboros_mode", "day"),
                # Issue #72: Full Audit Trail
                first_agg_trade_id_in_bar=data.get("first_agg_trade_id_in_bar"),
                last_agg_trade_id_in_bar=data.get("last_agg_trade_id_in_bar"),
                # Issue #111 (Ariadne): High-water mark
                last_processed_agg_trade_id=data.get("last_processed_agg_trade_id"),
            )
    except (ImportError, ConnectionError, OSError, RuntimeError) as e:
        logger.debug("ClickHouse checkpoint load failed: %s", e)
        return None


def _save_checkpoint_to_clickhouse(checkpoint: PopulationCheckpoint) -> None:
    """Save checkpoint to ClickHouse for cross-machine resume.

    Issue #69: Hybrid checkpoint storage enables resume on different machines.

    Parameters
    ----------
    checkpoint : PopulationCheckpoint
        Checkpoint to save.
    """
    try:
        from opendeviationbar.clickhouse import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            cache.save_checkpoint(
                symbol=checkpoint.symbol,
                threshold_decimal_bps=checkpoint.threshold_bps,
                start_date=checkpoint.start_date,
                end_date=checkpoint.end_date,
                last_completed_date=checkpoint.last_completed_date,
                last_trade_timestamp_ms=checkpoint.last_trade_timestamp_ms,
                processor_checkpoint=json.dumps(checkpoint.processor_checkpoint)
                if checkpoint.processor_checkpoint
                else "",
                bars_written=checkpoint.bars_written,
                include_microstructure=checkpoint.include_microstructure,
                ouroboros_mode=checkpoint.ouroboros_mode,
                # Issue #72: Full Audit Trail
                first_agg_trade_id_in_bar=checkpoint.first_agg_trade_id_in_bar,
                last_agg_trade_id_in_bar=checkpoint.last_agg_trade_id_in_bar,
                # Issue #111 (Ariadne): High-water mark
                last_processed_agg_trade_id=checkpoint.last_processed_agg_trade_id,
            )
    except (ImportError, ConnectionError, OSError, RuntimeError) as e:
        logger.debug("ClickHouse checkpoint save failed (non-fatal): %s", e)


def _checkpoint_provenance(trace_id: str | None = None) -> dict:
    """Build provenance metadata for checkpoint files (Issue #97)."""
    import opendeviationbar

    return {
        "service": "opendeviationbar-py",
        "version": getattr(opendeviationbar, "__version__", "unknown"),
        "git_sha": os.environ.get("OPENDEVIATIONBAR_GIT_SHA", "unknown"),
        "host": socket.gethostname(),
        "pid": os.getpid(),
        "created_utc": datetime.now(UTC).isoformat(),
        "trace_id": trace_id,
    }


def _is_ouroboros_boundary(date_str: str, ouroboros_mode: str) -> bool:  # noqa: ARG001
    """Check if a date falls on an ouroboros reset boundary.

    Issue #97: Used by populate_cache_resumable() to reset the threaded
    processor at day boundaries. Only "day" mode is supported.
    """
    if ouroboros_mode != "day":
        msg = f"Invalid ouroboros mode: {ouroboros_mode!r}. Only 'day' mode is supported."
        raise ValueError(msg)
    return True  # Every day is a boundary in day mode


def _date_range(start_date: str, end_date: str) -> Iterator[str]:
    """Generate dates from start to end (inclusive).

    Parameters
    ----------
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).

    Yields
    ------
    str
        Dates in YYYY-MM-DD format.
    """
    from datetime import timedelta

    start = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
    end = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)

    current = start
    while current <= end:
        yield current.strftime("%Y-%m-%d")
        current += timedelta(days=1)


def _ariadne_disabled() -> bool:
    """Check if Ariadne (trade-ID resume) is explicitly disabled.

    Ariadne is ON by default. Set ``OPENDEVIATIONBAR_ARIADNE_ENABLED=false`` to
    fall back to timestamp-based resume.
    """
    val = os.environ.get("OPENDEVIATIONBAR_ARIADNE_ENABLED", "true")
    return val.lower() in ("0", "false", "no")


def _write_bars(
    bars_df: Any,  # noqa: ANN401
    ctx: PopulationContext,
    fatal_cache_write: Any,  # noqa: ANN401
) -> int:
    """Write bars to ClickHouse (thread pool or synchronous).

    Parameters
    ----------
    bars_df
        DataFrame of bars (Pandas or Polars).
    ctx : PopulationContext
        Population context with thread pool and metadata.
    fatal_cache_write
        Callable that writes bars to ClickHouse (raises on failure).

    Returns
    -------
    int
        Number of bars written.
    """
    if bars_df is None:
        return 0
    # Handle both Pandas (.empty) and Polars (.is_empty())
    if hasattr(bars_df, "empty") and bars_df.empty:
        return 0
    if hasattr(bars_df, "is_empty") and bars_df.is_empty():
        return 0

    if ctx.thread_pool is not None:
        ctx.thread_pool.submit(
            fatal_cache_write,
            bars_df,
            ctx.symbol,
            ctx.threshold_decimal_bps,
            ctx.ouroboros_mode,
            ctx.source_version,
        )
        return len(bars_df)

    return fatal_cache_write(
        bars_df,
        ctx.symbol,
        ctx.threshold_decimal_bps,
        ctx.ouroboros_mode,
        version=ctx.source_version,
    )


def _process_day(
    date: str,
    ctx: PopulationContext,
    active_processor: OpenDeviationBarProcessor | None,
    fatal_cache_write: Any,  # noqa: ANN401
) -> tuple[int, OpenDeviationBarProcessor | None]:
    """Process a single day: fetch ticks -> stream bars -> write to ClickHouse.

    Uses the streaming Arrow path — peak memory ~150MB per day vs ~2GB
    with the deprecated _fetch_binance() batch path.

    Parameters
    ----------
    date : str
        Date to process (YYYY-MM-DD).
    ctx : PopulationContext
        Loop-constant population state.
    active_processor : OpenDeviationBarProcessor | None
        Processor with state for cross-day continuity.
    fatal_cache_write
        Callable that writes bars to ClickHouse (raises on failure).

    Returns
    -------
    tuple[int, OpenDeviationBarProcessor | None]
        (bars_written, updated_processor).
    """
    import gc

    import polars as pl

    from opendeviationbar.orchestration.helpers import (
        _create_processor,
        _datetime_to_end_ms,
        _datetime_to_start_ms,
        _stream_bars_from_trades,
        _stream_open_deviation_bars_binance,
    )

    day_dt = datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=UTC)
    day_start_ms = _datetime_to_start_ms(day_dt)
    day_end_ms = _datetime_to_end_ms(day_dt)
    cache_symbol = f"BINANCE_SPOT_{ctx.symbol}".upper()

    # Ensure we have a processor (creates one if first day)
    if active_processor is None:
        from opendeviationbar.orchestration.helpers import (
            _parse_microstructure_env_vars,
        )

        eff_count, eff_bars, enable_intra = _parse_microstructure_env_vars(
            ctx.include_microstructure,
            ctx.inter_bar_lookback_count,
            ctx.inter_bar_lookback_bars,
        )
        active_processor = _create_processor(
            ctx.threshold_decimal_bps,
            ctx.include_microstructure,
            symbol=ctx.symbol,
            inter_bar_lookback_count=eff_count,
            include_intra_bar_features=enable_intra,
            inter_bar_lookback_bars=eff_bars,
        )

    def _backfill_cfg() -> BackfillConfig:
        from opendeviationbar.config.backfill import BackfillConfig
        return BackfillConfig()

    flush_threshold = _backfill_cfg().flush_threshold
    bar_frames: list[pl.DataFrame] = []
    accumulated_count = 0
    total_bars_written = 0
    flush_count = 0

    def _flush_accumulated() -> int:
        """Concat accumulated frames, enrich if needed, write to CH."""
        nonlocal bar_frames, accumulated_count, flush_count
        if not bar_frames:
            return 0
        batch_df = pl.concat(bar_frames, rechunk=False)
        bar_frames = []
        accumulated_count = 0
        flush_count += 1

        # Plugin enrichment (only if plugins installed and microstructure enabled)
        if ctx.include_microstructure and not batch_df.is_empty():
            from opendeviationbar.plugins.loader import enrich_bars

            bars_pd = batch_df.to_pandas()
            batch_df = enrich_bars(bars_pd, ctx.symbol, ctx.threshold_decimal_bps)

        written = _write_bars(batch_df, ctx, fatal_cache_write)
        del batch_df
        return written

    has_cached = ctx.storage.has_ticks(cache_symbol, day_start_ms, day_end_ms)
    if has_cached:
        # Cached path: stream from Parquet via _stream_bars_from_trades
        from opendeviationbar.orchestration.helpers import _select_trade_columns

        day_ticks = ctx.storage.read_ticks(cache_symbol, day_start_ms, day_end_ms)
        if day_ticks.is_empty():
            return 0, active_processor
        day_ticks = _select_trade_columns(day_ticks)
        for bar_batch in _stream_bars_from_trades(
            day_ticks,
            ctx.threshold_decimal_bps,
            ctx.include_microstructure,
            active_processor,
        ):
            bar_frames.append(bar_batch)
            accumulated_count += len(bar_batch)
            if accumulated_count >= flush_threshold:
                total_bars_written += _flush_accumulated()
        del day_ticks
    else:
        # Network path: stream from Binance API (Arrow, zero-copy)
        for bar_batch in _stream_open_deviation_bars_binance(
            ctx.symbol,
            date,
            date,
            ctx.threshold_decimal_bps,
            ctx.market,
            include_microstructure=ctx.include_microstructure,
            processor=active_processor,
            inter_bar_lookback_count=ctx.inter_bar_lookback_count,
        ):
            bar_frames.append(bar_batch)
            accumulated_count += len(bar_batch)
            if accumulated_count >= flush_threshold:
                total_bars_written += _flush_accumulated()

    # Final flush for remaining bars
    total_bars_written += _flush_accumulated()

    if flush_count > 1:
        logger.debug(
            "Bounded flush: %d flushes for %s on %s (%d bars total)",
            flush_count,
            ctx.symbol,
            date,
            total_bars_written,
        )

    gc.collect()

    return total_bars_written, active_processor


def populate_cache_resumable(
    symbol: str,
    start_date: str,
    end_date: str,
    *,
    threshold_decimal_bps: int = 250,
    force_refresh: bool = False,
    include_microstructure: bool = False,
    ouroboros_mode: str | None = None,
    checkpoint_dir: Path | None = None,
    notify: bool = True,
    verbose: bool = True,
    inter_bar_lookback_bars: int | None = None,
    num_async_writers: int = 0,
    max_days: int | None = 366,
    skip_missing_days: bool = False,
    market: str | None = None,
) -> int:
    """Populate cache for a date range with automatic checkpointing.

    This function fetches tick data and computes open deviation bars day-by-day,
    saving progress after each day. If interrupted, simply run again
    with the same parameters to resume from the last checkpoint.

    Issue #69 Enhancements:
    - Bar-level resumability: Preserves incomplete bar state via processor checkpoints
    - Hybrid storage: Checkpoints saved to both local and ClickHouse
    - force_refresh: Wipe cache and checkpoint to start fresh

    Issue #126: ouroboros defaults to None, resolved
    from OPENDEVIATIONBAR_OUROBOROS_MODE.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT").
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).
    threshold_decimal_bps : int
        Threshold in decimal basis points (default: 250 = 0.25%).
    force_refresh : bool
        If True, wipe existing cache and checkpoint, start fresh.
        If False (default), resume from last checkpoint.
    include_microstructure : bool
        Include microstructure features (default: False).
    ouroboros_mode : str
        Ouroboros reset mode (default: "day"). Only "day" is supported.
    checkpoint_dir : Path | None
        Custom checkpoint directory. Uses default if None.
    notify : bool
        Whether to emit hook events for progress tracking.
    verbose : bool
        Show progress bar (tqdm) and structured logging (default: True).
        Set to False for batch/CI environments.
    num_async_writers : int
        Number of concurrent ClickHouse write workers (default: 0 = disabled).
        If > 0, enables async writes with connection pooling for 2-3x speedup.
        Issue #96 Phase 4: Async cache writes + connection pooling.
    max_days : int | None
        Maximum allowed date range span in days (default: 366).
        Raises ValueError if the range exceeds this limit.
        Pass ``None`` to bypass (used by kintsugi for authorized historical backfill).
        Issue #158 Phase 9: Prevents monolithic multi-year jobs.
    skip_missing_days : bool
        If True, skip days that return 404/no-data instead of failing.
        Used by Vision-first path where recent days may not yet be published.
    market : str | None
        Market type override. If None, resolved from symbol registry.

    Returns
    -------
    int
        Total number of bars written.

    Examples
    --------
    >>> bars = populate_cache_resumable("BTCUSDT", "2024-01-01", "2024-06-30")
    >>> print(f"Populated {bars} bars")
    14523

    >>> # If interrupted, just run again:
    >>> bars = populate_cache_resumable("BTCUSDT", "2024-01-01", "2024-06-30")
    >>> # Will resume from last checkpoint (bar-level accuracy)

    >>> # Start fresh (wipe cache and checkpoint):
    >>> bars = populate_cache_resumable(
    ...     "BTCUSDT", "2024-01-01", "2024-06-30",
    ...     force_refresh=True,
    ... )
    """
    from opendeviationbar.hooks import HookEvent, emit_hook
    from opendeviationbar.logging import generate_trace_id, log_checkpoint_event

    trace_id = generate_trace_id("pop")

    # Symbol registry gate + start date clamping (Issue #79)
    from opendeviationbar.symbol_registry import (
        validate_and_clamp_start_date,
        validate_symbol_registered,
    )

    validate_symbol_registered(symbol, operation="populate_cache_resumable")
    start_date = validate_and_clamp_start_date(symbol, start_date)

    # Resolve market from registry if not explicitly provided
    from opendeviationbar.symbol_registry import resolve_market

    market_resolved = resolve_market(symbol, market)

    # Issue #158 Phase 9: Hard guardrail against monolithic multi-year jobs.
    # Use kintsugi:catchup for historical backfill (recency-first ordering).
    if max_days is not None:  # SSoT-OK: intentional safety guard
        _parsed_start = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
        _parsed_end = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)
        span = (_parsed_end - _parsed_start).days
        if span > max_days:
            msg = (
                f"Date range {start_date} to {end_date} spans {span} days "
                f"(max_days={max_days}). Use kintsugi for historical backfill "
                f"(mise run kintsugi:catchup), or pass max_days=None to override."
            )
            raise ValueError(msg)

    # Threshold validation: enforce per-symbol minimums (Issue #115)
    from opendeviationbar.threshold import resolve_and_validate_threshold

    threshold_decimal_bps = resolve_and_validate_threshold(
        symbol,
        threshold_decimal_bps,
    )

    # Issue #126: Resolve ouroboros mode from config if not specified
    if ouroboros_mode is None:
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode

        ouroboros_mode = get_operational_ouroboros_mode()
    logger.info("populate_cache_resumable: ouroboros_mode=%s", ouroboros_mode)

    # T-1 guard: Binance Vision publishes with ~1-day lag.
    # Attempting to fetch today's data causes RuntimeError: "No data available"
    # which crashes the job after checkpoint is written but before ClickHouse
    # receives bars (gap regression scenario).
    today_utc = datetime.now(UTC).date()
    t1_cutoff = today_utc - timedelta(days=1)
    end_date_parsed = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC).date()
    if end_date_parsed >= today_utc:
        logger.warning(
            "end_date %s is today or future; clamping to T-1 (%s) "
            "(Binance Vision publishes with ~1-day lag)",
            end_date_parsed,
            t1_cutoff,
        )
        end_date = t1_cutoff.strftime("%Y-%m-%d")

    checkpoint_path = _get_checkpoint_path(
        symbol,
        threshold_decimal_bps,
        start_date,
        end_date,
        checkpoint_dir,
        ouroboros_mode=ouroboros_mode,  # Issue #126: mode in filename
    )

    # Issue #69: Handle force_refresh - wipe cache and checkpoint
    if force_refresh:
        logger.info("Force refresh: clearing cache and checkpoint for %s", symbol)
        # Delete local checkpoint (missing_ok=True prevents race with concurrent jobs)
        checkpoint_path.unlink(missing_ok=True)
        logger.debug("Deleted local checkpoint: %s", checkpoint_path)

        # Delete cached bars and ClickHouse checkpoint
        try:
            from opendeviationbar.clickhouse import OpenDeviationBarCache

            # Convert dates to timestamps for deletion
            from opendeviationbar.orchestration.helpers import (
                _datetime_to_end_ms,
                _datetime_to_start_ms,
            )

            start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
            end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)
            start_ts = _datetime_to_start_ms(start_dt)
            end_ts = _datetime_to_end_ms(end_dt)

            with OpenDeviationBarCache() as cache:
                cache.delete_bars(
                    symbol,
                    threshold_decimal_bps,
                    start_ts,
                    end_ts,
                    ouroboros_mode=ouroboros_mode,  # Issue #126
                )
                cache.delete_checkpoint(
                    symbol,
                    threshold_decimal_bps,
                    start_date,
                    end_date,
                    ouroboros_mode=ouroboros_mode,
                )
                logger.debug("Deleted cached bars and ClickHouse checkpoint")
        except (ImportError, ConnectionError) as e:
            logger.debug("ClickHouse cleanup skipped: %s", e)

        checkpoint = None
    else:
        # Check for existing checkpoint (local first, then ClickHouse)
        checkpoint = PopulationCheckpoint.load(checkpoint_path)

        # Issue #69: Try ClickHouse checkpoint for cross-machine resume
        if checkpoint is None:
            checkpoint = _load_checkpoint_from_clickhouse(
                symbol,
                threshold_decimal_bps,
                start_date,
                end_date,
                ouroboros_mode=ouroboros_mode,
            )

    resume_date = start_date
    total_bars = 0

    if checkpoint:
        # Validate checkpoint matches our parameters
        if (
            checkpoint.symbol == symbol
            and checkpoint.threshold_bps == threshold_decimal_bps
            and checkpoint.start_date == start_date
            and checkpoint.end_date == end_date
        ):
            # Issue #126: Verify ouroboros_mode consistency
            if (
                hasattr(checkpoint, "ouroboros_mode")
                and checkpoint.ouroboros_mode
                and checkpoint.ouroboros_mode != ouroboros_mode
            ):
                msg = (
                    f"Checkpoint ouroboros_mode mismatch: checkpoint has "
                    f"{checkpoint.ouroboros_mode!r} but current mode is "
                    f"{ouroboros_mode!r}. Delete the checkpoint or use matching mode."
                )
                raise ValueError(msg)

            # Resume from day after last completed
            last_completed = datetime.strptime(
                checkpoint.last_completed_date, "%Y-%m-%d"
            ).replace(tzinfo=UTC)
            resume_date = (last_completed + timedelta(days=1)).strftime("%Y-%m-%d")
            total_bars = checkpoint.bars_written

            logger.info(
                "Resuming %s population from %s (%d bars already written)",
                symbol,
                resume_date,
                total_bars,
            )

            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_SAVED,
                    symbol=symbol,
                    trace_id=trace_id,
                    action="resumed",
                    last_completed_date=checkpoint.last_completed_date,
                    bars_written=total_bars,
                )
        else:
            logger.warning("Checkpoint parameters don't match, starting fresh")
            checkpoint = None

    # Issue #96: Compute full date range once (was materialized 3 times)
    all_dates = list(_date_range(start_date, end_date))
    total_all_days = len(all_dates)

    # Process day by day
    dates = list(_date_range(resume_date, end_date))
    total_days = len(dates)

    # Issue #70: Progress tracking with ResumableObserver
    # Calculate initial count for resumed operations
    initial_days = total_all_days - total_days if checkpoint else 0

    # Initialize progress observer (lazy import to avoid circular deps)
    observer = None
    if verbose:
        from opendeviationbar.progress import ResumableObserver

        observer = ResumableObserver(
            total=total_all_days,
            initial=initial_days,
            desc=symbol,
            unit="days",
            verbose=True,
        )

    # Issue #97: Thread processor across days for bar-level continuity.
    import opendeviationbar as _odb_mod
    from opendeviationbar.exceptions import CacheWriteError
    from opendeviationbar.orchestration.helpers import _parse_microstructure_env_vars
    from opendeviationbar.orchestration.open_deviation_bars_cache import (
        fatal_cache_write,
    )
    from opendeviationbar.processors.core import OpenDeviationBarProcessor
    from opendeviationbar.storage.parquet import TickStorage

    source_version = f"{getattr(_odb_mod, '__version__', 'unknown')}+populate"

    eff_count, eff_bars, enable_intra = _parse_microstructure_env_vars(
        include_microstructure,
        None,
        inter_bar_lookback_bars,
    )
    active_processor: OpenDeviationBarProcessor | None = None

    # Restore processor from checkpoint if available
    if checkpoint and checkpoint.processor_checkpoint:
        try:
            active_processor = OpenDeviationBarProcessor.from_checkpoint(
                checkpoint.processor_checkpoint,
            )
            # Issue #128: Warn if feature config changed since checkpoint
            pop = _get_population_config()
            if (
                checkpoint.compute_tier2 != pop.compute_tier2
                or checkpoint.compute_tier3 != pop.compute_tier3
                or checkpoint.compute_hurst != pop.compute_hurst
                or checkpoint.compute_permutation_entropy
                != pop.compute_permutation_entropy
            ):
                logger.warning(
                    "Feature config changed since checkpoint "
                    "(ckpt: T2=%s T3=%s H=%s PE=%s → "
                    "now: T2=%s T3=%s H=%s PE=%s). "
                    "New bars use current config; "
                    "backfill old bars if needed.",
                    checkpoint.compute_tier2,
                    checkpoint.compute_tier3,
                    checkpoint.compute_hurst,
                    checkpoint.compute_permutation_entropy,
                    pop.compute_tier2,
                    pop.compute_tier3,
                    pop.compute_hurst,
                    pop.compute_permutation_entropy,
                )
            # Re-enable microstructure features after checkpoint restore
            if include_microstructure:
                active_processor.enable_microstructure(
                    inter_bar_lookback_count=eff_count,
                    inter_bar_lookback_bars=eff_bars,
                    include_intra_bar_features=enable_intra,
                    compute_tier2=pop.compute_tier2,
                    compute_tier3=pop.compute_tier3,
                    compute_hurst=pop.compute_hurst or None,
                    compute_permutation_entropy=pop.compute_permutation_entropy or None,
                )
            logger.info(
                "Restored processor from checkpoint "
                "(has_incomplete_bar=%s, defer_open=%s)",
                checkpoint.processor_checkpoint.get("has_incomplete_bar"),
                checkpoint.processor_checkpoint.get("defer_open"),
            )
            log_checkpoint_event(
                "checkpoint_restore",
                symbol,
                trace_id,
                threshold_dbps=threshold_decimal_bps,
                last_completed_date=checkpoint.last_completed_date,
                bars_written=checkpoint.bars_written,
                has_incomplete_bar=checkpoint.processor_checkpoint.get(
                    "has_incomplete_bar"
                ),
                last_trade_id=checkpoint.processor_checkpoint.get("last_trade_id"),
            )
            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_RESTORED,
                    symbol=symbol,
                    trace_id=trace_id,
                    threshold_dbps=threshold_decimal_bps,
                    has_incomplete_bar=checkpoint.processor_checkpoint.get(
                        "has_incomplete_bar"
                    ),
                    last_trade_id=checkpoint.processor_checkpoint.get("last_trade_id"),
                    defer_open=checkpoint.processor_checkpoint.get("defer_open"),
                )
        except Exception:
            logger.exception(
                "Failed to restore processor from checkpoint, starting fresh",
            )
            log_checkpoint_event(
                "checkpoint_restore_failed",
                symbol,
                trace_id,
                threshold_dbps=threshold_decimal_bps,
                fallback="fresh_processor",
            )
            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_RESTORE_FAILED,
                    symbol=symbol,
                    trace_id=trace_id,
                    threshold_dbps=threshold_decimal_bps,
                )
            active_processor = None

    created_at = checkpoint.created_at if checkpoint else datetime.now(UTC).isoformat()

    # Issue #96 Phase 4: Initialize thread pool for concurrent ClickHouse writes
    thread_pool = None
    if num_async_writers > 0:
        try:
            from concurrent.futures import ThreadPoolExecutor

            from opendeviationbar.clickhouse.connection_pool import get_connection_pool

            get_connection_pool(max_pool_size=num_async_writers * 2)
            logger.info(
                "Initialized ClickHouse connection pool: max=%d",
                num_async_writers * 2,
            )
            thread_pool = ThreadPoolExecutor(max_workers=num_async_writers)
            logger.info(
                "Concurrent cache writer initialized: %d workers",
                num_async_writers,
            )
        except (ImportError, ConnectionError, OSError) as e:
            logger.warning("Failed to initialize thread pool: %s", e)
            logger.info("Falling back to synchronous writes")
            thread_pool = None

    # Build loop-constant context
    ctx = PopulationContext(
        symbol=symbol,
        threshold_decimal_bps=threshold_decimal_bps,
        ouroboros_mode=ouroboros_mode,
        include_microstructure=include_microstructure,
        notify=notify,
        inter_bar_lookback_count=eff_count,
        inter_bar_lookback_bars=eff_bars,
        include_intra_bar_features=enable_intra,
        source_version=source_version,
        storage=TickStorage(),
        thread_pool=thread_pool,
        observer=observer,
        trace_id=trace_id,
        created_at=created_at,
        start_date=start_date,
        end_date=end_date,
        market=market_resolved,
    )

    for i, date in enumerate(dates, 1):
        logger.info(
            "Processing %s [%d/%d]: %s",
            symbol,
            i,
            total_days,
            date,
        )

        try:
            # Issue #97: Handle ouroboros boundary — reset processor state
            if active_processor is not None and _is_ouroboros_boundary(
                date, ouroboros_mode
            ):
                logger.info(
                    "Ouroboros %s boundary at %s — resetting processor",
                    ouroboros_mode,
                    date,
                )
                log_checkpoint_event(
                    "ouroboros_reset",
                    symbol,
                    trace_id,
                    date=date,
                    boundary_type=ouroboros_mode,
                    had_incomplete_bar=active_processor.get_incomplete_bar()
                    is not None,
                )
                orphan_bar = active_processor.reset_at_ouroboros()
                if orphan_bar is not None:
                    import polars as pl

                    orphan_bar["is_orphan"] = True
                    orphan_df = pl.DataFrame([orphan_bar])
                    written = _write_bars(orphan_df, ctx, fatal_cache_write)
                    total_bars += written

            # Process day: fetch ticks → stream bars → write to ClickHouse
            bars_today, active_processor = _process_day(
                date, ctx, active_processor, fatal_cache_write,
            )
            total_bars += bars_today

            logger.debug(
                "%s %s: %d bars (total: %d)",
                symbol,
                date,
                bars_today,
                total_bars,
            )

            # Issue #70: Update progress observer
            if observer is not None:
                observer.update(1)
                observer.set_postfix(bars=total_bars)
                observer.log_event(
                    "day_complete",
                    date=date,
                    bars_today=bars_today,
                    total_bars=total_bars,
                )

            # Issue #97: Save processor checkpoint for bar-level resumability
            proc_cp = None
            if active_processor is not None:
                proc_cp = active_processor.create_checkpoint(symbol)

            checkpoint = PopulationCheckpoint.from_processor_state(
                ctx, date, total_bars, proc_cp,
            )
            checkpoint.save(checkpoint_path)
            _save_checkpoint_to_clickhouse(checkpoint)

            log_checkpoint_event(
                "checkpoint_save",
                symbol,
                trace_id,
                threshold_dbps=threshold_decimal_bps,
                date=date,
                bars_today=bars_today,
                total_bars=total_bars,
                has_incomplete_bar=proc_cp is not None
                and proc_cp.get("has_incomplete_bar", False),
                defer_open=proc_cp.get("defer_open") if proc_cp else None,
            )

            if notify:
                emit_hook(
                    HookEvent.CHECKPOINT_SAVED,
                    symbol=symbol,
                    trace_id=trace_id,
                    date=date,
                    bars_today=bars_today,
                    total_bars=total_bars,
                    progress_pct=round(i / total_days * 100, 1),
                )

        except (ValueError, RuntimeError, OSError, CacheWriteError) as exc:
            # Defense-in-depth: if the final day has no data (archive not yet
            # published on Binance Vision), complete gracefully instead of
            # crashing the entire multi-day job at 99.96% completion.
            exc_str = str(exc)
            is_no_data = "No data available" in exc_str or "HTTP 404" in exc_str

            if is_no_data and skip_missing_days:
                logger.warning(
                    "No data for %s (skipping, skip_missing_days=True) "
                    "— continuing with %d bars so far",
                    date,
                    total_bars,
                )
                continue

            if is_no_data and date == dates[-1]:
                logger.warning(
                    "No data for final day %s (archive not yet published?) "
                    "— completing gracefully with %d bars",
                    date,
                    total_bars,
                )
                break

            logger.exception(
                "Failed to process %s for %s",
                symbol,
                date,
            )
            if notify:
                emit_hook(
                    HookEvent.POPULATION_FAILED,
                    symbol=symbol,
                    trace_id=trace_id,
                    date=date,
                    error=str(exc),
                    total_bars=total_bars,
                )
            raise

    # Flush final orphan — the last day's incomplete bar is still in the processor.
    # Without this, low-volatility days at high thresholds produce 0 completed bars
    # and the final incomplete bar is lost.
    if active_processor is not None:
        orphan_bar = active_processor.reset_at_ouroboros()
        if orphan_bar is not None:
            import polars as pl

            orphan_bar["is_orphan"] = True
            orphan_df = pl.DataFrame([orphan_bar])
            written = _write_bars(orphan_df, ctx, fatal_cache_write)
            total_bars += written

    # Issue #70: Close progress observer
    if observer is not None:
        observer.close()

    # Issue #77: Deduplicate after population completes
    # ReplacingMergeTree only deduplicates during background merges,
    # so we force immediate deduplication to ensure clean data.
    # Issue #90: Non-fatal — dedup failure should not crash the job.
    # INSERT dedup tokens (Issue #90) prevent most duplicates at source,
    # read queries use FINAL for on-the-fly dedup, and the pipeline-level
    # OPTIMIZE TABLE FINAL job catches any stragglers.
    try:
        from opendeviationbar.clickhouse import OpenDeviationBarCache

        with OpenDeviationBarCache() as cache:
            logger.info(
                "Running post-population deduplication for %s @ %d dbps",
                symbol,
                threshold_decimal_bps,
            )
            cache.deduplicate_bars(symbol, threshold_decimal_bps)
            logger.debug("Deduplication complete")
    except (ImportError, ConnectionError) as e:
        logger.debug("Post-population deduplication skipped: %s", e)
    except Exception as e:
        # Non-fatal: data is already written. Common cause is ClickHouse
        # OperationalError from OPTIMIZE TABLE timeout on large partitions.
        # Background merges or the pipeline-level OPTIMIZE job handle dedup.
        logger.warning(
            "Post-population deduplication did not complete for %s @ %d dbps: %s. "
            "Data is written. Run OPTIMIZE TABLE FINAL manually or via pipeline.",
            symbol,
            threshold_decimal_bps,
            e,
        )

    # Clean up checkpoint on success (missing_ok for concurrent jobs)
    checkpoint_path.unlink(missing_ok=True)
    logger.debug("Removed checkpoint file after successful completion")

    logger.info(
        "Population complete for %s: %d bars from %s to %s",
        symbol,
        total_bars,
        start_date,
        end_date,
    )

    if notify:
        emit_hook(
            HookEvent.POPULATION_COMPLETE,
            symbol=symbol,
            trace_id=trace_id,
            start_date=start_date,
            end_date=end_date,
            total_bars=total_bars,
        )

    # Issue #96 Phase 4: Wait for pending async writes to complete
    if thread_pool is not None:
        logger.info("Waiting for pending ClickHouse writes to complete...")
        thread_pool.shutdown(wait=True)
        logger.info("All writes completed, thread pool shut down")

    return total_bars


def list_checkpoints(
    checkpoint_dir: Path | None = None,
) -> list[PopulationCheckpoint]:
    """List all existing checkpoints.

    Parameters
    ----------
    checkpoint_dir : Path | None
        Custom checkpoint directory. Uses default if None.

    Returns
    -------
    list[PopulationCheckpoint]
        List of checkpoints found.
    """
    if checkpoint_dir is None:
        checkpoint_dir = _CHECKPOINT_DIR

    if not checkpoint_dir.exists():
        return []

    checkpoints = []
    for path in checkpoint_dir.glob("*.json"):
        checkpoint = PopulationCheckpoint.load(path)
        if checkpoint:
            checkpoints.append(checkpoint)

    return checkpoints


def clear_checkpoint(
    symbol: str,
    threshold_decimal_bps: int,
    start_date: str,
    end_date: str,
    checkpoint_dir: Path | None = None,
    ouroboros_mode: str = "day",
) -> bool:
    """Clear a specific checkpoint.

    Parameters
    ----------
    symbol : str
        Trading symbol.
    threshold_decimal_bps : int
        Threshold in decimal basis points.
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).
    checkpoint_dir : Path | None
        Custom checkpoint directory. Uses default if None.
    ouroboros_mode : str
        Ouroboros mode for checkpoint filename (Issue #126).

    Returns
    -------
    bool
        True if checkpoint was found and removed.
    """
    checkpoint_path = _get_checkpoint_path(
        symbol,
        threshold_decimal_bps,
        start_date,
        end_date,
        checkpoint_dir,
        ouroboros_mode=ouroboros_mode,
    )

    if checkpoint_path.exists():
        checkpoint_path.unlink(missing_ok=True)
        logger.info("Cleared checkpoint: %s", checkpoint_path)
        return True

    return False


__all__ = [
    "PopulationCheckpoint",
    "clear_checkpoint",
    "list_checkpoints",
    "populate_cache_resumable",
]
