"""Checkpoint processing engine: day processing, bar writing, multi-threshold fan-out."""

from __future__ import annotations

import gc
import logging
from collections.abc import Iterator
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any

from .models import PopulationContext

if TYPE_CHECKING:
    from opendeviationbar.config.backfill import BackfillConfig
    from opendeviationbar.processors.core import OpenDeviationBarProcessor


logger = logging.getLogger(__name__)


def _is_ouroboros_boundary(date_str: str, ouroboros_mode: str) -> bool:  # noqa: ARG001
    """Check if a date falls on an ouroboros reset boundary.

    Issue #97: Used by populate_cache_resumable() to reset the threaded
    processor at day boundaries. Both "day" and "aion" modes are supported.
    """
    if ouroboros_mode not in ("day", "aion"):
        msg = f"Invalid ouroboros mode: {ouroboros_mode!r}. Only 'day' and 'aion' modes are supported."
        raise ValueError(msg)
    return True  # Every day is a boundary in day mode


def _date_range(start_date: str, end_date: str) -> Iterator[str]:
    """Generate dates from start to end (inclusive).

    Parameters
    ----------
    start_date : str
        Start date (YYYY-MM-DD).
    end_date : str
        End date (YYYY-MM-DD).

    Yields
    ------
    str
        Dates in YYYY-MM-DD format.
    """
    from datetime import timedelta

    start = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
    end = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)

    current = start
    while current <= end:
        yield current.strftime("%Y-%m-%d")
        current += timedelta(days=1)


def _write_bars(
    bars_df: Any,  # noqa: ANN401
    ctx: PopulationContext,
    fatal_cache_write: Any,  # noqa: ANN401
) -> int:
    """Write bars to ClickHouse (thread pool or synchronous).

    Parameters
    ----------
    bars_df
        DataFrame of bars (Pandas or Polars).
    ctx : PopulationContext
        Population context with thread pool and metadata.
    fatal_cache_write
        Callable that writes bars to ClickHouse (raises on failure).

    Returns
    -------
    int
        Number of bars written.
    """
    if bars_df is None:
        return 0
    # Handle both Pandas (.empty) and Polars (.is_empty())
    if hasattr(bars_df, "empty") and bars_df.empty:
        return 0
    if hasattr(bars_df, "is_empty") and bars_df.is_empty():
        return 0

    if ctx.thread_pool is not None:
        ctx.thread_pool.submit(
            fatal_cache_write,
            bars_df,
            ctx.symbol,
            ctx.threshold_decimal_bps,
            ctx.ouroboros_mode,
            ctx.source_version,
        )
        return len(bars_df)

    return fatal_cache_write(
        bars_df,
        ctx.symbol,
        ctx.threshold_decimal_bps,
        ctx.ouroboros_mode,
        version=ctx.source_version,
    )


def _process_day(
    date: str,
    ctx: PopulationContext,
    active_processor: OpenDeviationBarProcessor | None,
    fatal_cache_write: Any,  # noqa: ANN401
) -> tuple[int, OpenDeviationBarProcessor | None]:
    """Process a single day: fetch ticks -> stream bars -> write to ClickHouse.

    Uses the streaming Arrow path -- peak memory ~150MB per day vs ~2GB
    with the deprecated _fetch_binance() batch path.

    Parameters
    ----------
    date : str
        Date to process (YYYY-MM-DD).
    ctx : PopulationContext
        Loop-constant population state.
    active_processor : OpenDeviationBarProcessor | None
        Processor with state for cross-day continuity.
    fatal_cache_write
        Callable that writes bars to ClickHouse (raises on failure).

    Returns
    -------
    tuple[int, OpenDeviationBarProcessor | None]
        (bars_written, updated_processor).
    """
    import polars as pl

    from opendeviationbar.orchestration.helpers import (
        _create_processor,
        _datetime_to_end_ms,
        _datetime_to_start_ms,
        _stream_bars_from_trades,
        _stream_open_deviation_bars_binance,
    )

    day_dt = datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=UTC)
    day_start_ms = _datetime_to_start_ms(day_dt)
    day_end_ms = _datetime_to_end_ms(day_dt)
    cache_symbol = f"BINANCE_SPOT_{ctx.symbol}".upper()

    # Ensure we have a processor (creates one if first day)
    if active_processor is None:
        from opendeviationbar.orchestration.helpers import (
            _parse_microstructure_env_vars,
        )

        eff_count, eff_bars, enable_intra = _parse_microstructure_env_vars(
            ctx.include_microstructure,
            ctx.inter_bar_lookback_count,
            ctx.inter_bar_lookback_bars,
        )
        active_processor = _create_processor(
            ctx.threshold_decimal_bps,
            ctx.include_microstructure,
            symbol=ctx.symbol,
            inter_bar_lookback_count=eff_count,
            include_intra_bar_features=enable_intra,
            inter_bar_lookback_bars=eff_bars,
        )

    def _backfill_cfg() -> BackfillConfig:
        from opendeviationbar.config.backfill import BackfillConfig
        return BackfillConfig()

    flush_threshold = _backfill_cfg().flush_threshold
    bar_frames: list[pl.DataFrame] = []
    accumulated_count = 0
    total_bars_written = 0
    flush_count = 0

    def _flush_accumulated() -> int:
        """Concat accumulated frames, enrich if needed, write to CH."""
        nonlocal bar_frames, accumulated_count, flush_count
        if not bar_frames:
            return 0
        batch_df = pl.concat(bar_frames, rechunk=False)
        bar_frames = []
        accumulated_count = 0
        flush_count += 1

        # Plugin enrichment: skip Polars->Pandas roundtrip when no plugins (MEM-04)
        if ctx.include_microstructure and not batch_df.is_empty():
            from opendeviationbar.plugins.loader import discover_providers, enrich_bars

            if discover_providers():  # Non-empty list = plugins installed
                bars_pd = batch_df.to_pandas()
                batch_df = enrich_bars(bars_pd, ctx.symbol, ctx.threshold_decimal_bps)

        written = _write_bars(batch_df, ctx, fatal_cache_write)
        del batch_df
        return written

    has_cached = ctx.storage.has_ticks(cache_symbol, day_start_ms, day_end_ms)
    if has_cached:
        # Cached path: stream from Parquet via _stream_bars_from_trades
        from opendeviationbar.orchestration.helpers import _select_trade_columns

        day_ticks = ctx.storage.read_ticks(cache_symbol, day_start_ms, day_end_ms)
        if day_ticks.is_empty():
            return 0, active_processor
        day_ticks = _select_trade_columns(day_ticks)
        for bar_batch in _stream_bars_from_trades(
            day_ticks,
            ctx.threshold_decimal_bps,
            ctx.include_microstructure,
            active_processor,
        ):
            bar_frames.append(bar_batch)
            accumulated_count += len(bar_batch)
            if accumulated_count >= flush_threshold:
                total_bars_written += _flush_accumulated()
        del day_ticks
    else:
        # Network path: stream from Binance API (Arrow, zero-copy)
        for bar_batch in _stream_open_deviation_bars_binance(
            ctx.symbol,
            date,
            date,
            ctx.threshold_decimal_bps,
            ctx.market,
            include_microstructure=ctx.include_microstructure,
            processor=active_processor,
            inter_bar_lookback_count=ctx.inter_bar_lookback_count,
        ):
            bar_frames.append(bar_batch)
            accumulated_count += len(bar_batch)
            if accumulated_count >= flush_threshold:
                total_bars_written += _flush_accumulated()

    # Final flush for remaining bars
    total_bars_written += _flush_accumulated()

    if flush_count > 1:
        logger.debug(
            "Bounded flush: %d flushes for %s on %s (%d bars total)",
            flush_count,
            ctx.symbol,
            date,
            total_bars_written,
        )

    gc.collect()

    return total_bars_written, active_processor


def _process_one_threshold_day(
    ticks_df: Any,  # noqa: ANN401 -- polars.DataFrame
    ctx: PopulationContext,
    active_processor: OpenDeviationBarProcessor | None,
    fatal_cache_write: Any,  # noqa: ANN401
) -> tuple[int, OpenDeviationBarProcessor | None]:
    """Process one threshold for one day using pre-loaded tick data.

    Accepts already-loaded ticks (as a Polars DataFrame) instead of reading
    from Parquet or the network.  Used by the multi-threshold fan-out path so
    the Parquet file is read only once and shared across thresholds.

    Parameters
    ----------
    ticks_df
        Polars DataFrame of tick data for a single day, already column-selected
        (output of ``_select_trade_columns``).  May be ``None`` or empty.
    ctx : PopulationContext
        Loop-constant state for *this threshold* (threshold_decimal_bps etc.).
    active_processor : OpenDeviationBarProcessor | None
        Processor carrying Rust state for this threshold.
    fatal_cache_write
        Callable that writes bars to ClickHouse (raises on failure).

    Returns
    -------
    tuple[int, OpenDeviationBarProcessor | None]
        (bars_written, updated_processor)
    """
    import polars as pl

    from opendeviationbar.orchestration.helpers import (
        _create_processor,
        _parse_microstructure_env_vars,
        _stream_bars_from_trades,
    )

    # Empty ticks -> nothing to do
    if ticks_df is None or (hasattr(ticks_df, "is_empty") and ticks_df.is_empty()):
        return 0, active_processor

    # Ensure we have a processor for this threshold
    if active_processor is None:
        eff_count, eff_bars, enable_intra = _parse_microstructure_env_vars(
            ctx.include_microstructure,
            ctx.inter_bar_lookback_count,
            ctx.inter_bar_lookback_bars,
        )
        active_processor = _create_processor(
            ctx.threshold_decimal_bps,
            ctx.include_microstructure,
            symbol=ctx.symbol,
            inter_bar_lookback_count=eff_count,
            include_intra_bar_features=enable_intra,
            inter_bar_lookback_bars=eff_bars,
        )

    def _backfill_cfg() -> BackfillConfig:
        from opendeviationbar.config.backfill import BackfillConfig
        return BackfillConfig()

    flush_threshold = _backfill_cfg().flush_threshold
    bar_frames: list[pl.DataFrame] = []
    accumulated_count = 0
    total_bars_written = 0
    flush_count = 0

    def _flush_accumulated() -> int:
        nonlocal bar_frames, accumulated_count, flush_count
        if not bar_frames:
            return 0
        batch_df = pl.concat(bar_frames, rechunk=False)
        bar_frames = []
        accumulated_count = 0
        flush_count += 1
        # Plugin enrichment: skip Polars->Pandas roundtrip when no plugins (MEM-04)
        if ctx.include_microstructure and not batch_df.is_empty():
            from opendeviationbar.plugins.loader import discover_providers, enrich_bars
            if discover_providers():  # Non-empty list = plugins installed
                bars_pd = batch_df.to_pandas()
                batch_df = enrich_bars(bars_pd, ctx.symbol, ctx.threshold_decimal_bps)
        written = _write_bars(batch_df, ctx, fatal_cache_write)
        del batch_df
        return written

    for bar_batch in _stream_bars_from_trades(
        ticks_df,
        ctx.threshold_decimal_bps,
        ctx.include_microstructure,
        active_processor,
    ):
        bar_frames.append(bar_batch)
        accumulated_count += len(bar_batch)
        if accumulated_count >= flush_threshold:
            total_bars_written += _flush_accumulated()

    total_bars_written += _flush_accumulated()

    if flush_count > 1:
        logger.debug(
            "Bounded flush: %d flushes for %s@%d on day (%d bars total)",
            flush_count,
            ctx.symbol,
            ctx.threshold_decimal_bps,
            total_bars_written,
        )

    gc.collect()
    return total_bars_written, active_processor


def _process_day_multi_threshold(
    date: str,
    ctxs: dict[int, PopulationContext],
    processors: dict[int, OpenDeviationBarProcessor | None],
    fatal_cache_write: Any,  # noqa: ANN401
    *,
    n_workers: int | None = None,
) -> dict[int, tuple[int, OpenDeviationBarProcessor | None]]:
    """Read ticks ONCE for *date*, fan out to multiple threshold processors.

    Performs a single Parquet / network read for the given date, then fans out
    to one ``_process_one_threshold_day()`` call per threshold using a
    ``ThreadPoolExecutor``.  Because ``process_trades_arrow()`` releases the
    GIL, the threads run in true parallel on multi-core hosts.

    Parameters
    ----------
    date : str
        Date to process (YYYY-MM-DD).
    ctxs : dict[int, PopulationContext]
        Map of threshold_decimal_bps -> context.  All contexts must share the
        same symbol/market/storage (i.e. same underlying tick data).
    processors : dict[int, OpenDeviationBarProcessor | None]
        Map of threshold_decimal_bps -> active processor (may be None).
    fatal_cache_write
        Callable forwarded to each per-threshold write.

    Returns
    -------
    dict[int, tuple[int, OpenDeviationBarProcessor | None]]
        Map of threshold -> (bars_written, updated_processor).
    """
    from concurrent.futures import ThreadPoolExecutor, as_completed

    from opendeviationbar.orchestration.helpers import (
        _datetime_to_end_ms,
        _datetime_to_start_ms,
        _select_trade_columns,
    )

    # Pick any context to resolve shared params (symbol / storage / market are
    # identical across all thresholds).
    any_ctx = next(iter(ctxs.values()))
    symbol = any_ctx.symbol
    market = any_ctx.market

    day_dt = datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=UTC)
    day_start_ms = _datetime_to_start_ms(day_dt)
    day_end_ms = _datetime_to_end_ms(day_dt)
    cache_symbol = f"BINANCE_SPOT_{symbol}".upper()

    # --- Single I/O operation ------------------------------------------------
    has_cached = any_ctx.storage.has_ticks(cache_symbol, day_start_ms, day_end_ms)
    if has_cached:
        day_ticks = any_ctx.storage.read_ticks(cache_symbol, day_start_ms, day_end_ms)
        if day_ticks.is_empty():
            return {t: (0, processors.get(t)) for t in ctxs}
        day_ticks_selected = _select_trade_columns(day_ticks)
        del day_ticks
    else:
        # Network path: stream raw ticks once via Vision ZIP, write to the
        # Parquet cache, then fall through to the shared fan-out below.
        # Previously this called _stream_open_deviation_bars_binance() which
        # never wrote to TickStorage, so has_ticks() always returned False and
        # the code fell into a per-threshold loop -- N redundant Vision downloads
        # per day.  This fix downloads once and writes to Parquet so that (a)
        # all thresholds share one download, and (b) the seeder / future runs
        # get a cache hit and skip the Vision download entirely.
        import polars as pl

        try:
            from opendeviationbar._core import stream_binance_trades_arrow
            from opendeviationbar.orchestration.helpers import _market_to_enum
        except ImportError:
            logger.warning(
                "Multi-threshold: stream_binance_trades_arrow unavailable for %s %s; "
                "falling back to per-threshold _process_day()",
                symbol,
                date,
            )
            results: dict[int, tuple[int, OpenDeviationBarProcessor | None]] = {}
            for threshold, ctx in ctxs.items():
                results[threshold] = _process_day(
                    date, ctx, processors.get(threshold), fatal_cache_write
                )
            return results

        market_enum = _market_to_enum(market)
        chunks: list[pl.DataFrame] = []
        for trade_arrow in stream_binance_trades_arrow(
            symbol, date, date, chunk_hours=6, market_type=market_enum,
            verify_checksum=True,
        ):
            chunks.append(pl.from_arrow(trade_arrow))

        if not chunks:
            return {t: (0, processors.get(t)) for t in ctxs}

        day_ticks_raw = pl.concat(chunks)
        del chunks
        # Persist to Parquet: future runs for any threshold on this date will
        # hit the cache and skip the Vision download.
        any_ctx.storage.write_ticks(cache_symbol, day_ticks_raw)
        day_ticks_selected = _select_trade_columns(day_ticks_raw)
        del day_ticks_raw

    # Polars .clone() is O(1) -- Arrow ref-count bump, no data copy.
    ticks_per_threshold = {t: day_ticks_selected.clone() for t in ctxs}
    del day_ticks_selected

    # --- Parallel fan-out via threads ----------------------------------------
    results = {}
    symbol = any_ctx.symbol

    def _run_threshold(threshold: int) -> tuple[int, tuple[int, OpenDeviationBarProcessor | None]]:
        ctx = ctxs[threshold]
        proc = processors.get(threshold)
        bars_written, updated_proc = _process_one_threshold_day(
            ticks_per_threshold[threshold],
            ctx,
            proc,
            fatal_cache_write,
        )
        return threshold, (bars_written, updated_proc)

    _n = n_workers if n_workers is not None else min(len(ctxs), 4)
    with ThreadPoolExecutor(max_workers=_n) as pool:
        futures = {pool.submit(_run_threshold, t): t for t in ctxs}
        for fut in as_completed(futures):
            threshold = futures[fut]
            try:
                _, result = fut.result()
                results[threshold] = result
            except Exception:
                logger.exception(
                    "Multi-threshold fan-out: threshold %d failed for %s %s; "
                    "skipping -- checkpoint will not advance for this threshold-day",
                    threshold,
                    symbol,
                    date,
                )
                # Return (0, unchanged processor) so this threshold-day is
                # retried on the next run while other thresholds continue.
                results[threshold] = (0, processors.get(threshold))

    gc.collect()
    return results
