# polars-exception: backtesting.py requires Pandas DataFrames with DatetimeIndex
# Issue #46: Modularization M4 - Extract precompute_open_deviation_bars from __init__.py
"""Batch precomputation pipeline for open deviation bars.

Provides precompute_open_deviation_bars() for ML workflows requiring continuous
bar sequences with cache invalidation and continuity validation.
"""

from __future__ import annotations

from collections.abc import Callable

import pandas as pd

from opendeviationbar.conversion import _concat_pandas_via_polars
from opendeviationbar.processors.core import OpenDeviationBarProcessor
from opendeviationbar.validation.continuity import ContinuityError, validate_continuity

from .helpers import (
    MARKET_TYPE_MAP,
    _datetime_to_end_ms,
    _datetime_to_start_ms,
    _fetch_binance,
    _fetch_exness,
    _parse_and_validate_dates,
)
from .models import PrecomputeProgress, PrecomputeResult


def _emit_progress(
    progress_callback: Callable[[PrecomputeProgress], None] | None,
    phase: str,
    period_str: str,
    periods_completed: int,
    periods_total: int,
    all_bars: list[pd.DataFrame],
    ticks_processed: int,
    start_time: float,
) -> None:
    """Emit PrecomputeProgress if callback is provided."""
    if progress_callback:
        import time

        progress_callback(
            PrecomputeProgress(
                phase=phase,
                current_period=period_str,
                periods_completed=periods_completed,
                periods_total=periods_total,
                bars_generated=sum(len(b) for b in all_bars),
                ticks_processed=ticks_processed,
                elapsed_seconds=time.time() - start_time,
            )
        )


def precompute_open_deviation_bars(
    symbol: str,
    start_date: str,
    end_date: str,
    threshold_decimal_bps: int = 250,
    *,
    source: str = "binance",
    market: str | None = None,
    chunk_size: int = 100_000,
    invalidate_existing: str = "smart",
    progress_callback: Callable[[PrecomputeProgress], None] | None = None,
    include_microstructure: bool = False,
    validate_on_complete: str = "error",
    continuity_tolerance_pct: float = 0.001,
    cache_dir: str | None = None,
    max_memory_gb: float | None = None,
    inter_bar_lookback_bars: int | None = None,
    ouroboros_mode: str | None = None,
) -> PrecomputeResult:
    """Precompute continuous open deviation bars for a date range (single-pass, guaranteed continuity).

    Designed for ML workflows requiring continuous bar sequences for training/validation.
    Uses Checkpoint API for memory-efficient chunked processing with state preservation.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT")
    start_date : str
        Start date (inclusive) "YYYY-MM-DD"
    end_date : str
        End date (inclusive) "YYYY-MM-DD"
    threshold_decimal_bps : int, default=250
        Open deviation bar threshold (250 = 0.25%)
    source : str, default="binance"
        Data source ("binance" or "exness")
    market : str, default="spot"
        Market type for Binance ("spot", "futures-um", "futures-cm")
    chunk_size : int, default=100_000
        Ticks per processing chunk (~15MB memory per 100K ticks)
    invalidate_existing : str, default="smart"
        Cache invalidation strategy:
        - "overlap": Invalidate only bars in date range
        - "full": Invalidate ALL bars for symbol/threshold
        - "none": Skip if any cached bars exist in range
        - "smart": Invalidate overlapping + validate junction continuity
    progress_callback : Callable, optional
        Optional callback for progress updates
    include_microstructure : bool, default=False
        Include order flow metrics (buy_volume, sell_volume, vwap)
    validate_on_complete : str, default="error"
        Continuity validation mode after precomputation:
        - "error": Raise ContinuityError if discontinuities found
        - "warn": Log warning but continue (sets continuity_valid=False)
        - "skip": Skip validation entirely (continuity_valid=None)
    continuity_tolerance_pct : float, default=0.001
        Maximum allowed price gap percentage for continuity validation.
        Default 0.1% (0.001) accommodates market microstructure events.
        The total allowed gap is threshold_pct + continuity_tolerance_pct.
    cache_dir : str or None, default=None
        Custom cache directory for tick data
    max_memory_gb : float or None, default=None
        Process-level memory cap in GB. Sets RLIMIT_AS so that
        exceeding the limit raises MemoryError instead of OOM kill.
        None disables the cap.

    Returns
    -------
    PrecomputeResult
        Result with statistics and cache key

    Raises
    ------
    ValueError
        Invalid parameters
    RuntimeError
        Fetch or processing failure
    ContinuityError
        If validate_on_complete=True and discontinuities found

    Examples
    --------
    Basic precomputation:

    >>> result = precompute_open_deviation_bars("BTCUSDT", "2024-01-01", "2024-06-30")
    >>> print(f"Generated {result.total_bars} bars")

    With progress callback:

    >>> def on_progress(p):
    ...     print(f"[{p.current_period}] {p.bars_generated} bars")
    >>> result = precompute_open_deviation_bars(
    ...     "BTCUSDT", "2024-01-01", "2024-03-31",
    ...     progress_callback=on_progress
    ... )
    """
    import gc
    import time
    from datetime import datetime
    from pathlib import Path

    from opendeviationbar.clickhouse import CacheKey, OpenDeviationBarCache
    from opendeviationbar.storage.parquet import TickStorage

    # Resolve ouroboros_mode from config if not explicitly provided
    if ouroboros_mode is None:
        from opendeviationbar.ouroboros import get_operational_ouroboros_mode

        ouroboros_mode = get_operational_ouroboros_mode()

    # MEM-009: Set process-level memory cap if requested (Issue #49)
    if max_memory_gb is not None:
        from opendeviationbar.resource_guard import set_memory_limit

        set_memory_limit(max_gb=max_memory_gb)

    start_time = time.time()

    # Validate parameters
    if invalidate_existing not in ("overlap", "full", "none", "smart"):
        msg = f"Invalid invalidate_existing: {invalidate_existing!r}. Must be 'overlap', 'full', 'none', or 'smart'"
        raise ValueError(msg)

    if validate_on_complete not in ("error", "warn", "skip"):
        msg = f"Invalid validate_on_complete: {validate_on_complete!r}. Must be 'error', 'warn', or 'skip'"
        raise ValueError(msg)

    # Symbol registry gate + start date clamping (Issue #79)
    from opendeviationbar.symbol_registry import (
        validate_and_clamp_start_date,
        validate_symbol_registered,
    )

    validate_symbol_registered(symbol, operation="precompute_open_deviation_bars")
    start_date = validate_and_clamp_start_date(symbol, start_date)

    # Validate threshold with asset-class minimum (Issue #62)
    from opendeviationbar.threshold import resolve_and_validate_threshold

    threshold_decimal_bps = resolve_and_validate_threshold(
        symbol, threshold_decimal_bps
    )

    # Parse dates
    start_dt, end_dt = _parse_and_validate_dates(start_date, end_date)

    # Resolve and normalize market type
    from opendeviationbar.symbol_registry import resolve_market

    market = resolve_market(symbol, market)
    market_normalized = MARKET_TYPE_MAP.get(market.lower(), market.lower())

    # Initialize storage and cache
    storage = TickStorage(cache_dir=Path(cache_dir) if cache_dir else None)
    cache = OpenDeviationBarCache()

    # Generate list of periods to process
    months: list[tuple[int, int]] = []
    current = start_dt.replace(day=1)
    _december = 12
    while current <= end_dt:
        months.append((current.year, current.month))
        # Move to next month
        if current.month == _december:
            current = current.replace(year=current.year + 1, month=1)
        else:
            current = current.replace(month=current.month + 1)

    # Handle cache invalidation
    start_ts = _datetime_to_start_ms(start_dt)
    end_ts = _datetime_to_end_ms(end_dt)
    cache_key = CacheKey(
        symbol=symbol,
        threshold_decimal_bps=threshold_decimal_bps,
        start_ts=start_ts,
        end_ts=end_ts,
        ouroboros_mode=ouroboros_mode,
    )

    if invalidate_existing == "full":
        cache.invalidate_open_deviation_bars(cache_key)
    elif invalidate_existing in ("overlap", "smart"):
        # Check for overlapping bars - will be handled after processing
        # For now, just invalidate the date range to ensure clean slate
        cache.invalidate_open_deviation_bars(cache_key)
    elif invalidate_existing == "none":
        # Check if any bars exist in range by counting
        bar_count = cache.count_bars(
            symbol, threshold_decimal_bps, ouroboros_mode=ouroboros_mode
        )
        if bar_count > 0:
            # Return early - some cached data exists
            # Note: This is approximate; full implementation would check time range
            return PrecomputeResult(
                symbol=symbol,
                threshold_decimal_bps=threshold_decimal_bps,
                start_date=start_date,
                end_date=end_date,
                total_bars=bar_count,
                total_ticks=0,
                elapsed_seconds=time.time() - start_time,
                continuity_valid=True,  # Assume valid for cached data
                cache_key=f"{symbol}_{threshold_decimal_bps}",
            )

    # Initialize processor (single instance for continuity)
    # Issue #81: Bar-relative lookback mode
    # Issue #128: Wire per-feature computation toggles from Settings
    from opendeviationbar.config import Settings

    pop = Settings.get().population
    processor = OpenDeviationBarProcessor(
        threshold_decimal_bps,
        symbol=symbol,
        inter_bar_lookback_bars=inter_bar_lookback_bars,
        include_intra_bar_features=include_microstructure,
        compute_tier2=pop.compute_tier2,
        compute_tier3=pop.compute_tier3,
        compute_hurst=pop.compute_hurst or None,
        compute_permutation_entropy=pop.compute_permutation_entropy or None,
    )

    all_bars: list[pd.DataFrame] = []
    period_bars: list[
        pd.DataFrame
    ] = []  # Issue #27: Track per-period bars for incremental caching
    total_ticks = 0
    cache_symbol = f"{source}_{market_normalized}_{symbol}".upper()

    for i, (year, month) in enumerate(months):
        period_str = f"{year}-{month:02d}"

        # Report progress - fetching
        _emit_progress(
            progress_callback,
            "fetching",
            period_str,
            i,
            len(months),
            all_bars,
            total_ticks,
            start_time,
        )

        # Calculate period boundaries
        period_start = datetime(year, month, 1)
        _december = 12
        if month == _december:
            period_end = datetime(year + 1, 1, 1)
        else:
            period_end = datetime(year, month + 1, 1)

        # Adjust to fit within requested date range
        actual_start = max(period_start, start_dt)
        actual_end = min(period_end, end_dt + pd.Timedelta(days=1))

        # Fetch tick data for this period
        start_ts_period = int(actual_start.timestamp() * 1000)
        end_ts_period = int(actual_end.timestamp() * 1000)

        # Check if data is already cached
        data_cached = storage.has_ticks(cache_symbol, start_ts_period, end_ts_period)

        if data_cached:
            # STREAMING READ: Use row-group based streaming to avoid OOM (Issue #12)
            # The Rust processor maintains state between process_trades() calls,
            # so we stream chunks directly without loading entire period into memory
            period_has_data = False
            for raw_tick_chunk in storage.read_ticks_streaming(
                cache_symbol, start_ts_period, end_ts_period, chunk_size=chunk_size
            ):
                period_has_data = True

                # Deduplicate trades by trade_id within chunk
                tick_chunk = raw_tick_chunk
                if "agg_trade_id" in tick_chunk.columns:
                    tick_chunk = tick_chunk.unique(
                        subset=["agg_trade_id"], maintain_order=True
                    )
                elif "trade_id" in tick_chunk.columns:
                    tick_chunk = tick_chunk.unique(
                        subset=["trade_id"], maintain_order=True
                    )

                # Sort by (timestamp, trade_id) - Rust crate requires this order
                if "timestamp" in tick_chunk.columns:
                    if "agg_trade_id" in tick_chunk.columns:
                        tick_chunk = tick_chunk.sort(["timestamp", "agg_trade_id"])
                    elif "trade_id" in tick_chunk.columns:
                        tick_chunk = tick_chunk.sort(["timestamp", "trade_id"])
                    else:
                        tick_chunk = tick_chunk.sort("timestamp")

                total_ticks += len(tick_chunk)

                # Report progress - processing
                _emit_progress(
                    progress_callback,
                    "processing",
                    period_str,
                    i,
                    len(months),
                    all_bars,
                    total_ticks,
                    start_time,
                )

                # Issue #88: Arrow-native input — eliminates .to_dicts() bottleneck
                import polars as pl

                from opendeviationbar.processors.api import _arrow_bars_to_pandas

                chunk_arrow = tick_chunk.to_arrow()
                bars_arrow = processor.process_trades_arrow(chunk_arrow)
                bars_pl = pl.from_arrow(bars_arrow)
                if not bars_pl.is_empty():
                    # Issue #30: Always include microstructure for ClickHouse cache
                    bars_df = _arrow_bars_to_pandas(
                        bars_pl, include_microstructure=True
                    )
                    period_bars.append(bars_df)  # Issue #27: Track per-period bars

                del tick_chunk

            gc.collect()

            if not period_has_data:
                continue
        else:
            # DATA NOT CACHED: Fetch from source day-by-day to prevent OOM
            # Issue #14: Fetching entire period at once causes OOM for high-volume
            # periods like March 2024. Fetch day-by-day instead.
            current_day = actual_start
            while current_day < actual_end:
                next_day = current_day + pd.Timedelta(days=1)
                next_day = min(next_day, actual_end)

                day_start_str = current_day.strftime("%Y-%m-%d")
                day_end_str = (next_day - pd.Timedelta(seconds=1)).strftime("%Y-%m-%d")

                if source == "binance":
                    tick_data = _fetch_binance(
                        symbol, day_start_str, day_end_str, market_normalized
                    )
                else:
                    tick_data = _fetch_exness(
                        symbol, day_start_str, day_end_str, "strict"
                    )

                if not tick_data.is_empty():
                    storage.write_ticks(cache_symbol, tick_data)

                    # Deduplicate trades by trade_id
                    if "agg_trade_id" in tick_data.columns:
                        tick_data = tick_data.unique(
                            subset=["agg_trade_id"], maintain_order=True
                        )
                    elif "trade_id" in tick_data.columns:
                        tick_data = tick_data.unique(
                            subset=["trade_id"], maintain_order=True
                        )

                    # Sort by (timestamp, trade_id) - Rust crate requires order
                    if "timestamp" in tick_data.columns:
                        if "agg_trade_id" in tick_data.columns:
                            tick_data = tick_data.sort(["timestamp", "agg_trade_id"])
                        elif "trade_id" in tick_data.columns:
                            tick_data = tick_data.sort(["timestamp", "trade_id"])
                        else:
                            tick_data = tick_data.sort("timestamp")

                    total_ticks += len(tick_data)

                    # Report progress - processing
                    _emit_progress(
                        progress_callback,
                        "processing",
                        period_str,
                        i,
                        len(months),
                        all_bars,
                        total_ticks,
                        start_time,
                    )

                    # Issue #88: Arrow-native input — eliminates .to_dicts() bottleneck
                    import polars as pl

                    from opendeviationbar.processors.api import _arrow_bars_to_pandas

                    tick_count = len(tick_data)
                    for chunk_start in range(0, tick_count, chunk_size):
                        chunk_end = min(chunk_start + chunk_size, tick_count)
                        chunk_df = tick_data.slice(chunk_start, chunk_end - chunk_start)
                        chunk_arrow = chunk_df.to_arrow()

                        # Stream to Rust processor via Arrow input
                        bars_arrow = processor.process_trades_arrow(chunk_arrow)
                        bars_pl = pl.from_arrow(bars_arrow)
                        if not bars_pl.is_empty():
                            # Issue #30: Always include microstructure for ClickHouse cache
                            bars_df = _arrow_bars_to_pandas(
                                bars_pl, include_microstructure=True
                            )
                            period_bars.append(bars_df)

                        del chunk_df

                    del tick_data
                    gc.collect()

                current_day = next_day

        # Issue #27: Incremental caching - store bars to ClickHouse after each period
        # This provides crash resilience and bounded memory for DB writes
        if period_bars:
            # MEM-006: Use Polars for memory-efficient concatenation
            period_df = _concat_pandas_via_polars(period_bars)

            # Report progress - caching this period
            _emit_progress(
                progress_callback,
                "caching",
                period_str,
                i + 1,
                len(months),
                [*all_bars, period_df],
                total_ticks,
                start_time,
            )

            # Cache immediately (idempotent via ReplacingMergeTree)
            # Phase 3+4: Route through store_bars_batch (Arrow path, 2-3 copies)
            # instead of store_bars_bulk (pandas path, 5-7 copies)
            rows_sent = len(period_df)
            period_pl = pl.from_pandas(period_df.reset_index())
            rows_inserted = cache.store_bars_batch(
                symbol,
                threshold_decimal_bps,
                period_pl,
                ouroboros_mode=ouroboros_mode,
            )

            # Post-cache validation: FAIL LOUDLY if ClickHouse didn't receive all bars
            if rows_inserted != rows_sent:
                msg = (
                    f"ClickHouse cache validation FAILED for {period_str}: "
                    f"sent {rows_sent} bars but only {rows_inserted} inserted. "
                    f"Data integrity compromised - aborting."
                )
                raise RuntimeError(msg)

            # Preserve for final validation and return
            all_bars.append(period_df)
            period_bars = []  # Clear to reclaim memory
            gc.collect()

    # Combine all bars (MEM-006: use Polars for memory efficiency)
    if all_bars:
        final_bars = _concat_pandas_via_polars(all_bars)
    else:
        final_bars = pd.DataFrame(columns=["Open", "High", "Low", "Close", "Volume"])

    # Note: Caching now happens incrementally after each period (Issue #27)
    # No final bulk store needed - all bars already cached per-period

    # Validate continuity (Issue #19: configurable tolerance and validation mode)
    continuity_valid: bool | None = None

    if validate_on_complete == "skip":
        # Skip validation entirely
        continuity_valid = None
    else:
        continuity_result = validate_continuity(
            final_bars,
            tolerance_pct=continuity_tolerance_pct,
            threshold_decimal_bps=threshold_decimal_bps,
        )
        continuity_valid = continuity_result["is_valid"]

        if not continuity_valid:
            if validate_on_complete == "error":
                msg = f"Found {continuity_result['discontinuity_count']} discontinuities in precomputed bars"
                raise ContinuityError(msg, continuity_result["discontinuities"])
            if validate_on_complete == "warn":
                import warnings

                msg = (
                    f"Found {continuity_result['discontinuity_count']} discontinuities "
                    f"in precomputed bars (tolerance: {continuity_tolerance_pct:.4%})"
                )
                warnings.warn(msg, stacklevel=2)

    return PrecomputeResult(
        symbol=symbol,
        threshold_decimal_bps=threshold_decimal_bps,
        start_date=start_date,
        end_date=end_date,
        total_bars=len(final_bars),
        total_ticks=total_ticks,
        elapsed_seconds=time.time() - start_time,
        continuity_valid=continuity_valid,
        cache_key=f"{symbol}_{threshold_decimal_bps}",
    )
