# FILE-SIZE-OK: Multi-method processor cohesion (M2 modularization hub)
# polars-exception: backtesting.py requires Pandas DataFrames with DatetimeIndex
# Issue #46: Modularization M2 - Extract
# OpenDeviationBarProcessor from __init__.py
"""Core processor for tick-to-bar conversion.

This module contains the OpenDeviationBarProcessor class,
which wraps the Rust-based PyOpenDeviationBarProcessor to
provide a Pythonic interface for open deviation bar
construction.
"""

from __future__ import annotations

from typing import TYPE_CHECKING

import pandas as pd

from opendeviationbar._core import PositionVerification
from opendeviationbar._core import (
    PyOpenDeviationBarProcessor as _PyOpenDeviationBarProcessor,
)

if TYPE_CHECKING:
    from arro3.core import RecordBatch as PyRecordBatch


class OpenDeviationBarProcessor:
    """Process tick-level trade data into open deviation bars.

    Open deviation bars close when price moves ±threshold from the bar's opening price,
    providing market-adaptive time intervals that eliminate arbitrary time-based
    artifacts.

    Parameters
    ----------
    threshold_decimal_bps : int
        Threshold in decimal basis points.
        Examples: 250 = 25bps = 0.25%, 100 = 10bps = 0.1%
        Valid range: [1, 100_000] (0.001% to 100%)
    symbol : str, optional
        Trading symbol (e.g., "BTCUSDT"). Required for checkpoint creation.
    prevent_same_timestamp_close : bool, default=True
        Timestamp gating for flash crash prevention (Issue #36).
        If True (default): A bar cannot close on the same timestamp it opened.
        This prevents flash crash scenarios from creating thousands of bars
        at identical timestamps. If False: Legacy v8 behavior where bars can
        close immediately on breach regardless of timestamp.
    inter_bar_lookback_count : int, optional
        Number of trades to keep in lookback buffer for inter-bar feature
        computation (Issue #59). If set, enables 16 inter-bar features
        computed from trades BEFORE each bar opens. Recommended: 100-500.
        If None (default), inter-bar features are disabled.
    inter_bar_lookback_bars : int, optional
        Bar-relative lookback (Issue #81). Number of completed bars whose
        trades to use as the lookback window. Self-adapts to bar size.
        Takes precedence over inter_bar_lookback_count.

    Raises
    ------
    ValueError
        If threshold_decimal_bps is out of valid range [1, 100_000]

    Examples
    --------
    Create processor and convert trades to DataFrame:

    >>> processor = OpenDeviationBarProcessor(threshold_decimal_bps=250)  # 0.25%
    >>> trades = [
    ...     {"timestamp": 1704067200000, "price": 42000.0, "quantity": 1.5},
    ...     {"timestamp": 1704067210000, "price": 42105.0, "quantity": 2.3},
    ... ]
    >>> bars = processor.process_trades(trades)
    >>> df = processor.to_dataframe(bars)
    >>> print(df.columns.tolist())
    ['Open', 'High', 'Low', 'Close', 'Volume']

    Cross-file continuity with checkpoints:

    >>> processor = OpenDeviationBarProcessor(250, symbol="BTCUSDT")
    >>> bars_file1 = processor.process_trades(file1_trades)
    >>> checkpoint = processor.create_checkpoint()
    >>> # Save checkpoint to JSON...
    >>> # Later, resume from checkpoint:
    >>> processor2 = OpenDeviationBarProcessor.from_checkpoint(checkpoint)
    >>> bars_file2 = processor2.process_trades(file2_trades)
    >>> # Incomplete bar from file1 continues correctly!

    Notes
    -----
    Non-lookahead bias guarantee:
    - Thresholds computed ONLY from bar open price (never recalculated)
    - Breaching trade INCLUDED in closing bar
    - Breaching trade also OPENS next bar

    Temporal integrity:
    - All trades processed in strict chronological order
    - Unsorted trades raise RuntimeError

    Cross-file continuity (v6.1.0+):
    - Incomplete bars are preserved across file boundaries via checkpoints
    - Thresholds are IMMUTABLE for bar's lifetime (computed from open)
    - Price hash verification detects gaps in data stream
    """

    def __init__(
        self,
        threshold_decimal_bps: int,
        symbol: str | None = None,
        *,
        prevent_same_timestamp_close: bool = True,
        inter_bar_lookback_count: int | None = None,
        include_intra_bar_features: bool = False,
        inter_bar_lookback_bars: int | None = None,
        # Issue #128: Per-feature computation toggles
        compute_tier2: bool = True,
        compute_tier3: bool = False,
        compute_hurst: bool | None = None,
        compute_permutation_entropy: bool | None = None,
    ) -> None:
        """Initialize processor with given threshold.

        Parameters
        ----------
        threshold_decimal_bps : int
            Threshold in decimal basis points (250 = 25bps = 0.25%)
        symbol : str, optional
            Trading symbol for checkpoint creation
        prevent_same_timestamp_close : bool, default=True
            Timestamp gating for flash crash prevention (Issue #36)
        inter_bar_lookback_count : int, optional
            Lookback trade count for inter-bar features (Issue #59).
            Enables 16 lookback_* features computed from trades BEFORE bar opens.
        include_intra_bar_features : bool, default=False
            Enable intra-bar features (Issue #59). When True, computes 22 intra_*
            features from trades WITHIN each bar, including ITH metrics.
        inter_bar_lookback_bars : int, optional
            Bar-relative lookback (Issue #81). Number of completed bars to use
            as lookback window. Takes precedence over inter_bar_lookback_count.

        Raises
        ------
        ThresholdError
            If threshold is below configured minimum for symbol's asset class
        ValueError
            If threshold is out of valid range [1, 100_000]
        """
        # Python-layer validation (Issue #62: crypto minimum threshold)
        from opendeviationbar.threshold import resolve_and_validate_threshold

        threshold_decimal_bps = resolve_and_validate_threshold(
            symbol, threshold_decimal_bps
        )

        # Rust-layer validation (additional range checks)
        self._processor = _PyOpenDeviationBarProcessor(
            threshold_decimal_bps,
            symbol,
            prevent_same_timestamp_close,
            inter_bar_lookback_count,
            include_intra_bar_features,
            inter_bar_lookback_bars,
            # Issue #128: Per-feature computation toggles
            compute_tier2=compute_tier2,
            compute_tier3=compute_tier3,
            compute_hurst=compute_hurst,
            compute_permutation_entropy=compute_permutation_entropy,
        )
        self.threshold_decimal_bps = threshold_decimal_bps
        self.symbol = symbol
        self.prevent_same_timestamp_close = prevent_same_timestamp_close
        self.inter_bar_lookback_count = inter_bar_lookback_count
        self.include_intra_bar_features = include_intra_bar_features
        self.inter_bar_lookback_bars = inter_bar_lookback_bars

    @classmethod
    def from_checkpoint(cls, checkpoint: dict) -> OpenDeviationBarProcessor:
        """Create processor from checkpoint for cross-file continuation.

        Restores processor state including any incomplete bar that was being
        built when the checkpoint was created. The incomplete bar will continue
        building from where it left off.

        Parameters
        ----------
        checkpoint : dict
            Checkpoint state from create_checkpoint()

        Returns
        -------
        OpenDeviationBarProcessor
            New processor with restored state

        Raises
        ------
        ThresholdError
            If checkpoint threshold is below configured minimum for symbol's asset class
        ValueError
            If checkpoint is invalid or corrupted

        Examples
        --------
        >>> import json
        >>> with open("checkpoint.json") as f:
        ...     checkpoint = json.load(f)
        >>> processor = OpenDeviationBarProcessor.from_checkpoint(checkpoint)
        >>> bars = processor.process_trades(next_file_trades)
        """
        # Python-layer validation (Issue #62: crypto minimum threshold)
        from opendeviationbar.threshold import validate_checkpoint_threshold

        validate_checkpoint_threshold(checkpoint)

        instance = cls.__new__(cls)
        instance._processor = _PyOpenDeviationBarProcessor.from_checkpoint(checkpoint)
        instance.threshold_decimal_bps = checkpoint["threshold_decimal_bps"]
        instance.symbol = checkpoint.get("symbol")
        # Default to True for old checkpoints without this field
        instance.prevent_same_timestamp_close = checkpoint.get(
            "prevent_same_timestamp_close", True
        )
        return instance

    def process_trades(
        self,
        trades: list[dict[str, int | float]],
        return_format: str = "dict",
    ) -> list[dict[str, str | float | int]] | PyRecordBatch:
        """Process trades into open deviation bars.

        Parameters
        ----------
        trades : List[Dict]
            List of trade dictionaries with keys:
            - timestamp: int (milliseconds since epoch)
            - price: float
            - quantity: float (or 'volume')

            Optional keys:
            - agg_trade_id: int
            - first_trade_id: int
            - last_trade_id: int
            - is_buyer_maker: bool
        return_format : str, default="dict"
            Output format: "dict" (list of dicts) or "arrow" (Arrow RecordBatch).
            Arrow format enables zero-copy conversion to Polars/pandas (recommended
            for performance-critical applications, Issue #96).

        Returns
        -------
        List[Dict] or PyRecordBatch
            If return_format="dict":
                List of open deviation bar dictionaries with keys:
                - timestamp: str (RFC3339 format)
                - open: float, high, low, close: float
                - volume: float, vwap: float, buy_volume, sell_volume: float
                - individual_trade_count, agg_record_count: int

            If return_format="arrow":
                Arrow RecordBatch with 30 columns (OHLCV + microstructure).
                Zero-copy compatible with Polars/PyArrow.

        Raises
        ------
        KeyError
            If required trade fields are missing
        RuntimeError
            If trades are not sorted chronologically
        ValueError
            If return_format is neither "dict" nor "arrow"

        Examples
        --------
        Dictionary output (default, backwards compatible):

        >>> processor = OpenDeviationBarProcessor(250)
        >>> trades = [
        ...     {"timestamp": 1704067200000, "price": 42000.0, "quantity": 1.0},
        ...     {"timestamp": 1704067210000, "price": 42105.0, "quantity": 2.0},
        ... ]
        >>> bars = processor.process_trades(trades)
        >>> len(bars)
        1
        >>> bars[0]["open"]
        42000.0

        Arrow output (3-5x faster, recommended):

        >>> import polars as pl
        >>> bars_arrow = processor.process_trades(trades, return_format="arrow")
        >>> df = pl.from_arrow(bars_arrow)  # Zero-copy!
        >>> print(df.shape)
        (1, 30)
        """
        if not trades:
            # Empty result in requested format
            if return_format == "arrow":
                # Return empty Arrow batch with correct schema
                try:
                    return self._processor.process_trades([], return_format="arrow")
                except ImportError as e:
                    msg = (
                        "Arrow output requires 'arro3' library. "
                        "Install with: pip install arro3"
                    )
                    raise ValueError(msg) from e
            return []

        return self._processor.process_trades(trades, return_format=return_format)

    def process_trades_streaming(
        self, trades: list[dict[str, int | float]]
    ) -> list[dict[str, str | float | int]]:
        """Process trades into open deviation bars (streaming mode - preserves state).

        Unlike `process_trades()`, this method maintains processor state across
        calls, enabling continuous processing across multiple batches (e.g.,
        period-by-period or chunk-by-chunk processing).

        Use this method for:
        - Multi-period precomputation (Issue #16)
        - Chunked processing of large datasets
        - Any scenario requiring bar continuity across batches

        Parameters
        ----------
        trades : List[Dict]
            List of trade dictionaries with keys:
            - timestamp: int (milliseconds since epoch)
            - price: float
            - quantity: float (or 'volume')

            Optional keys:
            - agg_trade_id: int
            - first_trade_id: int
            - last_trade_id: int
            - is_buyer_maker: bool

        Returns
        -------
        List[Dict]
            List of open deviation bar dictionaries (only completed bars).
            Same structure as process_trades().

        Notes
        -----
        State persistence: The processor remembers the incomplete bar from
        the previous call. When new trades arrive, they continue building
        that bar until threshold breach, ensuring continuity.

        Examples
        --------
        >>> processor = OpenDeviationBarProcessor(250)
        >>> # First batch (month 1)
        >>> bars1 = processor.process_trades_streaming(month1_trades)
        >>> # Second batch (month 2) - continues from month 1's state
        >>> bars2 = processor.process_trades_streaming(month2_trades)
        >>> # No discontinuity at month boundary
        """
        if not trades:
            return []

        return self._processor.process_trades_streaming(trades)

    def to_dataframe(
        self,
        bars: list[dict[str, str | float | int]],
        include_microstructure: bool = False,
    ) -> pd.DataFrame:
        """Convert open deviation bars to pandas DataFrame (backtesting.py compatible).

        Parameters
        ----------
        bars : List[Dict]
            List of open deviation bar dictionaries from process_trades()
        include_microstructure : bool, default=False
            If True, include all microstructure columns (vwap, buy_volume,
            sell_volume, ofi, kyle_lambda_proxy, etc.)

        Returns
        -------
        pd.DataFrame
            DataFrame with DatetimeIndex and OHLCV columns:
            - Index: timestamp (DatetimeIndex)
            - Columns: Open, High, Low, Close, Volume
            - (if include_microstructure) Additional microstructure columns

        Notes
        -----
        Output format is compatible with backtesting.py:
        - Column names are capitalized (Open, High, Low, Close, Volume)
        - Index is DatetimeIndex
        - No NaN values (all bars complete)
        - Sorted chronologically

        Examples
        --------
        >>> processor = OpenDeviationBarProcessor(250)
        >>> trades = [
        ...     {"timestamp": 1704067200000, "price": 42000.0, "quantity": 1.0},
        ...     {"timestamp": 1704067210000, "price": 42105.0, "quantity": 2.0},
        ... ]
        >>> bars = processor.process_trades(trades)
        >>> df = processor.to_dataframe(bars)
        >>> isinstance(df.index, pd.DatetimeIndex)
        True
        >>> list(df.columns)
        ['Open', 'High', 'Low', 'Close', 'Volume']
        """
        if not bars:
            return pd.DataFrame(
                columns=["Open", "High", "Low", "Close", "Volume"]
            ).set_index(pd.DatetimeIndex([]))

        result = pd.DataFrame(bars)

        # Convert close_time_us (Int64 μs) to DatetimeIndex
        result["timestamp"] = pd.to_datetime(
            result["close_time_us"],
            unit="us",
            utc=True,
        )
        result = result.set_index("timestamp")
        result = result.drop(columns=["close_time_us", "open_time_us"], errors="ignore")

        # Rename columns to backtesting.py format (capitalized)
        result = result.rename(
            columns={
                "open": "Open",
                "high": "High",
                "low": "Low",
                "close": "Close",
                "volume": "Volume",
            }
        )

        if include_microstructure:
            # Return all columns including microstructure
            return result

        # Return only OHLCV columns (drop microstructure fields for backtesting)
        return result[["Open", "High", "Low", "Close", "Volume"]]

    def create_checkpoint(self, symbol: str | None = None) -> dict:
        """Create checkpoint for cross-file continuation.

        Captures current processing state including incomplete bar (if any).
        The checkpoint can be serialized to JSON and used to resume processing
        across file boundaries while maintaining bar continuity.

        Parameters
        ----------
        symbol : str, optional
            Symbol being processed. If None, uses the symbol provided at
            construction time.

        Returns
        -------
        dict
            Checkpoint state (JSON-serializable) containing:
            - symbol: Trading symbol
            - threshold_decimal_bps: Threshold value
            - incomplete_bar: Incomplete bar state (if any)
            - thresholds: IMMUTABLE upper/lower thresholds for incomplete bar
            - last_timestamp_us: Last processed timestamp
            - last_trade_id: Last trade ID (for gap detection)
            - price_hash: Hash for position verification
            - anomaly_summary: Gap/overlap detection counters

        Raises
        ------
        ValueError
            If no symbol provided (neither at construction nor in this call)

        Examples
        --------
        >>> processor = OpenDeviationBarProcessor(250, symbol="BTCUSDT")
        >>> bars = processor.process_trades(trades)
        >>> checkpoint = processor.create_checkpoint()
        >>> # Save to JSON
        >>> import json
        >>> with open("checkpoint.json", "w") as f:
        ...     json.dump(checkpoint, f)
        """
        return self._processor.create_checkpoint(symbol)

    def verify_position(
        self, first_trade: dict[str, int | float]
    ) -> PositionVerification:
        """Verify position in data stream at file boundary.

        Checks if the first trade of the next file matches the expected
        position based on the processor's current state. Useful for
        detecting data gaps when resuming from checkpoint.

        Parameters
        ----------
        first_trade : dict
            First trade of the next file with keys: timestamp, price, quantity

        Returns
        -------
        PositionVerification
            Verification result:
            - is_exact: True if position matches exactly
            - has_gap: True if there's a gap (missing trades)
            - gap_details(): Returns (expected_id, actual_id, missing_count) if gap
            - timestamp_gap_ms(): Returns gap in ms for timestamp-only sources

        Examples
        --------
        >>> processor = OpenDeviationBarProcessor.from_checkpoint(checkpoint)
        >>> verification = processor.verify_position(next_file_trades[0])
        >>> if verification.has_gap:
        ...     expected, actual, missing = verification.gap_details()
        ...     print(f"Gap detected: {missing} trades missing")
        """
        return self._processor.verify_position(first_trade)

    def get_incomplete_bar(self) -> dict | None:
        """Get incomplete bar if any.

        Returns the bar currently being built (not yet breached threshold).
        Returns None if the last trade completed a bar cleanly.

        Returns
        -------
        dict or None
            Incomplete bar with OHLCV fields, or None
        """
        return self._processor.get_incomplete_bar()

    @property
    def has_incomplete_bar(self) -> bool:
        """Check if there's an incomplete bar."""
        return self._processor.has_incomplete_bar

    def process_trades_streaming_arrow(
        self, trades: list[dict[str, int | float]]
    ) -> PyRecordBatch:
        """Process trades into open deviation bars, returning Arrow RecordBatch.

        This is the most memory-efficient streaming API. Returns Arrow
        RecordBatch for zero-copy transfer to Polars or other Arrow-compatible
        systems.

        Parameters
        ----------
        trades : List[Dict]
            List of trade dictionaries with keys:
            - timestamp: int (milliseconds since epoch)
            - price: float
            - quantity: float (or 'volume')

        Returns
        -------
        PyRecordBatch
            Arrow RecordBatch with 30 columns (OHLCV + microstructure).
            Use `polars.from_arrow()` for zero-copy conversion.

        Examples
        --------
        >>> import polars as pl
        >>> processor = OpenDeviationBarProcessor(250)
        >>> for trade_batch in stream_binance_trades(  # doctest: +SKIP
        ...     "BTCUSDT", "2024-01-01", "2024-01-01"
        ... ):
        ...     arrow_batch = processor.process_trades_streaming_arrow(trade_batch)
        ...     df = pl.from_arrow(arrow_batch)  # Zero-copy!
        ...     process_batch(df)

        Notes
        -----
        Requires the `arrow-export` feature to be enabled (default in v8.0+).
        """
        if not trades:
            # Return empty batch with correct schema
            from opendeviationbar._core import bars_to_arrow

            return bars_to_arrow([])

        return self._processor.process_trades_streaming_arrow(trades)

    def process_trades_arrow(self, batch: PyRecordBatch) -> PyRecordBatch:
        """Process trades from Arrow RecordBatch (Issue #88: Arrow-native input).

        Zero-copy input path: accepts Arrow RecordBatch or Table directly,
        avoiding the .to_dicts() bottleneck that consumed 65% of pipeline time.

        Parameters
        ----------
        batch : PyRecordBatch or pyarrow.Table or pyarrow.RecordBatch
            Arrow data with columns: timestamp (int64 ms),
            price (float64), volume/quantity (float64).
            Optional: agg_trade_id, first_trade_id, last_trade_id,
            is_buyer_maker, is_best_match.

        Returns
        -------
        PyRecordBatch
            Arrow RecordBatch of completed open deviation bars.
        """
        # Polars .to_arrow() returns pyarrow.Table, not RecordBatch.
        # Convert Table → single RecordBatch for the Rust binding.
        if hasattr(batch, "combine_chunks"):
            batch = batch.combine_chunks().to_batches()
            if not batch:
                from opendeviationbar._core import bars_to_arrow

                return bars_to_arrow([])
            batch = batch[0]

        if batch.num_rows == 0:
            from opendeviationbar._core import bars_to_arrow

            return bars_to_arrow([])

        return self._processor.process_trades_arrow(batch)

    def process_trades_arrow_native(self, batch: PyRecordBatch) -> PyRecordBatch:
        """Process trades from Arrow RecordBatch with native microsecond timestamps.

        Unlike process_trades_arrow() which expects millisecond input,
        this method passes timestamps through without conversion — the Rust
        processor receives and emits raw microsecond values.

        Use this when Parquet tick data is already in microseconds (post Phase 4).
        """
        return self._processor.process_trades_arrow_native(batch)

    def enable_microstructure(
        self,
        *,
        inter_bar_lookback_count: int | None = None,
        inter_bar_lookback_bars: int | None = None,
        include_intra_bar_features: bool = False,
        compute_tier2: bool = True,
        compute_tier3: bool = False,
        compute_hurst: bool | None = None,
        compute_permutation_entropy: bool | None = None,
    ) -> None:
        """Re-enable microstructure features after checkpoint restore (Issue #97).

        After ``from_checkpoint()``, the processor preserves bar state but
        loses microstructure configuration. Call this before processing
        trades to re-enable inter-bar lookback and intra-bar features.

        Parameters
        ----------
        inter_bar_lookback_count : int, optional
            Fixed trade count for lookback window.
        inter_bar_lookback_bars : int, optional
            Bar-relative lookback (takes precedence over count).
        include_intra_bar_features : bool, default=False
            Enable intra-bar features.
        compute_tier2 : bool, default=True
            Enable Tier 2 inter-bar features.
        compute_tier3 : bool, default=False
            Enable Tier 3 inter-bar features (Issue #128).
        compute_hurst : bool | None, default=None
            Per-feature override for Hurst. None = follow compute_tier3.
        compute_permutation_entropy : bool | None, default=None
            Per-feature override for PE. None = follow compute_tier3.
        """
        self._processor.enable_microstructure(
            inter_bar_lookback_count,
            inter_bar_lookback_bars,
            include_intra_bar_features,
            compute_tier2=compute_tier2,
            compute_tier3=compute_tier3,
            compute_hurst=compute_hurst,
            compute_permutation_entropy=compute_permutation_entropy,
        )
        self.inter_bar_lookback_count = inter_bar_lookback_count
        self.inter_bar_lookback_bars = inter_bar_lookback_bars
        self.include_intra_bar_features = include_intra_bar_features

    def last_agg_trade_id(self) -> int | None:
        """Get last processed aggregate trade ID (for gap detection on reconnect).

        Returns None if no trades have been processed yet.
        """
        return self._processor.last_agg_trade_id()

    def last_completed_bar_tid(self) -> int | None:
        """Get last COMPLETED bar's aggregate trade ID.

        Unlike last_agg_trade_id() which includes the forming bar tail,
        this returns the trade ID from the most recently completed or orphaned bar.
        Used by committed_floors to avoid suppressing junction bars.

        Returns None if no bar has been completed yet.
        """
        return self._processor.last_completed_bar_tid()

    def reset_at_ouroboros(self) -> dict | None:
        """Reset processor state at an ouroboros boundary.

        Clears the incomplete bar and position tracking while preserving
        the threshold configuration. Use this when starting fresh at a
        known boundary (year/month/week/day) for reproducibility.

        Returns
        -------
        dict or None
            The orphaned incomplete bar (if any), or None.
            Mark returned bars with `is_orphan=True` for ML filtering.

        Examples
        --------
        >>> # At year boundary (Jan 1 00:00:00 UTC)
        >>> orphaned = processor.reset_at_ouroboros()
        >>> if orphaned:
        ...     orphaned["is_orphan"] = True
        ...     orphaned["ouroboros_boundary"] = "2024-01-01T00:00:00Z"
        ...     orphaned["reason"] = "day_boundary"
        >>> # Continue processing with clean state
        """
        return self._processor.reset_at_ouroboros()
