# polars-exception: backtesting.py requires Pandas DataFrames with DatetimeIndex
# Issue #46: Modularization M4 - Extract shared helpers from __init__.py
"""Shared helper functions for orchestration modules.

These internal functions handle data fetching, trade processing,
and streaming for both Binance and Exness data sources.
"""

from __future__ import annotations

import os
from collections.abc import Iterator
from datetime import UTC, datetime
from typing import TYPE_CHECKING

import pandas as pd
import polars as pl

from opendeviationbar.processors.core import OpenDeviationBarProcessor

if TYPE_CHECKING:
    pass  # polars already imported above


# =============================================================================
# Shared Constants & Utilities (Issue #79 follow-up: code clone cleanup)
# =============================================================================

MARKET_TYPE_MAP: dict[str, str] = {
    "spot": "spot",
    "futures-um": "um",
    "futures-cm": "cm",
    "um": "um",
    "cm": "cm",
}

_END_OF_DAY_OFFSET_SECONDS = 86399


def _market_to_enum(market: str):  # noqa: ANN202
    """Map normalized market string to MarketType enum (DRY helper).

    Parameters
    ----------
    market : str
        Normalized market type: "spot", "um", or "cm".

    Returns
    -------
    MarketType
        The corresponding MarketType enum variant.
    """
    from opendeviationbar._core import MarketType

    return {
        "spot": MarketType.Spot,
        "um": MarketType.FuturesUM,
        "cm": MarketType.FuturesCM,
    }[market]


def _validate_and_normalize_source_market(
    source: str,
    market: str | None,
    symbol: str | None = None,
) -> tuple[str, str]:
    """Validate source/market and return (source_lower, market_normalized).

    If market is None and symbol is provided, resolves market from the
    symbol registry. Raises ValueError for unsupported source or invalid
    Binance market.
    """
    source = source.lower()
    if source not in ("binance", "exness"):
        msg = f"Unknown source: {source!r}. Must be 'binance' or 'exness'"
        raise ValueError(msg)
    if market is None and symbol is not None:
        from opendeviationbar.symbol_registry import resolve_market

        market = resolve_market(symbol, None)
    elif market is None:
        market = "spot"
    market = market.lower()
    if source == "binance" and market not in MARKET_TYPE_MAP:
        msg = (
            f"Unknown market: {market!r}. "
            "Must be 'spot', 'futures-um'/'um', or 'futures-cm'/'cm'"
        )
        raise ValueError(msg)
    return source, MARKET_TYPE_MAP.get(market, market)


def _parse_and_validate_dates(
    start_date: str,
    end_date: str,
) -> tuple[datetime, datetime]:
    """Parse YYYY-MM-DD strings and validate start <= end.

    Returns (start_dt, end_dt) as datetime objects.
    Raises ValueError for bad format or start > end.
    """
    try:
        start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
        end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)
    except ValueError as e:
        msg = f"Invalid date format. Use YYYY-MM-DD: {e}"
        raise ValueError(msg) from e
    if start_dt > end_dt:
        msg = f"start_date must be <= end_date: {start_date} > {end_date}"
        raise ValueError(msg)
    return start_dt, end_dt


def _datetime_to_start_ms(dt: datetime) -> int:
    """Convert datetime to start-of-day millisecond timestamp."""
    return int(dt.timestamp() * 1000)


def _datetime_to_end_ms(dt: datetime) -> int:
    """Convert datetime to end-of-day millisecond timestamp (23:59:59)."""
    return int((dt.timestamp() + _END_OF_DAY_OFFSET_SECONDS) * 1000)


def _parse_microstructure_env_vars(
    include_microstructure: bool,
    inter_bar_lookback_count: int | None,
    inter_bar_lookback_bars: int | None = None,
) -> tuple[int | None, int | None, bool]:
    """Parse lookback env vars and OPENDEVIATIONBAR_INCLUDE_INTRA_BAR_FEATURES.

    Returns (effective_lookback_count, effective_lookback_bars, enable_intra_bar_features).

    Priority: inter_bar_lookback_bars > inter_bar_lookback_count > env vars.
    Env var OPENDEVIATIONBAR_INTER_BAR_LOOKBACK_BARS takes precedence over
    OPENDEVIATIONBAR_INTER_BAR_LOOKBACK_COUNT.
    """
    effective_count = inter_bar_lookback_count
    effective_bars = inter_bar_lookback_bars

    if include_microstructure and effective_count is None and effective_bars is None:
        # Check env vars: OPENDEVIATIONBAR_INTER_BAR_LOOKBACK_BARS takes precedence
        bars_env = os.environ.get("OPENDEVIATIONBAR_INTER_BAR_LOOKBACK_BARS")
        if bars_env is not None:
            effective_bars = int(bars_env)
        else:
            effective_count = int(
                os.environ.get("OPENDEVIATIONBAR_INTER_BAR_LOOKBACK_COUNT", "200")
            )

    enable_intra = False
    if include_microstructure:
        intra_env = os.environ.get(
            "OPENDEVIATIONBAR_INCLUDE_INTRA_BAR_FEATURES", "true"
        )
        enable_intra = intra_env.lower() in ("true", "1", "yes")

    return effective_count, effective_bars, enable_intra


def _create_processor(
    threshold_decimal_bps: int,
    include_microstructure: bool,
    *,
    symbol: str = "",
    prevent_same_timestamp_close: bool = True,
    inter_bar_lookback_count: int | None = None,
    inter_bar_lookback_bars: int | None = None,
    include_intra_bar_features: bool = False,
    processor: OpenDeviationBarProcessor | None = None,
) -> OpenDeviationBarProcessor:
    """Create or reuse a OpenDeviationBarProcessor with env var defaults.

    If processor is provided, returns it as-is (streaming continuation).
    Otherwise creates a new one with parsed env vars.
    """
    if processor is not None:
        return processor

    effective_count, effective_bars, enable_intra = _parse_microstructure_env_vars(
        include_microstructure,
        inter_bar_lookback_count,
        inter_bar_lookback_bars,
    )

    # Honour explicit include_intra_bar_features=True even when env says false
    enable_intra = enable_intra or include_intra_bar_features

    # Issue #128: Per-feature computation toggles from Settings
    from opendeviationbar.config import Settings

    pop = Settings.get().population

    return OpenDeviationBarProcessor(
        threshold_decimal_bps,
        symbol=symbol,
        prevent_same_timestamp_close=prevent_same_timestamp_close,
        inter_bar_lookback_count=effective_count,
        include_intra_bar_features=enable_intra,
        inter_bar_lookback_bars=effective_bars,
        compute_tier2=pop.compute_tier2,
        compute_tier3=pop.compute_tier3,
        compute_hurst=pop.compute_hurst if pop.compute_hurst else None,
        compute_permutation_entropy=pop.compute_permutation_entropy
        if pop.compute_permutation_entropy
        else None,
    )


def _select_trade_columns(
    trades: pl.LazyFrame | pl.DataFrame,
) -> pl.LazyFrame | pl.DataFrame:
    """Select minimal trade columns for open deviation bar processing (MEM-003).

    Handles: timestamp, price, quantity/volume alias, is_buyer_maker,
    trade ID columns (Issue #75).
    Works with both LazyFrame (predicate pushdown) and DataFrame.
    """
    # polars imported at module level

    # Determine volume column name (works for both DataFrame and LazyFrame)
    if isinstance(trades, pl.LazyFrame):
        available_cols = trades.collect_schema().names()
    else:
        available_cols = trades.columns

    volume_col = "quantity" if "quantity" in available_cols else "volume"

    columns = [
        pl.col("timestamp"),
        pl.col("price"),
        pl.col(volume_col).alias("quantity"),
    ]
    if "is_buyer_maker" in available_cols:
        columns.append(pl.col("is_buyer_maker"))

    # Issue #75: Include trade ID columns for data integrity tracking (Issue #72)
    for trade_id_col in ("agg_trade_id", "first_trade_id", "last_trade_id"):
        if trade_id_col in available_cols:
            columns.append(pl.col(trade_id_col))

    return trades.select(columns)


def _empty_ohlcv_dataframe() -> pd.DataFrame:
    """Return an empty OHLCV DataFrame with standard columns."""
    return pd.DataFrame(columns=["Open", "High", "Low", "Close", "Volume"]).set_index(
        pd.DatetimeIndex([])
    )


def _no_data_error_message(
    symbol: str,
    start_date: str,
    end_date: str,
    market: str | None = None,
) -> str:
    """Build informative error message when no data is available."""
    from opendeviationbar.symbol_registry import resolve_market

    market = resolve_market(symbol, market)
    earliest = detect_earliest_available_date(symbol, market)
    if earliest and earliest > start_date:
        return (
            f"No data available for {symbol} from {start_date} to {end_date}. "
            f"Symbol listing date detected: {earliest}. "
            f"Try: get_open_deviation_bars('{symbol}', '{earliest}', '{end_date}')"
        )
    return f"No data available for {symbol} from {start_date} to {end_date}"


# =============================================================================
# Streaming & Processing
# =============================================================================


def _stream_bars_with_processor(
    trade_batches: Iterator[pl.DataFrame],
    processor: OpenDeviationBarProcessor,
    batch_size: int = 10_000,
    include_incomplete: bool = False,
    *,
    use_native_arrow: bool = False,
) -> Iterator[pl.DataFrame]:
    """Stream bars from any trade source through processor (Arrow path).

    Trade-source-agnostic streaming core. Accepts an iterator of Polars
    DataFrames (trade chunks), processes each through the Arrow path
    (zero-copy), and yields bar batches.

    Parameters
    ----------
    trade_batches : Iterator[pl.DataFrame]
        Iterator of Polars DataFrames containing trade data.
    processor : OpenDeviationBarProcessor
        Processor with state for cross-chunk continuity.
    batch_size : int, default=10_000
        Number of bars per yielded DataFrame (~500 KB each).
    include_incomplete : bool, default=False
        Include the final incomplete bar.
    use_native_arrow : bool, default=False
        Use process_trades_arrow_native() (μs timestamps, Binance path)
        instead of process_trades_arrow() (general path).

    Yields
    ------
    pl.DataFrame
        Batches of bar DataFrames (bounded memory).

    Memory Usage
    ------------
    Peak: ~50 MB (one trade chunk + bar buffer).
    """
    bar_frames: list[pl.DataFrame] = []
    total_buffered = 0

    for trade_chunk in trade_batches:
        chunk = trade_chunk
        chunk_arrow = chunk.to_arrow()
        if use_native_arrow:
            # µs timestamps: use process_trades_arrow_native (timestamp_is_microseconds=true).
            # Sources: stream_binance_trades_arrow (Rust WS) and Parquet tick cache (now µs-native).
            bars_arrow = processor.process_trades_arrow_native(chunk_arrow)
        else:
            bars_arrow = processor.process_trades_arrow(chunk_arrow)
        _bars_result = pl.from_arrow(bars_arrow)
        bars_df = _bars_result if isinstance(_bars_result, pl.DataFrame) else _bars_result.to_frame()

        # Normalize Arrow timestamps to match ClickHouse schema:
        # Arrow path returns open_time/close_time in microseconds (μs).
        # Keep _ms column names until CH schema migration (Plan 05-04).
        # Values are microseconds but column names match live CH schema.
        ts_exprs = []
        if "open_time" in bars_df.columns:
            ts_exprs.append(
                pl.col("open_time").alias("open_time_us")
            )
        if "close_time" in bars_df.columns:
            ts_exprs.append(
                pl.col("close_time").alias("close_time_us")
            )
        if ts_exprs:
            drop_cols = [c for c in ("open_time", "close_time") if c in bars_df.columns]
            bars_df = bars_df.with_columns(ts_exprs).drop(drop_cols)

        if not bars_df.is_empty():
            if __debug__:
                try:
                    from opendeviationbar.validation.schemas import BarOutputSchema

                    BarOutputSchema.validate(bars_df)
                except ImportError:
                    pass  # pandera not installed (prod)

            bar_frames.append(bars_df)
            total_buffered += len(bars_df)

        while total_buffered >= batch_size:
            combined = pl.concat(bar_frames, rechunk=False)
            yield combined.head(batch_size)
            remainder = combined.slice(batch_size)
            is_remainder_empty = remainder.is_empty()
            bar_frames = [remainder] if not is_remainder_empty else []
            total_buffered = len(remainder) if not is_remainder_empty else 0

    if include_incomplete:
        incomplete = processor.get_incomplete_bar()
        if incomplete:
            incomplete_df = pl.DataFrame([incomplete], infer_schema_length=None)
            # Align schema to Arrow-path frames to prevent pl.concat width mismatch.
            # Arrow export produces a different column set than the dict export:
            # Arrow has turnover/buy_trade_count/sell_trade_count/etc. (32 cols),
            # dict has lookback_* features (31 cols). Normalize to Arrow schema
            # so pl.concat succeeds. Missing cols are null-filled; extra cols dropped.
            if bar_frames:
                ref_cols = bar_frames[0].columns
                ref_schema = bar_frames[0].schema
                cast_exprs = []
                for col in ref_cols:
                    if col not in incomplete_df.columns:
                        # Add missing Arrow columns as null with correct dtype.
                        incomplete_df = incomplete_df.with_columns(
                            pl.lit(None).cast(ref_schema[col]).alias(col)
                        )
                    elif incomplete_df[col].dtype != ref_schema[col]:
                        # Cast existing columns to match Arrow dtype (e.g. Int64
                        # from dict vs UInt32 from Arrow for trade counts).
                        cast_exprs.append(pl.col(col).cast(ref_schema[col]))
                if cast_exprs:
                    incomplete_df = incomplete_df.with_columns(cast_exprs)
                incomplete_df = incomplete_df.select(ref_cols)
            bar_frames.append(incomplete_df)

    if bar_frames:
        yield pl.concat(bar_frames, rechunk=False)


def _stream_open_deviation_bars_binance(
    symbol: str,
    start_date: str,
    end_date: str,
    threshold_decimal_bps: int,
    market: str,
    batch_size: int = 10_000,
    include_microstructure: bool = False,
    include_incomplete: bool = False,
    prevent_same_timestamp_close: bool = True,
    verify_checksum: bool = True,
    inter_bar_lookback_count: int | None = None,
    *,
    processor: OpenDeviationBarProcessor | None = None,
) -> Iterator[pl.DataFrame]:
    """Stream open deviation bars from Binance API.

    Delegates to _stream_bars_with_processor() for buffer/yield logic.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT")
    start_date : str
        Start date "YYYY-MM-DD"
    end_date : str
        End date "YYYY-MM-DD"
    threshold_decimal_bps : int
        Open deviation bar threshold (250 = 0.25%)
    market : str
        Normalized market type: "spot", "um", or "cm"
    batch_size : int, default=10_000
        Number of bars per yielded DataFrame (~500 KB each)
    include_microstructure : bool, default=False
        Include microstructure columns in output
    include_incomplete : bool, default=False
        Include the final incomplete bar
    prevent_same_timestamp_close : bool, default=True
        Timestamp gating for flash crash prevention
    verify_checksum : bool, default=True
        Verify SHA-256 checksum of downloaded data
    inter_bar_lookback_count : int, optional
        Lookback trade count for inter-bar features (Issue #59)
    processor : OpenDeviationBarProcessor, optional
        Existing processor with state (for cross-call continuity).
        If None, creates a new one.

    Yields
    ------
    pl.DataFrame
        Batches of open deviation bars (OHLCV format, backtesting.py compatible)

    Memory Usage
    ------------
    Peak: ~50 MB (6-hour trade chunk + bar buffer)
    Per yield: ~500 KB (10,000 bars)
    """
    try:
        from opendeviationbar._core import stream_binance_trades_arrow
    except ImportError as e:
        msg = (
            "Streaming requires the 'data-providers' and 'arrow-export' features. "
            "Rebuild with: maturin develop --features data-providers,arrow-export"
        )
        raise RuntimeError(msg) from e

    market_enum = _market_to_enum(market)

    processor = _create_processor(
        threshold_decimal_bps,
        include_microstructure,
        symbol=symbol,
        prevent_same_timestamp_close=prevent_same_timestamp_close,
        inter_bar_lookback_count=inter_bar_lookback_count,
        processor=processor,
    )

    def trade_batches() -> Iterator[pl.DataFrame]:
        for trade_arrow in stream_binance_trades_arrow(
            symbol,
            start_date,
            end_date,
            chunk_hours=6,
            market_type=market_enum,
            verify_checksum=verify_checksum,
        ):
            result = pl.from_arrow(trade_arrow)
            if isinstance(result, pl.DataFrame):
                yield result
            else:
                yield result.to_frame()

    yield from _stream_bars_with_processor(
        trade_batches(),
        processor,
        batch_size,
        include_incomplete,
        use_native_arrow=True,
    )


def _fetch_binance(
    symbol: str,
    start_date: str,
    end_date: str,
    market: str,
) -> pl.DataFrame:
    """Fetch Binance aggTrades data (internal).

    DEPRECATED: Use stream_binance_trades() for memory-efficient streaming.
    This function loads all trades into memory at once.
    """
    import warnings
    from datetime import datetime

    # polars imported at module level

    # MEM-007: Guard deprecated batch path with date range limit (Issue #49)
    # This function loads ALL trades into a single DataFrame. For high-volume
    # symbols (BTCUSDT), a single month can be ~6GB. Limit to 30 days.
    max_days = 30
    days = (
        datetime.strptime(end_date, "%Y-%m-%d")
        - datetime.strptime(start_date, "%Y-%m-%d")
    ).days
    if days > max_days:
        msg = (
            f"_fetch_binance() cannot safely load {days} days of data. "
            f"This deprecated path loads all trades into memory at once "
            f"(limit: {max_days} days). Use precompute_open_deviation_bars() or "
            f"get_open_deviation_bars() with per-segment loading instead."
        )
        raise MemoryError(msg)

    warnings.warn(
        "_fetch_binance() is deprecated. Use stream_binance_trades() for "
        "memory-efficient streaming. This function will be removed in v9.0.",
        DeprecationWarning,
        stacklevel=2,
    )

    try:
        from opendeviationbar._core import fetch_binance_aggtrades

        market_enum = _market_to_enum(market)

        trades_list = fetch_binance_aggtrades(symbol, start_date, end_date, market_enum)
        return pl.DataFrame(trades_list)

    except ImportError as e:
        msg = (
            "Binance data fetching requires the 'data-providers' feature. "
            "Rebuild with: maturin develop --features data-providers"
        )
        raise RuntimeError(msg) from e

    except RuntimeError as e:
        # Issue #76: Enhance "No data available" errors with listing date detection
        error_msg = str(e)
        if "No data available" in error_msg or "HTTP 404" in error_msg:
            enhanced = _no_data_error_message(symbol, start_date, end_date, market)
            if enhanced != error_msg:
                raise RuntimeError(enhanced) from e
        raise


def _fetch_exness(
    symbol: str,
    start_date: str,
    end_date: str,
    validation: str,
) -> pl.DataFrame:
    """Fetch Exness tick data (internal)."""
    try:
        from opendeviationbar.exness import fetch_exness_ticks

        return fetch_exness_ticks(symbol, start_date, end_date)

    except ImportError as e:
        msg = (
            "Exness data fetching requires the 'exness' feature. "
            "Rebuild with: maturin develop --features data-providers"
        )
        raise RuntimeError(msg) from e


def _stream_bars_from_trades(
    trades: pl.DataFrame,
    threshold_decimal_bps: int,
    include_microstructure: bool,
    processor: OpenDeviationBarProcessor,
    bar_batch_size: int = 10_000,
) -> Iterator[pl.DataFrame]:
    """Stream open deviation bars from existing DataFrame. Delegates to _stream_bars_with_processor().

    Parameters
    ----------
    trades : pl.DataFrame
        Polars DataFrame with tick data (already column-selected)
    threshold_decimal_bps : int
        Threshold in decimal basis points (unused, retained for API compat)
    include_microstructure : bool
        Affects trade chunk size (50K vs 100K)
    processor : OpenDeviationBarProcessor
        Processor with state for cross-file continuity
    bar_batch_size : int, default=10_000
        Number of bars per yielded batch (~60 MB with microstructure)

    Yields
    ------
    pl.DataFrame
        Batches of bar DataFrames (bounded memory)
    """
    _ = threshold_decimal_bps  # retained for caller API compatibility

    # MEM-002 + MEM-011: Adaptive chunk size based on output features
    trade_chunk_size = 50_000 if include_microstructure else 100_000

    def trade_batches() -> Iterator[pl.DataFrame]:
        n_rows = len(trades)
        for start in range(0, n_rows, trade_chunk_size):
            yield trades.slice(start, trade_chunk_size)

    yield from _stream_bars_with_processor(
        trade_batches(), processor, bar_batch_size,
    )


def _process_binance_trades(
    trades: pl.DataFrame,
    threshold_decimal_bps: int,
    include_microstructure: bool,
    *,
    processor: OpenDeviationBarProcessor | None = None,
    symbol: str | None = None,
    prevent_same_timestamp_close: bool = True,
    inter_bar_lookback_count: int | None = None,
    include_intra_bar_features: bool = False,
    inter_bar_lookback_bars: int | None = None,
    return_format: str = "pandas",  # Issue #96 Task #13: Arrow optimization
) -> tuple[pd.DataFrame | pl.DataFrame, OpenDeviationBarProcessor]:
    """Process Binance trades to open deviation bars (internal).

    Parameters
    ----------
    trades : pl.DataFrame
        Polars DataFrame with tick data
    threshold_decimal_bps : int
        Threshold in decimal basis points
    include_microstructure : bool
        Include microstructure columns
    processor : OpenDeviationBarProcessor, optional
        Existing processor with state (for cross-file continuity).
        If None, creates a new processor.
    symbol : str, optional
        Symbol for checkpoint creation
    prevent_same_timestamp_close : bool, default=True
        Timestamp gating for flash crash prevention
    inter_bar_lookback_count : int, optional
        Lookback trade count for inter-bar features (Issue #59)
    include_intra_bar_features : bool, default=False
        Enable intra-bar features (Issue #59)

    Returns
    -------
    tuple[pd.DataFrame, OpenDeviationBarProcessor]
        (bars DataFrame, processor with updated state)
        The processor can be used to create a checkpoint for the next file.
    """
    # polars imported at module level

    # MEM-003: Apply column selection BEFORE collecting LazyFrame
    trades_selected = _select_trade_columns(trades)

    # Collect AFTER selection (for LazyFrame)
    if isinstance(trades_selected, pl.LazyFrame):
        trades_minimal = trades_selected.collect()
    else:
        trades_minimal = trades_selected

    # Use provided processor or create new one (with env var defaults)
    processor = _create_processor(
        threshold_decimal_bps,
        include_microstructure,
        symbol=symbol or "",
        prevent_same_timestamp_close=prevent_same_timestamp_close,
        inter_bar_lookback_count=inter_bar_lookback_count,
        include_intra_bar_features=include_intra_bar_features,
        inter_bar_lookback_bars=inter_bar_lookback_bars,
        processor=processor,
    )

    # MEM-012: Stream bars in batches instead of accumulating all in memory
    # This prevents OOM on large date ranges (2.5+ years) with microstructure
    # Previous: all_bars.extend() caused 2.9GB allocation for 50K bars
    # Now: bounded batches of 10K bars (~60 MB each)
    # Issue #88: _stream_bars_from_trades now yields pl.DataFrame directly (Arrow path)
    bar_batches: list[pl.DataFrame] = []

    for bar_batch_df in _stream_bars_from_trades(
        trades_minimal,
        threshold_decimal_bps,
        include_microstructure,
        processor,
        bar_batch_size=10_000,
    ):
        if not bar_batch_df.is_empty():
            bar_batches.append(bar_batch_df)

    if not bar_batches:
        return _empty_ohlcv_dataframe(), processor

    # MEM-006: Single Polars concat (efficient, no 2x memory spike like pandas)
    from opendeviationbar.processors.api import _arrow_bars_to_pandas

    result_pl = pl.concat(bar_batches, rechunk=False)

    # Issue #96 Task #13: Arrow optimization for cache pipeline
    # Skip Pandas conversion when return_format="arrow" (1.3-1.5x speedup)
    if return_format == "arrow":
        result = result_pl
    else:
        result = _arrow_bars_to_pandas(result_pl, include_microstructure=True)

    # Issue #75: Always return all available columns including trade IDs
    # The caller (get_open_deviation_bars) handles filtering for user-facing output AFTER cache write.
    # This ensures cache always has trade IDs for data integrity verification.
    return result, processor


def _process_exness_ticks(
    ticks: pl.DataFrame,
    symbol: str,
    threshold_decimal_bps: int,
    validation: str,
    include_incomplete: bool,
    include_microstructure: bool,
    *,
    inter_bar_lookback_count: int | None = None,
) -> pd.DataFrame:
    """Process Exness ticks to open deviation bars (internal).

    Note: inter_bar_lookback_count is accepted but not yet supported for Exness.
    Exness uses a separate processing path without inter-bar feature computation.
    """
    _ = inter_bar_lookback_count  # Not supported for Exness processing path
    try:
        # Map validation string to enum
        from opendeviationbar._core import ValidationStrictness
        from opendeviationbar.exness import process_exness_ticks_to_dataframe

        validation_enum = {
            "permissive": ValidationStrictness.Permissive,
            "strict": ValidationStrictness.Strict,
            "paranoid": ValidationStrictness.Paranoid,
        }[validation]

        # Get instrument enum
        from opendeviationbar._core import ExnessInstrument

        instrument_map = {
            "EURUSD": ExnessInstrument.EURUSD,
            "GBPUSD": ExnessInstrument.GBPUSD,
            "USDJPY": ExnessInstrument.USDJPY,
            "AUDUSD": ExnessInstrument.AUDUSD,
            "USDCAD": ExnessInstrument.USDCAD,
            "NZDUSD": ExnessInstrument.NZDUSD,
            "EURGBP": ExnessInstrument.EURGBP,
            "EURJPY": ExnessInstrument.EURJPY,
            "GBPJPY": ExnessInstrument.GBPJPY,
            "XAUUSD": ExnessInstrument.XAUUSD,
        }

        if symbol.upper() not in instrument_map:
            msg = (
                f"Unknown Exness instrument: {symbol}. "
                f"Valid instruments: {list(instrument_map.keys())}"
            )
            raise ValueError(msg)

        instrument = instrument_map[symbol.upper()]

        df = process_exness_ticks_to_dataframe(
            ticks.to_pandas(),
            instrument,
            threshold_decimal_bps,
            validation_enum,
        )

        if not include_microstructure:
            return df[["Open", "High", "Low", "Close", "Volume"]]

        return df

    except ImportError as e:
        msg = (
            "Exness processing requires the 'exness' feature. "
            "Rebuild with: maturin develop --features data-providers"
        )
        raise RuntimeError(msg) from e


# Issue #76: Auto-detect earliest available date for Binance symbols
# REMOVED (Issue #79): KNOWN_LISTING_DATES dict migrated to symbols.toml
# SSoT is now python/opendeviationbar/data/symbols.toml (Unified Symbol Registry)


def detect_earliest_available_date(
    symbol: str,
    market: str | None = None,
    *,
    max_probes: int = 10,
) -> str | None:
    """Detect the earliest date with available data for a Binance symbol.

    Uses binary search on Binance Vision API to find the first available date.
    Returns known listing date if available to avoid network overhead.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT")
    market : str
        Market type: "spot", "um" (USDT-M futures), "cm" (COIN-M futures)
    max_probes : int
        Maximum number of HTTP probes for binary search (default: 10)

    Returns
    -------
    str | None
        Earliest available date as "YYYY-MM-DD", or None if detection fails

    Example
    -------
    >>> detect_earliest_available_date("SOLUSDT")
    '2020-08-11'
    """
    import urllib.error
    import urllib.request
    from datetime import UTC, datetime, timedelta

    http_ok = 200

    # Check symbol registry first (no network needed) -- Issue #79
    from opendeviationbar.symbol_registry import (
        get_effective_start_date,
        resolve_market,
    )

    effective = get_effective_start_date(symbol)
    if effective is not None:
        return effective

    # Resolve market from registry if not provided
    market = resolve_market(symbol, market)

    # Spot only (2026-03-11 policy) — Vision and REST produce identical boundaries
    market_path = "spot"

    def check_date(date_str: str) -> bool:
        """Check if data exists for a specific date via HTTP HEAD."""
        url = (
            f"https://data.binance.vision/data/{market_path}/daily/"
            f"aggTrades/{symbol}/{symbol}-aggTrades-{date_str}.zip"
        )
        try:
            req = urllib.request.Request(url, method="HEAD")
            with urllib.request.urlopen(req, timeout=5) as resp:
                return resp.status == http_ok
        except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError):
            # HTTPError: 404/403/etc, URLError: DNS/connection, TimeoutError, OSError: network
            return False

    # Binary search between 2017-01-01 and today
    left = datetime(2017, 1, 1, tzinfo=UTC)
    right = datetime.now(UTC)
    earliest_found: str | None = None

    for _ in range(max_probes):
        if left >= right:
            break

        mid = left + (right - left) // 2
        mid_str = mid.strftime("%Y-%m-%d")

        if check_date(mid_str):
            earliest_found = mid_str
            right = mid - timedelta(days=1)
        else:
            left = mid + timedelta(days=1)

    # Verify the found date
    if earliest_found:
        return earliest_found

    # Fallback: linear probe from recent history
    probe_date = datetime.now(UTC) - timedelta(days=365)
    for _ in range(30):
        date_str = probe_date.strftime("%Y-%m-%d")
        if check_date(date_str):
            return date_str
        probe_date -= timedelta(days=30)

    return None
