# FILE-SIZE-OK: query methods are cohesive
# polars-exception: backtesting.py requires Pandas DataFrames with DatetimeIndex
# Issue #46: Modularization M6 - Extract query operations from cache.py
# Issue #158: Stathera — get_latest_bar_with_trade_id, get_bars_overlapping_trade_span
"""Query operations for ClickHouse open deviation bar cache.

Provides mixin methods for retrieving open deviation bars by count (get_n_bars)
and by timestamp range (get_bars_by_timestamp_range). Used by OpenDeviationBarCache
via mixin inheritance.
"""

from __future__ import annotations

import logging

import pandas as pd

from ..constants import (
    _PLUGIN_FEATURE_COLUMNS,  # Issue #98: FeatureProvider plugin columns
    BAR_FLAG_COLUMNS,  # Issue #101: is_liquidation_cascade and future flags
    INTER_BAR_FEATURE_COLUMNS,
    INTRA_BAR_FEATURE_COLUMNS,
    MICROSTRUCTURE_COLUMNS,
    MIN_VERSION_FOR_MICROSTRUCTURE,
    TRADE_ID_RANGE_COLUMNS,  # Issue #111: Ariadne at every boundary
)
from ..conversion import normalize_arrow_dtypes
from ..exceptions import CacheReadError
from .constants import CLICKHOUSE_TRANSIENT_ERRORS, FINAL_READ_SETTINGS

logger = logging.getLogger(__name__)


class QueryOperationsMixin:
    """Mixin providing query operations for OpenDeviationBarCache.

    Requires `self.client` and `self.count_bars()` from parent class.
    """

    def get_n_bars(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        n_bars: int,
        before_ts: int | None = None,
        include_microstructure: bool = False,
        min_schema_version: str | None = None,
        include_plugin_features: bool = False,
        ouroboros_mode: str = "day",
    ) -> tuple[pd.DataFrame | None, int]:
        """Get N bars from cache, ordered chronologically (oldest first).

        Uses ORDER BY close_time_ms DESC LIMIT N for efficient retrieval,
        then reverses in Python for chronological order.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        n_bars : int
            Maximum number of bars to retrieve
        before_ts : int | None
            Only get bars with close_time_ms < before_ts.
            If None, gets most recent bars.
        include_microstructure : bool
            If True, includes vwap, buy_volume, sell_volume columns
        min_schema_version : str | None
            Minimum schema version required for cache hit. If specified,
            only returns data with opendeviationbar_version >= min_schema_version.
            When include_microstructure=True and min_schema_version=None,
            automatically requires version >= 7.0.0.

        Returns
        -------
        tuple[pd.DataFrame | None, int]
            (bars_df, available_count) where:
            - bars_df is OHLCV DataFrame (or None if no bars)
            - available_count is total bars available (may be > len(bars_df))
        """
        # First get the count (for reporting)
        available_count = self.count_bars(
            symbol,
            threshold_decimal_bps,
            before_ts,
            ouroboros_mode=ouroboros_mode,
        )

        if available_count == 0:
            return None, 0

        # Select columns
        base_cols = """
            close_time_ms,
            open as Open,
            high as High,
            low as Low,
            close as Close,
            volume as Volume
        """
        # Issue #101: Bar flags always included (lightweight boolean metadata)
        base_cols += "," + ",".join(BAR_FLAG_COLUMNS)
        # Issue #111 (Ariadne): Trade IDs at every read boundary
        base_cols += "," + ",".join(TRADE_ID_RANGE_COLUMNS)
        if include_microstructure:
            base_cols += """,
            vwap,
            buy_volume,
            sell_volume,
            duration_us,
            ofi,
            vwap_close_deviation,
            price_impact,
            kyle_lambda_proxy,
            trade_intensity,
            volume_per_trade,
            aggression_ratio,
            aggregation_density,
            turnover_imbalance
        """
            # Issue #78: Add inter-bar lookback features (were missing from queries)
            base_cols += "," + ",".join(INTER_BAR_FEATURE_COLUMNS)
            # Issue #78: Add intra-bar features
            base_cols += "," + ",".join(INTRA_BAR_FEATURE_COLUMNS)

        # Issue #98: Add plugin feature columns if registered and requested
        if include_plugin_features and _PLUGIN_FEATURE_COLUMNS:
            base_cols += "," + ",".join(_PLUGIN_FEATURE_COLUMNS)

        # Determine effective min version for schema evolution filtering
        effective_min_version = min_schema_version
        if include_microstructure and effective_min_version is None:
            effective_min_version = MIN_VERSION_FOR_MICROSTRUCTURE

        # Build version filter if specified
        version_filter = ""
        if effective_min_version:
            version_filter = """
              AND opendeviationbar_version != ''
              AND toUInt32OrZero(
                splitByChar('.', opendeviationbar_version)[1]
              ) >= toUInt32OrZero(
                splitByChar('.', {min_version:String})[1]
              )"""

        if before_ts is not None:
            # Split path: with end_ts filter
            query = f"""
                SELECT {base_cols}
                FROM opendeviationbar_cache.open_deviation_bars FINAL
                WHERE symbol = {{symbol:String}}
                  AND threshold_decimal_bps = {{threshold:UInt32}}
                  AND ouroboros_mode = {{ouroboros_mode:String}}
                  AND close_time_ms < {{end_ts:Int64}}
                  {version_filter}
                ORDER BY close_time_ms DESC
                LIMIT {{n_bars:UInt64}}
            """
            params: dict[str, str | int] = {
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
                "ouroboros_mode": ouroboros_mode,
                "end_ts": before_ts,
                "n_bars": n_bars,
            }
            if effective_min_version:
                params["min_version"] = effective_min_version
            df = self.client.query_df_arrow(
                query,
                parameters=params,
                settings=FINAL_READ_SETTINGS,
            )
        else:
            # Split path: no end_ts filter (most recent)
            query = f"""
                SELECT {base_cols}
                FROM opendeviationbar_cache.open_deviation_bars FINAL
                WHERE symbol = {{symbol:String}}
                  AND threshold_decimal_bps = {{threshold:UInt32}}
                  AND ouroboros_mode = {{ouroboros_mode:String}}
                  {version_filter}
                ORDER BY close_time_ms DESC
                LIMIT {{n_bars:UInt64}}
            """
            params = {
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
                "ouroboros_mode": ouroboros_mode,
                "n_bars": n_bars,
            }
            if effective_min_version:
                params["min_version"] = effective_min_version
            df = self.client.query_df_arrow(
                query,
                parameters=params,
                settings=FINAL_READ_SETTINGS,
            )

        if df.empty:
            return None, available_count

        # Issue #96 Task #38: Skip reset_index after reversal (set_index handles it)
        # Reverse to chronological order (oldest first)
        df = df.iloc[::-1]

        # Convert to TZ-aware UTC DatetimeIndex
        # (Issue #20: match get_open_deviation_bars output)
        df["timestamp"] = pd.to_datetime(df["close_time_ms"], unit="ms", utc=True)
        df = df.set_index("timestamp")
        df = df.drop(columns=["close_time_ms"])

        # Convert PyArrow dtypes to numpy for compatibility
        df = normalize_arrow_dtypes(df)

        # Convert microstructure columns if present
        if include_microstructure:
            df = normalize_arrow_dtypes(df, columns=list(MICROSTRUCTURE_COLUMNS))

        return df, available_count

    def get_latest_bar_timestamp(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        ouroboros_mode: str | None = None,
    ) -> int | None:
        """Get the most recent bar timestamp for a symbol x threshold pair.

        Used by backfill to detect the gap between cached data
        and current time.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        ouroboros_mode : str | None
            Ouroboros mode filter. If None, resolved from config.  # Issue #126

        Returns
        -------
        int | None
            Latest close_time_ms value, or None if no bars exist.
        """
        if ouroboros_mode is None:
            from opendeviationbar.ouroboros import get_operational_ouroboros_mode

            ouroboros_mode = get_operational_ouroboros_mode()

        query = """
            SELECT max(close_time_ms) as latest_ts
            FROM opendeviationbar_cache.open_deviation_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND ouroboros_mode = {ouroboros:String}
        """
        params: dict[str, str | int] = {
            "symbol": symbol,
            "threshold": threshold_decimal_bps,
            "ouroboros": ouroboros_mode,
        }

        try:
            result = self.client.query(
                query,
                parameters=params,
                settings=FINAL_READ_SETTINGS,
            )
        except CLICKHOUSE_TRANSIENT_ERRORS:
            logger.exception(
                "Failed to query latest timestamp for %s @ %d dbps",
                symbol,
                threshold_decimal_bps,
            )
            return None

        if not result.result_rows or result.result_rows[0][0] == 0:
            return None

        return result.result_rows[0][0]

    def get_ariadne_high_water_mark(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        ouroboros_mode: str | None = None,
    ) -> int | None:
        """Get the highest last_agg_trade_id written for a symbol x threshold.

        Issue #111 (Ariadne): This is the resume point — every boundary
        and every touchpoint. Returns the exact trade ID to pass to
        ``fetch_aggtrades_by_id(symbol, hwm + 1)`` for gap-free resume.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        ouroboros_mode : str | None
            Ouroboros mode filter. If None, resolved from config.  # Issue #126

        Returns
        -------
        int | None
            Highest last_agg_trade_id, or None if no bars have trade IDs.
        """
        if ouroboros_mode is None:
            from opendeviationbar.ouroboros import get_operational_ouroboros_mode

            ouroboros_mode = get_operational_ouroboros_mode()

        query = """
            SELECT max(last_agg_trade_id) as hwm
            FROM opendeviationbar_cache.open_deviation_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND ouroboros_mode = {ouroboros:String}
              AND last_agg_trade_id > 0
        """
        params: dict[str, str | int] = {
            "symbol": symbol,
            "threshold": threshold_decimal_bps,
            "ouroboros": ouroboros_mode,
        }

        try:
            result = self.client.query(
                query,
                parameters=params,
                settings=FINAL_READ_SETTINGS,
            )
        except CLICKHOUSE_TRANSIENT_ERRORS:
            logger.exception(
                "Failed to query Ariadne HWM for %s @ %d dbps",
                symbol,
                threshold_decimal_bps,
            )
            return None

        if not result.result_rows or result.result_rows[0][0] == 0:
            return None

        return result.result_rows[0][0]

    def get_latest_bar_with_trade_id(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        ouroboros_mode: str | None = None,  # SSoT-OK: resolved from config
    ) -> tuple[int | None, int | None]:
        """Get latest close_time_ms and last_agg_trade_id in one query.

        Issue #158 (Stathera Phase 4): Combined query replacing two separate
        round-trips to get_latest_bar_timestamp() + get_ariadne_high_water_mark().

        Returns
        -------
        tuple[int | None, int | None]
            (latest_close_time_ms, latest_last_agg_trade_id)
        """
        if ouroboros_mode is None:
            from opendeviationbar.ouroboros import get_operational_ouroboros_mode

            ouroboros_mode = get_operational_ouroboros_mode()

        query = """
            SELECT max(close_time_ms),
                   maxIf(last_agg_trade_id, last_agg_trade_id > 0)
            FROM opendeviationbar_cache.open_deviation_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND ouroboros_mode = {ouroboros:String}
        """
        try:
            result = self.client.query(
                query,
                parameters={
                    "symbol": symbol,
                    "threshold": threshold_decimal_bps,
                    "ouroboros": ouroboros_mode,
                },
                settings=FINAL_READ_SETTINGS,
            )
        except CLICKHOUSE_TRANSIENT_ERRORS:
            logger.exception(
                "Failed to query latest bar with trade ID for %s @ %d dbps",
                symbol,
                threshold_decimal_bps,
            )
            return None, None

        if not result.result_rows:
            return None, None

        latest_ts, latest_tid = result.result_rows[0]
        return (
            latest_ts if latest_ts and latest_ts > 0 else None,
            latest_tid if latest_tid and latest_tid > 0 else None,
        )

    def get_bars_overlapping_trade_span(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        new_first_tid: int,
        new_last_tid: int,
        ouroboros_mode: str | None = None,  # SSoT-OK: resolved from config
        limit: int = 10,
    ) -> list[tuple[int, int, int]]:
        """Find existing bars that overlap a trade-ID span.

        Issue #158 (Stathera Phase 2): Pre-write overlap check.

        Returns
        -------
        list[tuple[int, int, int]]
            List of (close_time_ms, first_agg_trade_id, last_agg_trade_id)
            for overlapping bars.
        """
        if ouroboros_mode is None:
            from opendeviationbar.ouroboros import get_operational_ouroboros_mode

            ouroboros_mode = get_operational_ouroboros_mode()

        query = """
            SELECT close_time_ms, first_agg_trade_id, last_agg_trade_id
            FROM opendeviationbar_cache.open_deviation_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND ouroboros_mode = {ouroboros:String}
              AND first_agg_trade_id <= {new_last:Int64}
              AND last_agg_trade_id >= {new_first:Int64}
              AND first_agg_trade_id > 0
            LIMIT {limit:UInt32}
        """
        try:
            result = self.client.query(
                query,
                parameters={
                    "symbol": symbol,
                    "threshold": threshold_decimal_bps,
                    "ouroboros": ouroboros_mode,
                    "new_first": new_first_tid,
                    "new_last": new_last_tid,
                    "limit": limit,
                },
                settings=FINAL_READ_SETTINGS,
            )
        except CLICKHOUSE_TRANSIENT_ERRORS:
            logger.exception(
                "Failed to check trade-ID overlap for %s @ %d dbps",
                symbol,
                threshold_decimal_bps,
            )
            return []

        return [
            (row[0], row[1], row[2])
            for row in result.result_rows
        ]

    def get_backfill_status(
        self,
        symbol: str,
        limit: int = 10,
    ) -> list[dict]:
        """Get backfill request status for a symbol (Issue #104).

        Returns recent backfill requests for flowsurface to poll.
        Includes in-flight progress (bars_written updates incrementally
        while status is 'running').

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        limit : int
            Maximum number of recent requests to return.

        Returns
        -------
        list[dict]
            List of request status dicts with keys:
            request_id, symbol, threshold_decimal_bps, status,
            requested_at, started_at, completed_at, bars_written,
            gap_seconds, error.
        """
        query = """
            SELECT
                toString(request_id) AS request_id,
                symbol,
                threshold_decimal_bps,
                status,
                requested_at,
                started_at,
                completed_at,
                bars_written,
                gap_seconds,
                error
            FROM opendeviationbar_cache.backfill_requests FINAL
            WHERE symbol = {symbol:String}
            ORDER BY requested_at DESC
            LIMIT {limit:UInt32}
        """
        try:
            result = self.client.query(
                query,
                parameters={"symbol": symbol, "limit": limit},
                settings=FINAL_READ_SETTINGS,
            )
        except CLICKHOUSE_TRANSIENT_ERRORS:
            logger.exception(
                "Failed to query backfill status for %s",
                symbol,
            )
            return []

        columns = [
            "request_id",
            "symbol",
            "threshold_decimal_bps",
            "status",
            "requested_at",
            "started_at",
            "completed_at",
            "bars_written",
            "gap_seconds",
            "error",
        ]
        return [dict(zip(columns, row, strict=False)) for row in result.result_rows]

    def get_bars_by_timestamp_range(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        start_ts: int,
        end_ts: int,
        include_microstructure: bool = False,
        include_exchange_sessions: bool = False,
        ouroboros_mode: str = "day",
        min_schema_version: str | None = None,
        include_plugin_features: bool = False,
    ) -> pd.DataFrame | None:
        """Get bars within a timestamp range (for get_open_deviation_bars cache lookup).

        Unlike get_open_deviation_bars() which requires exact CacheKey match,
        this method queries by timestamp range, returning any cached bars
        that fall within [start_ts, end_ts].

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        start_ts : int
            Start timestamp in milliseconds (inclusive)
        end_ts : int
            End timestamp in milliseconds (inclusive)
        include_microstructure : bool
            If True, includes vwap, buy_volume, sell_volume columns
        include_exchange_sessions : bool
            If True, includes exchange_session_* columns (Issue #8)
        ouroboros_mode : str
            Ouroboros reset mode (default: "day"). Only "day" is supported.
        min_schema_version : str | None
            Minimum schema version required for cache hit. If specified,
            only returns data with opendeviationbar_version >= min_schema_version.
            When include_microstructure=True and min_schema_version=None,
            automatically requires version >= 7.0.0.

        Returns
        -------
        pd.DataFrame | None
            OHLCV DataFrame with TZ-aware UTC timestamps if found, None otherwise.
            Returns None if no bars exist in the range or version mismatch.

        Raises
        ------
        CacheReadError
            If the query fails due to database errors.
        """
        # Build column list
        base_cols = """
            close_time_ms,
            open as Open,
            high as High,
            low as Low,
            close as Close,
            volume as Volume
        """
        # Issue #101: Bar flags always included (lightweight boolean metadata)
        base_cols += "," + ",".join(BAR_FLAG_COLUMNS)
        # Issue #111 (Ariadne): Trade IDs at every read boundary
        base_cols += "," + ",".join(TRADE_ID_RANGE_COLUMNS)
        if include_microstructure:
            base_cols += """,
            vwap,
            buy_volume,
            sell_volume,
            duration_us,
            ofi,
            vwap_close_deviation,
            price_impact,
            kyle_lambda_proxy,
            trade_intensity,
            volume_per_trade,
            aggression_ratio,
            aggregation_density,
            turnover_imbalance
        """
            # Issue #78: Add inter-bar lookback features (were missing from queries)
            base_cols += "," + ",".join(INTER_BAR_FEATURE_COLUMNS)
            # Issue #78: Add intra-bar features
            base_cols += "," + ",".join(INTRA_BAR_FEATURE_COLUMNS)

        # Issue #8: Exchange session flags
        if include_exchange_sessions:
            base_cols += """,
            exchange_session_sydney,
            exchange_session_tokyo,
            exchange_session_london,
            exchange_session_newyork
        """

        # Issue #98: Add plugin feature columns if registered and requested
        if include_plugin_features and _PLUGIN_FEATURE_COLUMNS:
            base_cols += "," + ",".join(_PLUGIN_FEATURE_COLUMNS)

        # Ouroboros mode filter ensures cache isolation between modes
        # Plan: sparkling-coalescing-dijkstra.md

        # Determine effective min version for schema evolution filtering
        effective_min_version = min_schema_version
        if include_microstructure and effective_min_version is None:
            # Auto-require v7.0.0+ for microstructure features
            effective_min_version = MIN_VERSION_FOR_MICROSTRUCTURE

        # Build version filter if specified
        version_filter = ""
        if effective_min_version:
            version_filter = """
              AND opendeviationbar_version != ''
              AND toUInt32OrZero(
                splitByChar('.', opendeviationbar_version)[1]
              ) >= toUInt32OrZero(
                splitByChar('.', {min_version:String})[1]
              )"""

        query = f"""
            SELECT {base_cols}
            FROM opendeviationbar_cache.open_deviation_bars FINAL
            WHERE symbol = {{symbol:String}}
              AND threshold_decimal_bps = {{threshold:UInt32}}
              AND ouroboros_mode = {{ouroboros_mode:String}}
              AND close_time_ms >= {{start_ts:Int64}}
              AND close_time_ms <= {{end_ts:Int64}}
              {version_filter}
            ORDER BY close_time_ms
        """

        # Build parameters
        params: dict[str, str | int] = {
            "symbol": symbol,
            "threshold": threshold_decimal_bps,
            "ouroboros_mode": ouroboros_mode,
            "start_ts": start_ts,
            "end_ts": end_ts,
        }
        if effective_min_version:
            params["min_version"] = effective_min_version

        try:
            df = self.client.query_df_arrow(
                query,
                parameters=params,
                settings=FINAL_READ_SETTINGS,
            )
        except CLICKHOUSE_TRANSIENT_ERRORS as e:
            logger.exception(
                "Cache read failed for %s @ %d dbps (range query)",
                symbol,
                threshold_decimal_bps,
            )
            msg = f"Failed to read bars for {symbol}: {e}"
            raise CacheReadError(
                msg,
                symbol=symbol,
                operation="read_range",
            ) from e

        if df.empty:
            logger.debug(
                "Cache miss for %s @ %d dbps (range: %d-%d)",
                symbol,
                threshold_decimal_bps,
                start_ts,
                end_ts,
            )
            return None

        logger.debug(
            "Cache hit: %d bars for %s @ %d dbps (range query)",
            len(df),
            symbol,
            threshold_decimal_bps,
        )

        # Convert to TZ-aware UTC DatetimeIndex (matches get_open_deviation_bars output)
        df["timestamp"] = pd.to_datetime(df["close_time_ms"], unit="ms", utc=True)
        df = df.set_index("timestamp")
        df = df.drop(columns=["close_time_ms"])

        # Convert PyArrow dtypes to numpy float64 for compatibility
        df = normalize_arrow_dtypes(df)

        if include_microstructure:
            df = normalize_arrow_dtypes(df, columns=list(MICROSTRUCTURE_COLUMNS))

        return df
