# FILE-SIZE-OK: 630 lines after mixin extraction (was 1258).
# Remaining methods are tightly coupled to CacheKey and
# OpenDeviationBarCache __init__/schema lifecycle.
"""ClickHouse cache for computed open deviation bars.

This module provides the OpenDeviationBarCache class that
caches computed open deviation bars
(Tier 2) in ClickHouse. Raw tick data is stored locally using Parquet files
via the `opendeviationbar.storage` module.

Architecture:
- Tier 1 (raw ticks): Local Parquet files via `opendeviationbar.storage.TickStorage`
- Tier 2 (computed bars): ClickHouse via this module

The cache uses mise environment variables for configuration and
supports multiple GPU workstations via SSH aliases.
"""

from __future__ import annotations

import hashlib
import logging
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING

import pandas as pd

from .._core import __version__
from ..constants import (
    EXCHANGE_SESSION_COLUMNS,
    MICROSTRUCTURE_COLUMNS,
    TIMESTAMP_COLUMNS,
    TRADE_ID_RANGE_COLUMNS,  # Issue #72
)
from ..conversion import normalize_arrow_dtypes
from ..exceptions import (
    CacheReadError,
    CacheWriteError,
)
from .bar_count_api import BarCountMixin
from .bulk_operations import BulkStoreMixin, _build_insert_settings
from .checkpoint_operations import CheckpointOperationsMixin
from .client import get_client
from .constants import CLICKHOUSE_TRANSIENT_ERRORS, FINAL_READ_SETTINGS
from .maintenance import MaintenanceMixin
from .mixin import ClickHouseClientMixin
from .preflight import (
    HostConnection,
    get_available_clickhouse_host,
)
from .query_operations import QueryOperationsMixin
from .tunnel import SSHTunnel

if TYPE_CHECKING:
    import clickhouse_connect

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class CacheKey:
    """Cache key for open deviation bar lookups.

    Uniquely identifies a set of computed open deviation bars based on:
    - Symbol (e.g., "BTCUSDT")
    - Threshold in decimal basis points (dbps)
    - Time range of source data
    - Ouroboros mode for reproducibility

    Attributes
    ----------
    symbol : str
        Trading symbol
    threshold_decimal_bps : int
        Threshold in decimal basis points (1 dbps = 0.001%)
    start_ts : int
        Start timestamp in milliseconds
    end_ts : int
        End timestamp in milliseconds
    ouroboros_mode : str
        Ouroboros reset mode (default: "aion").
    """

    symbol: str
    threshold_decimal_bps: int
    start_ts: int
    end_ts: int
    ouroboros_mode: str = "aion"

    @property
    def hash_key(self) -> str:
        """Get hash key for cache lookups.

        Returns
        -------
        str
            MD5 hash of cache key components
        """
        key_str = (
            f"{self.symbol}_{self.threshold_decimal_bps}_"
            f"{self.start_ts}_{self.end_ts}_{self.ouroboros_mode}"
        )
        return hashlib.md5(key_str.encode()).hexdigest()


class OpenDeviationBarCache(
    ClickHouseClientMixin,
    BulkStoreMixin,
    QueryOperationsMixin,
    CheckpointOperationsMixin,
    MaintenanceMixin,
    BarCountMixin,
):
    """ClickHouse cache for computed open deviation bars.

    Caches computed open deviation bars (Tier 2) in ClickHouse. For raw tick data
    storage (Tier 1), use `opendeviationbar.storage.TickStorage` instead.

    PREFLIGHT runs in __init__ - fails loudly if no ClickHouse available.

    Parameters
    ----------
    client : Client | None
        External ClickHouse client. If None, creates connection based on
        environment configuration (mise env vars).

    Raises
    ------
    ClickHouseNotConfiguredError
        If no ClickHouse hosts available (with guidance for setup)

    Examples
    --------
    >>> with OpenDeviationBarCache() as cache:
    ...     # Check cache
    ...     if cache.has_open_deviation_bars(key):
    ...         bars = cache.get_open_deviation_bars(key)
    ...     else:
    ...         # Compute bars and store
    ...         cache.store_open_deviation_bars(key, bars)

    See Also
    --------
    opendeviationbar.storage.TickStorage : Local Parquet storage for raw tick data
    """

    def __init__(self, client: clickhouse_connect.driver.Client | None = None) -> None:
        """Initialize cache with ClickHouse connection.

        Runs preflight check to find available ClickHouse host.
        Creates schema if it doesn't exist.
        """
        self._tunnel: SSHTunnel | None = None
        self._host_connection: HostConnection | None = None

        if client is None:
            # PREFLIGHT CHECK - runs before anything else
            # This function fails loudly if no CH available
            host_conn = get_available_clickhouse_host()
            self._host_connection = host_conn

            # Connect based on method (local, direct, or SSH tunnel)
            if host_conn.method == "local":
                self._init_client(get_client("localhost", host_conn.port))
            elif host_conn.method == "direct":
                from .preflight import _resolve_ssh_alias_to_ip

                ip = _resolve_ssh_alias_to_ip(host_conn.host)
                if ip is None:
                    msg = f"Could not resolve SSH alias: {host_conn.host}"
                    raise RuntimeError(msg)
                self._init_client(get_client(ip, host_conn.port))
            elif host_conn.method == "ssh_tunnel":
                self._tunnel = SSHTunnel(host_conn.host, host_conn.port)
                local_port = self._tunnel.start()
                self._init_client(get_client("localhost", local_port))
        else:
            self._init_client(client)

        self._ensure_schema()

    def close(self) -> None:
        """Close client and tunnel if owned."""
        super().close()
        if self._tunnel is not None:
            self._tunnel.stop()
            self._tunnel = None

    def _ensure_schema(self) -> None:
        """Create database and tables if they don't exist."""
        # Create database
        self.client.command("CREATE DATABASE IF NOT EXISTS opendeviationbar_cache")

        # Load schema.sql — try co-located path first (wheel installs),
        # fall back to importlib.resources.files() for maturin develop
        # non-editable installs where .sql files may not be copied to site-packages.
        # NOTE: the old importlib.resources.read_text() API had systemd path issues;
        # importlib.resources.files() (Python 3.9+) is reliable in all environments.
        _schema_path = Path(__file__).parent / "schema.sql"
        if _schema_path.exists():
            schema_sql = _schema_path.read_text()
        else:
            from importlib.resources import files as _res_files

            schema_sql = (
                _res_files("opendeviationbar.clickhouse")
                .joinpath("schema.sql")
                .read_text(encoding="utf-8")
            )

        # Split by semicolon and execute each statement
        for statement in schema_sql.split(";"):
            # Check if statement contains a CREATE TABLE/MATERIALIZED VIEW
            if "CREATE TABLE" in statement or "CREATE MATERIALIZED" in statement:
                # Strip single-line comments from each line, keep the SQL
                lines = []
                for line in statement.split("\n"):
                    line = line.strip()
                    # Skip pure comment lines
                    if line.startswith("--"):
                        continue
                    # Remove trailing comments
                    if "--" in line:
                        line = line[: line.index("--")].strip()
                    if line:
                        lines.append(line)
                clean_sql = " ".join(lines)
                if clean_sql:
                    self.client.command(clean_sql)

        self._migrate_table_settings()
        self._migrate_auxiliary_tables()

    def _migrate_table_settings(self) -> None:
        """Apply idempotent ALTER TABLE migrations to open_deviation_bars."""
        tbl = "opendeviationbar_cache.open_deviation_bars"

        # Issue #90: INSERT block deduplication.
        # Issue #164: deduplicate_merge_projection_mode for prj_time_range.
        try:
            self.client.command(
                f"ALTER TABLE {tbl} MODIFY SETTING "
                "non_replicated_deduplication_window = 1000, "
                "deduplicate_merge_projection_mode = 'drop'"
            )
        except Exception as e:
            logger.debug("Could not modify table settings: %s", e)

        # Issue #158 Phase 7: Skip index for close_time_us range queries.
        try:
            self.client.command(
                f"ALTER TABLE {tbl} "
                "ADD INDEX IF NOT EXISTS idx_close_time close_time_us "
                "TYPE minmax GRANULARITY 8192"
            )
        except Exception as e:
            logger.debug("Could not add close_time_us skip index: %s", e)

        # Detect legacy schema and conditionally add open_time_us
        try:
            result = self.client.query(
                "SELECT name FROM system.columns "
                "WHERE database='opendeviationbar_cache' "
                "AND table='open_deviation_bars' AND name='timestamp_ms'"
            )
            if result.result_rows:
                logger.warning(
                    "Legacy schema detected: 'timestamp_ms' column present. "
                    "Run migration script to rename to 'close_time_us'. "
                    "See scripts/migrate_column_rename_us.py"
                )
            else:
                self.client.command(
                    f"ALTER TABLE {tbl} "
                    "ADD COLUMN IF NOT EXISTS open_time_us Int64 "
                    "DEFAULT close_time_us - duration_us "
                    "AFTER close_time_us"
                )
        except Exception as e:
            logger.debug("Could not check/add open_time_us column: %s", e)

        # Issue #296: is_orphan column for midnight preflight verification.
        # Orphan bars (incomplete bars emitted at UTC midnight reset) need
        # this flag for get_yesterday_orphan_tid() queries.
        try:
            self.client.command(
                f"ALTER TABLE {tbl} "
                "ADD COLUMN IF NOT EXISTS is_orphan UInt8 DEFAULT 0"
            )
        except Exception as e:
            logger.debug("Could not add is_orphan column: %s", e)

        # Issue #164: Time-range projection for flowsurface queries.
        try:
            self.client.command(
                f"ALTER TABLE {tbl} "
                "ADD PROJECTION IF NOT EXISTS prj_time_range "
                "(SELECT * ORDER BY (symbol, threshold_decimal_bps, close_time_us))"
            )
        except Exception as e:
            logger.debug("Could not add prj_time_range projection: %s", e)

    def _migrate_auxiliary_tables(self) -> None:
        """Create auxiliary tables and run plugin migrations."""
        # Issue #163: Data freshness tracking table for SSE push endpoint.
        try:
            self.client.command(
                "CREATE TABLE IF NOT EXISTS opendeviationbar_cache.data_freshness ("
                "    symbol LowCardinality(String),"
                "    threshold_decimal_bps UInt32,"
                "    last_close_time_us Int64,"
                "    last_bar_count UInt64 DEFAULT 0,"
                "    updated_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')"
                ") ENGINE = ReplacingMergeTree(updated_at)"
                " ORDER BY (symbol, threshold_decimal_bps)"
            )
        except Exception as e:
            logger.debug("Could not create data_freshness table: %s", e)

        # Issue #98: Auto-migrate plugin feature columns.
        try:
            from ..plugins.loader import _get_providers
            from ..plugins.migration import migrate_plugin_columns

            providers = _get_providers()
            if providers:
                migrate_plugin_columns(self.client, providers)
        except (ImportError, OSError, RuntimeError) as e:
            logger.debug("Plugin column migration skipped: %s", e)

    # =========================================================================
    # Open Deviation Bars Cache (Tier 2)
    # =========================================================================
    # Note: Raw tick data (Tier 1) is now stored locally using Parquet files.
    # See opendeviationbar.storage.TickStorage for tick data caching.

    def store_open_deviation_bars(
        self,
        key: CacheKey,
        bars: pd.DataFrame,
        version: str | None = None,
    ) -> int:
        """Store computed open deviation bars in cache.

        Parameters
        ----------
        key : CacheKey
            Cache key identifying these bars
        bars : pd.DataFrame
            DataFrame with OHLCV columns (from opendeviationbar processing)
        version : str | None
            opendeviationbar-core version for cache invalidation. If None (default),
            uses current package version for schema evolution tracking.

        Returns
        -------
        int
            Number of rows inserted

        Raises
        ------
        CacheWriteError
            If the insert operation fails.
        """
        if bars.empty:
            logger.debug("Skipping cache write for %s: empty DataFrame", key.symbol)
            return 0

        # Issue #256: Reject writes below symbol's registry min_threshold
        from .bulk_operations import _guard_threshold_registry

        _guard_threshold_registry(key.symbol, key.threshold_decimal_bps)

        logger.debug(
            "Writing %d bars to cache for %s @ %d dbps",
            len(bars),
            key.symbol,
            key.threshold_decimal_bps,
        )

        # Memory v2: reset_index() already creates a new DataFrame (matches bulk_operations.py)
        if isinstance(bars.index, pd.DatetimeIndex):
            df = bars.reset_index()
            # After reset_index(), the DatetimeIndex becomes a column
            idx_col = bars.index.name or "index"
            df["close_time_us"] = df[idx_col].dt.as_unit("us").astype("int64")
            df = df.drop(columns=[idx_col], errors="ignore")
        elif "close_time_us" in bars.columns:
            df = bars.copy()
        else:
            msg = "DataFrame must have DatetimeIndex or 'close_time_us' column"
            raise ValueError(msg)

        # Normalize column names (lowercase)
        df.columns = df.columns.str.lower()

        # Add cache metadata
        df["symbol"] = key.symbol
        df["threshold_decimal_bps"] = key.threshold_decimal_bps
        df["ouroboros_mode"] = key.ouroboros_mode
        df["cache_key"] = key.hash_key
        df["opendeviationbar_version"] = version if version is not None else __version__
        df["source_start_ts"] = key.start_ts
        df["source_end_ts"] = key.end_ts

        # Select columns for insertion
        columns = [
            "symbol",
            "threshold_decimal_bps",
            "ouroboros_mode",
            "close_time_us",
            "open",
            "high",
            "low",
            "close",
            "volume",
            "cache_key",
            "opendeviationbar_version",
            "source_start_ts",
            "source_end_ts",
        ]

        # Add optional columns if present (from constants.py SSoT)
        for col in (
            *MICROSTRUCTURE_COLUMNS,
            *EXCHANGE_SESSION_COLUMNS,
            *TRADE_ID_RANGE_COLUMNS,
            *TIMESTAMP_COLUMNS,
        ):
            if col in df.columns and col not in columns:
                columns.append(col)

        # Filter to existing columns
        columns = [c for c in columns if c in df.columns]

        # Issue #90: INSERT dedup token for idempotent writes.
        insert_settings = _build_insert_settings(
            False,
            (lambda: key.hash_key) if key.hash_key else None,
        )

        try:
            summary = self.client.insert_df(
                "opendeviationbar_cache.open_deviation_bars",
                df[columns],
                settings=insert_settings,
            )
            written = summary.written_rows
            logger.info(
                "Cached %d bars for %s @ %d dbps",
                written,
                key.symbol,
                key.threshold_decimal_bps,
            )
            return written
        except CLICKHOUSE_TRANSIENT_ERRORS as e:
            logger.exception(
                "Cache write failed for %s @ %d dbps",
                key.symbol,
                key.threshold_decimal_bps,
            )
            msg = f"Failed to write bars for {key.symbol}: {e}"
            raise CacheWriteError(
                msg,
                symbol=key.symbol,
                operation="write",
            ) from e

    def get_open_deviation_bars(self, key: CacheKey) -> pd.DataFrame | None:
        """Get cached open deviation bars.

        Parameters
        ----------
        key : CacheKey
            Cache key to lookup

        Returns
        -------
        pd.DataFrame | None
            OHLCV DataFrame if found, None otherwise

        Raises
        ------
        CacheReadError
            If the query fails due to database errors.
        """
        query = """
            SELECT
                close_time_us,
                open as Open,
                high as High,
                low as Low,
                close as Close,
                volume as Volume
            FROM opendeviationbar_cache.open_deviation_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold_decimal_bps:UInt32}
              AND source_start_ts = {start_ts:Int64}
              AND source_end_ts = {end_ts:Int64}
            ORDER BY close_time_us
        """
        try:
            # Use Arrow-optimized query for 3x faster DataFrame creation
            df = self.client.query_df_arrow(
                query,
                parameters={
                    "symbol": key.symbol,
                    "threshold_decimal_bps": key.threshold_decimal_bps,
                    "start_ts": key.start_ts,
                    "end_ts": key.end_ts,
                },
                settings=FINAL_READ_SETTINGS,
            )
        except CLICKHOUSE_TRANSIENT_ERRORS as e:
            logger.exception(
                "Cache read failed for %s @ %d dbps",
                key.symbol,
                key.threshold_decimal_bps,
            )
            msg = f"Failed to read bars for {key.symbol}: {e}"
            raise CacheReadError(
                msg,
                symbol=key.symbol,
                operation="read",
            ) from e

        if df.empty:
            logger.debug(
                "Cache miss for %s @ %d dbps (key: %s)",
                key.symbol,
                key.threshold_decimal_bps,
                key.hash_key[:8],
            )
            return None

        logger.debug(
            "Cache hit: %d bars for %s @ %d dbps",
            len(df),
            key.symbol,
            key.threshold_decimal_bps,
        )

        # Convert to TZ-aware UTC DatetimeIndex (Issue #20: consistent timestamps)
        df["timestamp"] = pd.to_datetime(df["close_time_us"].values, unit="us", utc=True)
        df = df.set_index("timestamp")
        df = df.drop(columns=["close_time_us"])

        # Convert PyArrow dtypes to numpy for compatibility with fresh computation
        # query_df_arrow returns double[pyarrow], but process_trades returns float64
        df = normalize_arrow_dtypes(df)

        return df

    def has_open_deviation_bars(self, key: CacheKey) -> bool:
        """Check if open deviation bars exist in cache.

        Parameters
        ----------
        key : CacheKey
            Cache key to check

        Returns
        -------
        bool
            True if bars exist
        """
        query = """
            SELECT count() > 0
            FROM opendeviationbar_cache.open_deviation_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold_decimal_bps:UInt32}
              AND source_start_ts = {start_ts:Int64}
              AND source_end_ts = {end_ts:Int64}
            LIMIT 1
        """
        result = self.client.command(
            query,
            parameters={
                "symbol": key.symbol,
                "threshold_decimal_bps": key.threshold_decimal_bps,
                "start_ts": key.start_ts,
                "end_ts": key.end_ts,
            },
            settings=FINAL_READ_SETTINGS,
        )
        return bool(result)

    def invalidate_open_deviation_bars(self, key: CacheKey) -> int:
        """Invalidate (delete) cached open deviation bars.

        Parameters
        ----------
        key : CacheKey
            Cache key to invalidate

        Returns
        -------
        int
            Number of rows deleted
        """
        # Note: ClickHouse DELETE is async via mutations
        query = """
            ALTER TABLE opendeviationbar_cache.open_deviation_bars
            DELETE WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold_decimal_bps:UInt32}
              AND source_start_ts = {start_ts:Int64}
              AND source_end_ts = {end_ts:Int64}
        """
        self.client.command(
            query,
            parameters={
                "symbol": key.symbol,
                "threshold_decimal_bps": key.threshold_decimal_bps,
                "start_ts": key.start_ts,
                "end_ts": key.end_ts,
            },
        )
        return 0  # ClickHouse DELETE is async, can't return count

    def invalidate_open_deviation_bars_by_range(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        start_timestamp_ms: int,
        end_timestamp_ms: int,
        *,
        ouroboros_mode: str,  # Issue #126: MANDATORY — prevent cross-mode deletion
    ) -> int:
        """Invalidate (delete) cached bars within a timestamp range.

        Unlike `invalidate_open_deviation_bars()` which requires
        an exact CacheKey match,
        this method deletes all bars for a symbol/threshold within a time range.
        Useful for overlap detection during precomputation.

        Issue #126: ouroboros_mode is mandatory to prevent cross-mode data
        destruction during force_refresh operations.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        start_timestamp_ms : int
            Start timestamp in milliseconds (inclusive)
        end_timestamp_ms : int
            End timestamp in milliseconds (inclusive)
        ouroboros_mode : str
            Ouroboros mode filter — only delete bars with this mode.

        Returns
        -------
        int
            Always returns 0 (ClickHouse DELETE is async via mutations)

        Examples
        --------
        >>> cache.invalidate_open_deviation_bars_by_range(
        ...     "BTCUSDT", 250,
        ...     start_timestamp_ms=1704067200000,  # 2024-01-01
        ...     end_timestamp_ms=1706745600000,    # 2024-01-31
        ...     ouroboros_mode="aion",
        ... )
        """
        query = """
            ALTER TABLE opendeviationbar_cache.open_deviation_bars
            DELETE WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND ouroboros_mode = {ouroboros:String}
              AND close_time_us >= {start_ts:Int64}
              AND close_time_us <= {end_ts:Int64}
        """
        self.client.command(
            query,
            parameters={
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
                "ouroboros": ouroboros_mode,
                "start_ts": start_timestamp_ms,
                "end_ts": end_timestamp_ms,
            },
        )
        return 0  # ClickHouse DELETE is async, can't return count
