"""ClickHouse cache for computed range bars.

This module provides the RangeBarCache class that caches computed range bars
(Tier 2) in ClickHouse. Raw tick data is stored locally using Parquet files
via the `rangebar.storage` module.

Architecture:
- Tier 1 (raw ticks): Local Parquet files via `rangebar.storage.TickStorage`
- Tier 2 (computed bars): ClickHouse via this module

The cache uses mise environment variables for configuration and
supports multiple GPU workstations via SSH aliases.
"""

from __future__ import annotations

import hashlib
import logging
from dataclasses import dataclass
from importlib import resources
from typing import TYPE_CHECKING

import pandas as pd

from .._core import __version__
from ..constants import (
    EXCHANGE_SESSION_COLUMNS,
    MICROSTRUCTURE_COLUMNS,
    TRADE_ID_RANGE_COLUMNS,  # Issue #72
)
from ..conversion import normalize_arrow_dtypes
from ..exceptions import (
    CacheReadError,
    CacheWriteError,
)
from .bulk_operations import BulkStoreMixin, _build_insert_settings
from .client import get_client
from .mixin import ClickHouseClientMixin
from .preflight import (
    HostConnection,
    get_available_clickhouse_host,
)
from .query_operations import QueryOperationsMixin
from .tunnel import SSHTunnel

if TYPE_CHECKING:
    import clickhouse_connect

logger = logging.getLogger(__name__)

# Issue #90: Parallelize FINAL dedup across partitions for ~7x faster reads.
# Safe because our partitions (symbol, threshold, month) are independent.
_FINAL_READ_SETTINGS: dict[str, int] = {
    "do_not_merge_across_partitions_select_final": 1,
}


@dataclass(frozen=True)
class CacheKey:
    """Cache key for range bar lookups.

    Uniquely identifies a set of computed range bars based on:
    - Symbol (e.g., "BTCUSDT")
    - Threshold in decimal basis points (dbps)
    - Time range of source data
    - Ouroboros mode for reproducibility

    Attributes
    ----------
    symbol : str
        Trading symbol
    threshold_decimal_bps : int
        Threshold in decimal basis points (1 dbps = 0.001%)
    start_ts : int
        Start timestamp in milliseconds
    end_ts : int
        End timestamp in milliseconds
    ouroboros_mode : str
        Ouroboros reset mode: "year", "month", or "week" (default: "year")
    """

    symbol: str
    threshold_decimal_bps: int
    start_ts: int
    end_ts: int
    ouroboros_mode: str = "year"

    @property
    def hash_key(self) -> str:
        """Get hash key for cache lookups.

        Returns
        -------
        str
            MD5 hash of cache key components
        """
        key_str = (
            f"{self.symbol}_{self.threshold_decimal_bps}_"
            f"{self.start_ts}_{self.end_ts}_{self.ouroboros_mode}"
        )
        return hashlib.md5(key_str.encode()).hexdigest()


class RangeBarCache(ClickHouseClientMixin, BulkStoreMixin, QueryOperationsMixin):
    """ClickHouse cache for computed range bars.

    Caches computed range bars (Tier 2) in ClickHouse. For raw tick data
    storage (Tier 1), use `rangebar.storage.TickStorage` instead.

    PREFLIGHT runs in __init__ - fails loudly if no ClickHouse available.

    Parameters
    ----------
    client : Client | None
        External ClickHouse client. If None, creates connection based on
        environment configuration (mise env vars).

    Raises
    ------
    ClickHouseNotConfiguredError
        If no ClickHouse hosts available (with guidance for setup)

    Examples
    --------
    >>> with RangeBarCache() as cache:
    ...     # Check cache
    ...     if cache.has_range_bars(key):
    ...         bars = cache.get_range_bars(key)
    ...     else:
    ...         # Compute bars and store
    ...         cache.store_range_bars(key, bars)

    See Also
    --------
    rangebar.storage.TickStorage : Local Parquet storage for raw tick data
    """

    def __init__(self, client: clickhouse_connect.driver.Client | None = None) -> None:
        """Initialize cache with ClickHouse connection.

        Runs preflight check to find available ClickHouse host.
        Creates schema if it doesn't exist.
        """
        self._tunnel: SSHTunnel | None = None
        self._host_connection: HostConnection | None = None

        if client is None:
            # PREFLIGHT CHECK - runs before anything else
            # This function fails loudly if no CH available
            host_conn = get_available_clickhouse_host()
            self._host_connection = host_conn

            # Connect based on method (local, direct, or SSH tunnel)
            if host_conn.method == "local":
                self._init_client(get_client("localhost", host_conn.port))
            elif host_conn.method == "direct":
                from .preflight import _resolve_ssh_alias_to_ip

                ip = _resolve_ssh_alias_to_ip(host_conn.host)
                if ip is None:
                    msg = f"Could not resolve SSH alias: {host_conn.host}"
                    raise RuntimeError(msg)
                self._init_client(get_client(ip, host_conn.port))
            elif host_conn.method == "ssh_tunnel":
                self._tunnel = SSHTunnel(host_conn.host, host_conn.port)
                local_port = self._tunnel.start()
                self._init_client(get_client("localhost", local_port))
        else:
            self._init_client(client)

        self._ensure_schema()

    def close(self) -> None:
        """Close client and tunnel if owned."""
        super().close()
        if self._tunnel is not None:
            self._tunnel.stop()
            self._tunnel = None

    def _ensure_schema(self) -> None:
        """Create database and tables if they don't exist."""
        # Create database
        self.client.command("CREATE DATABASE IF NOT EXISTS rangebar_cache")

        # Load and execute schema
        schema_sql = resources.files(__package__).joinpath("schema.sql").read_text()

        # Split by semicolon and execute each statement
        for statement in schema_sql.split(";"):
            # Check if statement contains a CREATE TABLE/MATERIALIZED VIEW
            if "CREATE TABLE" in statement or "CREATE MATERIALIZED" in statement:
                # Strip single-line comments from each line, keep the SQL
                lines = []
                for line in statement.split("\n"):
                    line = line.strip()
                    # Skip pure comment lines
                    if line.startswith("--"):
                        continue
                    # Remove trailing comments
                    if "--" in line:
                        line = line[: line.index("--")].strip()
                    if line:
                        lines.append(line)
                clean_sql = " ".join(lines)
                if clean_sql:
                    self.client.command(clean_sql)

        # Issue #90: Enable INSERT block deduplication on existing tables.
        # ALTER TABLE MODIFY SETTING is idempotent — safe to run every init.
        try:
            self.client.command(
                "ALTER TABLE rangebar_cache.range_bars "
                "MODIFY SETTING non_replicated_deduplication_window = 1000"
            )
        except (OSError, RuntimeError) as e:
            # Graceful degradation: older ClickHouse may not support this setting.
            logger.debug("Could not set non_replicated_deduplication_window: %s", e)

    # =========================================================================
    # Range Bars Cache (Tier 2)
    # =========================================================================
    # Note: Raw tick data (Tier 1) is now stored locally using Parquet files.
    # See rangebar.storage.TickStorage for tick data caching.

    def store_range_bars(
        self,
        key: CacheKey,
        bars: pd.DataFrame,
        version: str | None = None,
    ) -> int:
        """Store computed range bars in cache.

        Parameters
        ----------
        key : CacheKey
            Cache key identifying these bars
        bars : pd.DataFrame
            DataFrame with OHLCV columns (from rangebar processing)
        version : str | None
            rangebar-core version for cache invalidation. If None (default),
            uses current package version for schema evolution tracking.

        Returns
        -------
        int
            Number of rows inserted

        Raises
        ------
        CacheWriteError
            If the insert operation fails.
        """
        if bars.empty:
            logger.debug("Skipping cache write for %s: empty DataFrame", key.symbol)
            return 0

        logger.debug(
            "Writing %d bars to cache for %s @ %d dbps",
            len(bars),
            key.symbol,
            key.threshold_decimal_bps,
        )

        df = bars.copy()

        # Handle DatetimeIndex
        if isinstance(df.index, pd.DatetimeIndex):
            df = df.reset_index()
            if "timestamp" in df.columns:
                df["timestamp_ms"] = df["timestamp"].dt.as_unit("ms").astype("int64")
                df = df.drop(columns=["timestamp"])
            elif "index" in df.columns:
                df["timestamp_ms"] = df["index"].dt.as_unit("ms").astype("int64")
                df = df.drop(columns=["index"])

        # Normalize column names (lowercase)
        df.columns = df.columns.str.lower()

        # Add cache metadata
        df["symbol"] = key.symbol
        df["threshold_decimal_bps"] = key.threshold_decimal_bps
        df["ouroboros_mode"] = key.ouroboros_mode
        df["cache_key"] = key.hash_key
        df["rangebar_version"] = version if version is not None else __version__
        df["source_start_ts"] = key.start_ts
        df["source_end_ts"] = key.end_ts

        # Select columns for insertion
        columns = [
            "symbol",
            "threshold_decimal_bps",
            "ouroboros_mode",
            "timestamp_ms",
            "open",
            "high",
            "low",
            "close",
            "volume",
            "cache_key",
            "rangebar_version",
            "source_start_ts",
            "source_end_ts",
        ]

        # Add optional microstructure columns if present (from constants.py SSoT)
        for col in MICROSTRUCTURE_COLUMNS:
            if col in df.columns:
                columns.append(col)

        # Add optional exchange session columns if present (Issue #8)
        for col in EXCHANGE_SESSION_COLUMNS:
            if col in df.columns:
                columns.append(col)

        # Add trade ID range columns if present (Issue #72)
        for col in TRADE_ID_RANGE_COLUMNS:
            if col in df.columns:
                columns.append(col)

        # Filter to existing columns
        columns = [c for c in columns if c in df.columns]

        # Issue #90: INSERT dedup token for idempotent writes.
        insert_settings = _build_insert_settings(
            False,
            (lambda: key.hash_key) if key.hash_key else None,
        )

        try:
            summary = self.client.insert_df(
                "rangebar_cache.range_bars",
                df[columns],
                settings=insert_settings,
            )
            written = summary.written_rows
            logger.info(
                "Cached %d bars for %s @ %d dbps",
                written,
                key.symbol,
                key.threshold_decimal_bps,
            )
            return written
        except (OSError, RuntimeError) as e:
            logger.exception(
                "Cache write failed for %s @ %d dbps",
                key.symbol,
                key.threshold_decimal_bps,
            )
            msg = f"Failed to write bars for {key.symbol}: {e}"
            raise CacheWriteError(
                msg,
                symbol=key.symbol,
                operation="write",
            ) from e

    def get_range_bars(self, key: CacheKey) -> pd.DataFrame | None:
        """Get cached range bars.

        Parameters
        ----------
        key : CacheKey
            Cache key to lookup

        Returns
        -------
        pd.DataFrame | None
            OHLCV DataFrame if found, None otherwise

        Raises
        ------
        CacheReadError
            If the query fails due to database errors.
        """
        query = """
            SELECT
                timestamp_ms,
                open as Open,
                high as High,
                low as Low,
                close as Close,
                volume as Volume
            FROM rangebar_cache.range_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold_decimal_bps:UInt32}
              AND source_start_ts = {start_ts:Int64}
              AND source_end_ts = {end_ts:Int64}
            ORDER BY timestamp_ms
        """
        try:
            # Use Arrow-optimized query for 3x faster DataFrame creation
            df = self.client.query_df_arrow(
                query,
                parameters={
                    "symbol": key.symbol,
                    "threshold_decimal_bps": key.threshold_decimal_bps,
                    "start_ts": key.start_ts,
                    "end_ts": key.end_ts,
                },
                settings=_FINAL_READ_SETTINGS,
            )
        except (OSError, RuntimeError) as e:
            logger.exception(
                "Cache read failed for %s @ %d dbps",
                key.symbol,
                key.threshold_decimal_bps,
            )
            msg = f"Failed to read bars for {key.symbol}: {e}"
            raise CacheReadError(
                msg,
                symbol=key.symbol,
                operation="read",
            ) from e

        if df.empty:
            logger.debug(
                "Cache miss for %s @ %d dbps (key: %s)",
                key.symbol,
                key.threshold_decimal_bps,
                key.hash_key[:8],
            )
            return None

        logger.debug(
            "Cache hit: %d bars for %s @ %d dbps",
            len(df),
            key.symbol,
            key.threshold_decimal_bps,
        )

        # Convert to TZ-aware UTC DatetimeIndex (Issue #20: consistent timestamps)
        df["timestamp"] = pd.to_datetime(df["timestamp_ms"], unit="ms", utc=True)
        df = df.set_index("timestamp")
        df = df.drop(columns=["timestamp_ms"])

        # Convert PyArrow dtypes to numpy for compatibility with fresh computation
        # query_df_arrow returns double[pyarrow], but process_trades returns float64
        df = normalize_arrow_dtypes(df)

        return df

    def has_range_bars(self, key: CacheKey) -> bool:
        """Check if range bars exist in cache.

        Parameters
        ----------
        key : CacheKey
            Cache key to check

        Returns
        -------
        bool
            True if bars exist
        """
        query = """
            SELECT count() > 0
            FROM rangebar_cache.range_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold_decimal_bps:UInt32}
              AND source_start_ts = {start_ts:Int64}
              AND source_end_ts = {end_ts:Int64}
            LIMIT 1
        """
        result = self.client.command(
            query,
            parameters={
                "symbol": key.symbol,
                "threshold_decimal_bps": key.threshold_decimal_bps,
                "start_ts": key.start_ts,
                "end_ts": key.end_ts,
            },
            settings=_FINAL_READ_SETTINGS,
        )
        return bool(result)

    def invalidate_range_bars(self, key: CacheKey) -> int:
        """Invalidate (delete) cached range bars.

        Parameters
        ----------
        key : CacheKey
            Cache key to invalidate

        Returns
        -------
        int
            Number of rows deleted
        """
        # Note: ClickHouse DELETE is async via mutations
        query = """
            ALTER TABLE rangebar_cache.range_bars
            DELETE WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold_decimal_bps:UInt32}
              AND source_start_ts = {start_ts:Int64}
              AND source_end_ts = {end_ts:Int64}
        """
        self.client.command(
            query,
            parameters={
                "symbol": key.symbol,
                "threshold_decimal_bps": key.threshold_decimal_bps,
                "start_ts": key.start_ts,
                "end_ts": key.end_ts,
            },
        )
        return 0  # ClickHouse DELETE is async, can't return count

    def invalidate_range_bars_by_range(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        start_timestamp_ms: int,
        end_timestamp_ms: int,
    ) -> int:
        """Invalidate (delete) cached bars within a timestamp range.

        Unlike `invalidate_range_bars()` which requires an exact CacheKey match,
        this method deletes all bars for a symbol/threshold within a time range.
        Useful for overlap detection during precomputation.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        start_timestamp_ms : int
            Start timestamp in milliseconds (inclusive)
        end_timestamp_ms : int
            End timestamp in milliseconds (inclusive)

        Returns
        -------
        int
            Always returns 0 (ClickHouse DELETE is async via mutations)

        Examples
        --------
        >>> cache.invalidate_range_bars_by_range(
        ...     "BTCUSDT", 250,
        ...     start_timestamp_ms=1704067200000,  # 2024-01-01
        ...     end_timestamp_ms=1706745600000,    # 2024-01-31
        ... )
        """
        query = """
            ALTER TABLE rangebar_cache.range_bars
            DELETE WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND timestamp_ms >= {start_ts:Int64}
              AND timestamp_ms <= {end_ts:Int64}
        """
        self.client.command(
            query,
            parameters={
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
                "start_ts": start_timestamp_ms,
                "end_ts": end_timestamp_ms,
            },
        )
        return 0  # ClickHouse DELETE is async, can't return count

    def get_last_bar_before(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        before_timestamp_ms: int,
    ) -> dict | None:
        """Get the last bar before a given timestamp.

        Useful for validating continuity at junction points during
        incremental precomputation.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        before_timestamp_ms : int
            Timestamp boundary in milliseconds

        Returns
        -------
        dict | None
            Last bar as dict with OHLCV keys, or None if no bars found

        Examples
        --------
        >>> bar = cache.get_last_bar_before("BTCUSDT", 250, 1704067200000)
        >>> if bar:
        ...     print(f"Last close: {bar['Close']}")
        """
        query = """
            SELECT timestamp_ms, open, high, low, close, volume
            FROM rangebar_cache.range_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND timestamp_ms < {before_ts:Int64}
            ORDER BY timestamp_ms DESC
            LIMIT 1
        """
        result = self.client.query(
            query,
            parameters={
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
                "before_ts": before_timestamp_ms,
            },
            settings=_FINAL_READ_SETTINGS,
        )

        rows = result.result_rows
        if not rows:
            return None

        row = rows[0]
        return {
            "timestamp_ms": row[0],
            "Open": row[1],
            "High": row[2],
            "Low": row[3],
            "Close": row[4],
            "Volume": row[5],
        }

    # =========================================================================
    # Bar-Count-Based API (get_n_range_bars support)
    # =========================================================================
    # These methods support retrieving a deterministic number of bars
    # regardless of time bounds. Uses split query paths to avoid OR conditions.

    def count_bars(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        before_ts: int | None = None,
    ) -> int:
        """Count available bars in cache.

        Uses split query paths to avoid OR conditions for ClickHouse optimization.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        before_ts : int | None
            Only count bars with timestamp_ms < before_ts.
            If None, counts all bars for symbol/threshold.

        Returns
        -------
        int
            Number of bars in cache
        """
        if before_ts is not None:
            # Split path: with end_ts filter
            query = """
                SELECT count()
                FROM rangebar_cache.range_bars FINAL
                WHERE symbol = {symbol:String}
                  AND threshold_decimal_bps = {threshold:UInt32}
                  AND timestamp_ms < {end_ts:Int64}
            """
            result = self.client.command(
                query,
                parameters={
                    "symbol": symbol,
                    "threshold": threshold_decimal_bps,
                    "end_ts": before_ts,
                },
                settings=_FINAL_READ_SETTINGS,
            )
        else:
            # Split path: no end_ts filter (most recent)
            query = """
                SELECT count()
                FROM rangebar_cache.range_bars FINAL
                WHERE symbol = {symbol:String}
                  AND threshold_decimal_bps = {threshold:UInt32}
            """
            result = self.client.command(
                query,
                parameters={
                    "symbol": symbol,
                    "threshold": threshold_decimal_bps,
                },
                settings=_FINAL_READ_SETTINGS,
            )
        return int(result) if result else 0


    def get_oldest_bar_timestamp(
        self,
        symbol: str,
        threshold_decimal_bps: int,
    ) -> int | None:
        """Get timestamp of oldest bar in cache.

        Useful for determining how far back we can query.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points

        Returns
        -------
        int | None
            Oldest bar timestamp in milliseconds, or None if no bars
        """
        query = """
            SELECT min(timestamp_ms)
            FROM rangebar_cache.range_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
        """
        result = self.client.command(
            query,
            parameters={
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
            },
            settings=_FINAL_READ_SETTINGS,
        )
        # ClickHouse returns 0 for min() on empty result
        return int(result) if result and result > 0 else None

    def get_newest_bar_timestamp(
        self,
        symbol: str,
        threshold_decimal_bps: int,
    ) -> int | None:
        """Get timestamp of newest bar in cache.

        Useful for determining the end of available data.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points

        Returns
        -------
        int | None
            Newest bar timestamp in milliseconds, or None if no bars
        """
        query = """
            SELECT max(timestamp_ms)
            FROM rangebar_cache.range_bars FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
        """
        result = self.client.command(
            query,
            parameters={
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
            },
            settings=_FINAL_READ_SETTINGS,
        )
        # ClickHouse returns 0 for max() on empty result
        return int(result) if result and result > 0 else None

    # =========================================================================
    # Population Checkpoints (Issue #69: Cross-machine resume)
    # =========================================================================
    # These methods support resumable cache population with bar-level accuracy.
    # Checkpoints are stored both locally and in ClickHouse for hybrid storage.

    def delete_bars(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        start_ts: int,
        end_ts: int,
    ) -> int:
        """Delete bars in timestamp range (for force_refresh).

        This is an alias for invalidate_range_bars_by_range() with clearer
        naming for the force_refresh use case.

        Parameters
        ----------
        symbol : str
            Trading symbol (e.g., "BTCUSDT")
        threshold_decimal_bps : int
            Threshold in decimal basis points
        start_ts : int
            Start timestamp in milliseconds (inclusive)
        end_ts : int
            End timestamp in milliseconds (inclusive)

        Returns
        -------
        int
            Always returns 0 (ClickHouse DELETE is async)
        """
        return self.invalidate_range_bars_by_range(
            symbol, threshold_decimal_bps, start_ts, end_ts
        )

    def save_checkpoint(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        start_date: str,
        end_date: str,
        last_completed_date: str,
        last_trade_timestamp_ms: int | None,
        processor_checkpoint: str,
        bars_written: int,
        include_microstructure: bool = False,
        ouroboros_mode: str = "year",
        # Issue #72: Full Audit Trail - agg_trade_id range in incomplete bar
        first_agg_trade_id_in_bar: int | None = None,
        last_agg_trade_id_in_bar: int | None = None,
    ) -> None:
        """Save population checkpoint to ClickHouse.

        Used for cross-machine resume support. Local checkpoints are
        saved separately via filesystem for faster same-machine resume.

        Parameters
        ----------
        symbol : str
            Trading symbol
        threshold_decimal_bps : int
            Threshold in decimal basis points
        start_date : str
            Population start date (YYYY-MM-DD)
        end_date : str
            Population end date (YYYY-MM-DD)
        last_completed_date : str
            Last fully processed date (YYYY-MM-DD)
        last_trade_timestamp_ms : int | None
            Timestamp of last processed trade (for mid-day resume)
        processor_checkpoint : str
            JSON-serialized processor state (incomplete bar, defer_open, etc.)
        bars_written : int
            Total bars written so far
        include_microstructure : bool
            Whether microstructure features are enabled
        ouroboros_mode : str
            Ouroboros reset mode
        """
        # Ensure table exists (in case schema wasn't updated)
        self._ensure_population_checkpoints_table()

        query = """
            INSERT INTO rangebar_cache.population_checkpoints
            (symbol, threshold_decimal_bps, start_date, end_date,
             last_completed_date, last_trade_timestamp_ms, bars_written,
             processor_checkpoint, include_microstructure, ouroboros_mode,
             first_agg_trade_id_in_bar, last_agg_trade_id_in_bar,
             updated_at)
            VALUES
            ({symbol:String}, {threshold:UInt32}, {start_date:String},
             {end_date:String}, {last_date:String}, {last_ts:Int64},
             {bars:UInt64}, {checkpoint:String}, {micro:UInt8},
             {ouroboros:String}, {first_agg:Int64}, {last_agg:Int64},
             now64(3))
        """
        self.client.command(
            query,
            parameters={
                "symbol": symbol,
                "threshold": threshold_decimal_bps,
                "start_date": start_date,
                "end_date": end_date,
                "last_date": last_completed_date,
                "last_ts": last_trade_timestamp_ms or 0,
                "bars": bars_written,
                "checkpoint": processor_checkpoint,
                "micro": 1 if include_microstructure else 0,
                "ouroboros": ouroboros_mode,
                "first_agg": first_agg_trade_id_in_bar or 0,  # Issue #72
                "last_agg": last_agg_trade_id_in_bar or 0,    # Issue #72
            },
        )
        logger.debug(
            "Saved checkpoint for %s @ %d dbps: %s (%d bars)",
            symbol,
            threshold_decimal_bps,
            last_completed_date,
            bars_written,
        )

    def load_checkpoint(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        start_date: str,
        end_date: str,
    ) -> dict | None:
        """Load population checkpoint from ClickHouse.

        Used for cross-machine resume when local checkpoint is not available.

        Parameters
        ----------
        symbol : str
            Trading symbol
        threshold_decimal_bps : int
            Threshold in decimal basis points
        start_date : str
            Population start date (YYYY-MM-DD)
        end_date : str
            Population end date (YYYY-MM-DD)

        Returns
        -------
        dict | None
            Checkpoint data if found, None otherwise.
            Keys: last_completed_date, last_trade_timestamp_ms,
                  processor_checkpoint, bars_written, include_microstructure,
                  ouroboros_mode
        """
        query = """
            SELECT
                last_completed_date,
                last_trade_timestamp_ms,
                processor_checkpoint,
                bars_written,
                include_microstructure,
                ouroboros_mode
            FROM rangebar_cache.population_checkpoints FINAL
            WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND start_date = {start_date:String}
              AND end_date = {end_date:String}
            LIMIT 1
        """
        try:
            result = self.client.query(
                query,
                parameters={
                    "symbol": symbol,
                    "threshold": threshold_decimal_bps,
                    "start_date": start_date,
                    "end_date": end_date,
                },
            )
        except (OSError, RuntimeError) as e:
            # Table might not exist yet
            logger.debug("Checkpoint load failed: %s", e)
            return None

        rows = result.result_rows
        if not rows:
            return None

        row = rows[0]
        return {
            "last_completed_date": row[0],
            "last_trade_timestamp_ms": row[1] if row[1] > 0 else None,
            "processor_checkpoint": row[2] if row[2] else None,
            "bars_written": row[3],
            "include_microstructure": bool(row[4]),
            "ouroboros_mode": row[5],
        }

    def delete_checkpoint(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        start_date: str,
        end_date: str,
    ) -> None:
        """Delete population checkpoint from ClickHouse.

        Used by force_refresh to clear checkpoint before repopulating.

        Parameters
        ----------
        symbol : str
            Trading symbol
        threshold_decimal_bps : int
            Threshold in decimal basis points
        start_date : str
            Population start date (YYYY-MM-DD)
        end_date : str
            Population end date (YYYY-MM-DD)
        """
        query = """
            ALTER TABLE rangebar_cache.population_checkpoints
            DELETE WHERE symbol = {symbol:String}
              AND threshold_decimal_bps = {threshold:UInt32}
              AND start_date = {start_date:String}
              AND end_date = {end_date:String}
        """
        try:
            self.client.command(
                query,
                parameters={
                    "symbol": symbol,
                    "threshold": threshold_decimal_bps,
                    "start_date": start_date,
                    "end_date": end_date,
                },
            )
            logger.debug(
                "Deleted checkpoint for %s @ %d dbps (%s to %s)",
                symbol,
                threshold_decimal_bps,
                start_date,
                end_date,
            )
        except (OSError, RuntimeError) as e:
            # Table might not exist yet - that's fine
            logger.debug("Checkpoint delete failed (table may not exist): %s", e)

    def _ensure_population_checkpoints_table(self) -> None:
        """Ensure population_checkpoints table exists.

        Called before checkpoint operations to handle upgrades from
        older installations without the table.
        """
        create_sql = """
            CREATE TABLE IF NOT EXISTS rangebar_cache.population_checkpoints (
                symbol LowCardinality(String),
                threshold_decimal_bps UInt32,
                start_date String,
                end_date String,
                last_completed_date String,
                last_trade_timestamp_ms Int64,
                bars_written UInt64,
                processor_checkpoint String DEFAULT '',
                first_agg_trade_id_in_bar Int64 DEFAULT 0,
                last_agg_trade_id_in_bar Int64 DEFAULT 0,
                include_microstructure UInt8 DEFAULT 0,
                ouroboros_mode LowCardinality(String) DEFAULT 'year',
                created_at DateTime64(3) DEFAULT now64(3),
                updated_at DateTime64(3) DEFAULT now64(3)
            )
            ENGINE = ReplacingMergeTree(updated_at)
            ORDER BY (symbol, threshold_decimal_bps, start_date, end_date)
        """
        try:
            self.client.command(create_sql)
        except (OSError, RuntimeError) as e:
            # Table already exists or other error - log and continue
            logger.debug("Checkpoints table creation: %s", e)

    # =========================================================================
    # Table Maintenance (Issue #77: Duplicate row handling)
    # =========================================================================
    # ReplacingMergeTree deduplicates rows during background merges, not at
    # insert time. This means duplicates can accumulate between merges.
    # Use these methods for maintenance and data quality checks.

    def count_duplicates(
        self,
        symbol: str | None = None,
        threshold_decimal_bps: int | None = None,
    ) -> list[dict]:
        """Count duplicate rows by timestamp (same ORDER BY key).

        ReplacingMergeTree deduplicates during background merges, not at insert
        time. This method identifies rows that will be deduplicated on next merge.

        Parameters
        ----------
        symbol : str | None
            Filter to specific symbol, or None for all symbols
        threshold_decimal_bps : int | None
            Filter to specific threshold, or None for all thresholds

        Returns
        -------
        list[dict]
            List of dicts with keys: symbol, threshold_decimal_bps, timestamp_ms,
            duplicate_count, opens, closes. Empty list if no duplicates found.

        Examples
        --------
        >>> with RangeBarCache() as cache:
        ...     dupes = cache.count_duplicates("BTCUSDT", 1000)
        ...     print(f"Found {len(dupes)} timestamp(s) with duplicates")
        """
        where_clauses = []
        params = {}

        if symbol is not None:
            where_clauses.append("symbol = {symbol:String}")
            params["symbol"] = symbol

        if threshold_decimal_bps is not None:
            where_clauses.append("threshold_decimal_bps = {threshold:UInt32}")
            params["threshold"] = threshold_decimal_bps

        where_sql = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else ""

        query = f"""
            SELECT
                symbol,
                threshold_decimal_bps,
                timestamp_ms,
                count(*) as duplicate_count,
                groupArray(open) as opens,
                groupArray(close) as closes
            FROM rangebar_cache.range_bars
            {where_sql}
            GROUP BY symbol, threshold_decimal_bps, timestamp_ms
            HAVING count(*) > 1
            ORDER BY symbol, threshold_decimal_bps, timestamp_ms
        """

        result = self.client.query(query, parameters=params)
        return [
            {
                "symbol": row[0],
                "threshold_decimal_bps": row[1],
                "timestamp_ms": row[2],
                "duplicate_count": row[3],
                "opens": row[4],
                "closes": row[5],
            }
            for row in result.result_rows
        ]

    def deduplicate_bars(
        self,
        symbol: str | None = None,
        threshold_decimal_bps: int | None = None,
        timeout: int = 600,
    ) -> None:
        """Force deduplication via OPTIMIZE TABLE FINAL with timeout resilience.

        Issue #90: Rewritten as fire-and-forget pattern. OPTIMIZE commands are
        submitted with short client-side timeouts. If the client times out,
        the merge continues server-side. Progress is monitored via system.merges.

        ReplacingMergeTree only deduplicates during background merges. This
        method forces an immediate merge to remove duplicate rows (same
        symbol + threshold + timestamp_ms), keeping the row with the latest
        computed_at timestamp.

        Parameters
        ----------
        symbol : str | None
            Optimize only partitions for this symbol, or None for all
        threshold_decimal_bps : int | None
            Optimize only partitions for this threshold, or None for all
        timeout : int
            Maximum seconds to wait for merges to complete (default: 600).
            After timeout, merges continue in background on the server.

        Examples
        --------
        >>> with RangeBarCache() as cache:
        ...     # Check for duplicates first
        ...     dupes = cache.count_duplicates("BTCUSDT", 1000)
        ...     if dupes:
        ...         print(f"Found {len(dupes)} duplicates, deduplicating...")
        ...         cache.deduplicate_bars("BTCUSDT", 1000)
        """
        if symbol is not None and threshold_decimal_bps is not None:
            logger.info(
                "Deduplicating bars for %s @ %d dbps (OPTIMIZE FINAL)",
                symbol,
                threshold_decimal_bps,
            )
            # Get list of partitions for this symbol/threshold
            partition_query = """
                SELECT DISTINCT partition
                FROM system.parts
                WHERE database = 'rangebar_cache'
                  AND table = 'range_bars'
                  AND active = 1
                  AND partition LIKE {pattern:String}
            """
            pattern = f"('{symbol}',{threshold_decimal_bps},%"
            result = self.client.query(partition_query, parameters={"pattern": pattern})

            for row in result.result_rows:
                partition_id = row[0]
                optimize_sql = (
                    f"OPTIMIZE TABLE rangebar_cache.range_bars "
                    f"PARTITION {partition_id} FINAL"
                )
                try:
                    self.client.command(
                        optimize_sql,
                        settings={
                            "send_timeout": 60,
                            "receive_timeout": 60,
                        },
                    )
                    logger.debug("Optimized partition %s", partition_id)
                except (OSError, RuntimeError) as e:
                    # Client timeout — OPTIMIZE continues server-side
                    logger.info(
                        "OPTIMIZE timed out client-side for partition %s, "
                        "merge continues on server: %s",
                        partition_id,
                        e,
                    )
        else:
            # Optimize entire table
            logger.info("Deduplicating all bars (OPTIMIZE TABLE FINAL)")
            try:
                self.client.command(
                    "OPTIMIZE TABLE rangebar_cache.range_bars FINAL",
                    settings={
                        "send_timeout": 60,
                        "receive_timeout": 60,
                    },
                )
            except (OSError, RuntimeError) as e:
                logger.info(
                    "Full-table OPTIMIZE timed out client-side, "
                    "merge continues on server: %s",
                    e,
                )

        # Poll system.merges until our table's merges complete
        self._wait_for_merges(timeout=timeout)
        logger.info("Deduplication complete")

    def _wait_for_merges(self, timeout: int = 600, poll_interval: int = 10) -> None:
        """Poll system.merges until no active merges for our table.

        Issue #90: After submitting OPTIMIZE, poll ClickHouse to wait for
        background merges to finish. If timeout is reached, log a warning
        and return — merges continue server-side.

        Parameters
        ----------
        timeout : int
            Maximum seconds to wait (default: 600).
        poll_interval : int
            Seconds between polls (default: 10).
        """
        import time

        deadline = time.monotonic() + timeout
        while time.monotonic() < deadline:
            result = self.client.query(
                "SELECT count() FROM system.merges "
                "WHERE database = 'rangebar_cache' AND table = 'range_bars'"
            )
            active = result.result_rows[0][0] if result.result_rows else 0
            if active == 0:
                logger.info("All merges complete")
                return
            logger.debug("Waiting for %d active merge(s)...", active)
            time.sleep(poll_interval)
        logger.warning(
            "Merge wait timed out after %ds, merges continue in background",
            timeout,
        )

