# Issue #46: Modularization - Extract maintenance operations from cache.py
# Issue #158: Stathera — store_bars_replacing, _wait_for_mutations
"""Table maintenance operations for ClickHouse open deviation bar cache.

Provides mixin methods for deduplication and merge monitoring.
Used by OpenDeviationBarCache via mixin inheritance.
"""

from __future__ import annotations

import datetime as _dt
import logging
import threading
from pathlib import Path as _Path
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from opendeviationbar.config.network import NetworkConfig

logger = logging.getLogger(__name__)

_DEAD_LETTER_DIR = _Path("/tmp/opendeviationbar-dead-letter")

# Issue #296: Per-partition locks serializing _do_replace() calls.
# ClickHouse mutations within the SAME partition must be serialized
# (concurrent mutations_sync=1 DELETEs queue up). Different partitions
# can mutate concurrently — keyed by (symbol, threshold_decimal_bps).
_replace_locks: dict[tuple[str, int], threading.Lock] = {}
_replace_locks_guard = threading.Lock()


def _get_partition_lock(symbol: str, threshold_decimal_bps: int) -> threading.Lock:
    """Return a per-partition lock for the given (symbol, threshold) pair.

    Different partitions get independent locks (concurrent mutation).
    Same partition shares one lock (serialized mutation).
    """
    key = (symbol, threshold_decimal_bps)
    lock = _replace_locks.get(key)
    if lock is not None:
        return lock
    with _replace_locks_guard:
        return _replace_locks.setdefault(key, threading.Lock())


def _dead_letter_replace_failure(
    symbol: str,
    threshold_decimal_bps: int,
    bars_df: object,
    new_first: int,
    new_last: int,
) -> None:
    """Save bars to dead-letter Parquet after INSERT failure in _do_replace.

    Uses the same directory as Niffler/Charon (/tmp/opendeviationbar-dead-letter/),
    so Charon's 5-min replay loop will pick these up automatically.

    #295 P0-2: DELETE succeeded but INSERT failed = bars permanently lost
    without this safety net.
    """
    try:
        _DEAD_LETTER_DIR.mkdir(parents=True, exist_ok=True)
        ts = _dt.datetime.now(_dt.UTC).strftime("%Y%m%dT%H%M%S")
        filename = f"{symbol}@{threshold_decimal_bps}_{new_first}_{new_last}_{ts}.parquet"
        path = _DEAD_LETTER_DIR / filename

        import polars as pl
        if isinstance(bars_df, pl.DataFrame):
            bars_df.write_parquet(path)
        else:
            # pandas DataFrame
            import pandas as pd
            if isinstance(bars_df, pd.DataFrame):
                bars_df.to_parquet(path, engine="pyarrow")
            else:
                logger.error(
                    "dead_letter_replace: unknown DataFrame type %s, cannot save",
                    type(bars_df).__name__,
                )
                return

        logger.critical(
            "dead_letter_replace: saved %d bars to %s (DELETE succeeded, INSERT failed)",
            len(bars_df), path,
        )
    except Exception:
        logger.exception("dead_letter_replace: failed to save dead-letter file")


def _net_cfg() -> NetworkConfig:
    from opendeviationbar.config.network import NetworkConfig
    return NetworkConfig()


class MaintenanceMixin:
    """Mixin for table maintenance operations (dedup, merge monitoring).

    Requires `self.client` from ClickHouseClientMixin.
    """

    def deduplicate_bars(
        self,
        symbol: str | None = None,
        threshold_decimal_bps: int | None = None,
        timeout: int = 600,
    ) -> None:
        """Force deduplication via OPTIMIZE TABLE FINAL with timeout resilience.

        Issue #90: Rewritten as fire-and-forget pattern. OPTIMIZE commands are
        submitted with short client-side timeouts. If the client times out,
        the merge continues server-side. Progress is monitored via system.merges.

        ReplacingMergeTree only deduplicates during background merges. This
        method forces an immediate merge to remove duplicate rows (same
        symbol + threshold + close_time_us), keeping the row with the latest
        computed_at timestamp.

        Parameters
        ----------
        symbol : str | None
            Optimize only partitions for this symbol, or None for all
        threshold_decimal_bps : int | None
            Optimize only partitions for this threshold, or None for all
        timeout : int
            Maximum seconds to wait for merges to complete (default: 600).
            After timeout, merges continue in background on the server.

        Examples
        --------
        >>> with OpenDeviationBarCache() as cache:
        ...     # Check for duplicates first
        ...     dupes = cache.count_duplicates("BTCUSDT", 1000)
        ...     if dupes:
        ...         print(f"Found {len(dupes)} duplicates, deduplicating...")
        ...         cache.deduplicate_bars("BTCUSDT", 1000)
        """
        if symbol is not None and threshold_decimal_bps is not None:
            logger.info(
                "Deduplicating bars for %s @ %d dbps (OPTIMIZE FINAL)",
                symbol,
                threshold_decimal_bps,
            )
            # Get list of partitions for this symbol/threshold.
            # Partition key is (symbol, threshold_decimal_bps) — 2-tuple, no date.
            # system.parts.partition shows e.g. "('BTCUSDT',250)".
            # With both symbol+threshold specified, there is exactly one partition.
            partition_query = """
                SELECT DISTINCT partition
                FROM system.parts
                WHERE database = 'opendeviationbar_cache'
                  AND table = 'open_deviation_bars'
                  AND active = 1
                  AND partition = {partition_id:String}
            """
            partition_id = f"('{symbol}',{threshold_decimal_bps})"
            result = self.client.query(partition_query, parameters={"partition_id": partition_id})

            for row in result.result_rows:
                part_id = row[0]
                optimize_sql = (
                    f"OPTIMIZE TABLE opendeviationbar_cache.open_deviation_bars "
                    f"PARTITION {part_id} FINAL"
                )
                try:
                    self.client.command(
                        optimize_sql,
                        settings={
                            "send_timeout": 60,
                            "receive_timeout": 60,
                        },
                    )
                    logger.debug("Optimized partition %s", part_id)
                except Exception as e:
                    # Client timeout — OPTIMIZE continues server-side
                    # Catches OSError, RuntimeError, and OperationalError
                    # from clickhouse-connect HTTP timeout (read timeout=300s)
                    logger.info(
                        "OPTIMIZE timed out client-side for partition %s, "
                        "merge continues on server: %s",
                        part_id,
                        e,
                    )
        else:
            # Optimize entire table
            logger.info("Deduplicating all bars (OPTIMIZE TABLE FINAL)")
            try:
                self.client.command(
                    "OPTIMIZE TABLE opendeviationbar_cache.open_deviation_bars FINAL",
                    settings={
                        "send_timeout": 60,
                        "receive_timeout": 60,
                    },
                )
            except Exception as e:
                logger.info(
                    "Full-table OPTIMIZE timed out client-side, "
                    "merge continues on server: %s",
                    e,
                )

        # Poll system.merges until our table's merges complete
        self._wait_for_merges(timeout=timeout)
        logger.info("Deduplication complete")

    def store_bars_replacing(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        bars_df: object,
        ouroboros_mode: str | None = None,  # SSoT-OK: resolved from config
        version: str | None = None,
        _skip_lock: bool = False,
    ) -> int:
        """INSERT-only replace using RMT(computed_at) native dedup. Inserts new bars first, then detects and cleans up any straddle bars with targeted single-row DELETEs.

        Issue #158: Stathera — used by kintsugi repairs where the writer is
        authoritative. INSERT-first strategy means crash at any point = no data
        loss (worst case: temporary Stathera overlap that kintsugi repairs).

        Issue #158 Phase 7: Wraps the full INSERT+cleanup in a Redis write lock
        to prevent concurrent writers from interleaving. The inner
        store_bars_batch call uses _skip_lock=True to avoid deadlock.

        Parameters
        ----------
        symbol : str
            Trading symbol
        threshold_decimal_bps : int
            Threshold in decimal basis points
        bars_df : polars.DataFrame
            New bars to insert (must have first_agg_trade_id, last_agg_trade_id)
        ouroboros_mode : str | None
            Ouroboros mode. Resolved from config if None.
        version : str | None
            Version string for source tagging.
        _skip_lock : bool
            If True, skip Redis lock acquisition. Used when caller already
            holds cache_write_lock (e.g. kintsugi _repair_under_lock).

        Returns
        -------
        int
            Number of rows inserted.
        """
        import polars as pl

        if ouroboros_mode is None:
            from opendeviationbar.ouroboros import get_operational_ouroboros_mode

            ouroboros_mode = get_operational_ouroboros_mode()

        if not isinstance(bars_df, pl.DataFrame) or bars_df.is_empty():
            return 0

        # Get trade-ID range of new bars
        new_first = bars_df["first_agg_trade_id"].min()
        new_last = bars_df["last_agg_trade_id"].max()

        if new_first is None or new_last is None or new_first <= 0:
            # No valid trade IDs — fall through to normal insert
            return self.store_bars_batch(
                symbol, threshold_decimal_bps, bars_df,
                version=version, ouroboros_mode=ouroboros_mode,
                _skip_lock=_skip_lock,
            )

        return self._store_bars_replacing_inner(
            symbol, threshold_decimal_bps, bars_df,
            ouroboros_mode=ouroboros_mode, version=version,
            new_first=new_first, new_last=new_last,
            _skip_lock=_skip_lock,
        )

    def _store_bars_replacing_inner(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        bars_df: object,
        *,
        ouroboros_mode: str,
        version: str | None,
        new_first: int,
        new_last: int,
        _skip_lock: bool,
    ) -> int:
        """Inner replace logic — INSERT new bars, then targeted straddle cleanup.

        When _skip_lock=False, acquires Redis write lock for the full
        INSERT+cleanup sequence. When _skip_lock=True, caller already
        holds the lock (e.g. kintsugi _repair_under_lock).
        """
        if _skip_lock:
            return self._do_replace(
                symbol, threshold_decimal_bps, bars_df,
                ouroboros_mode=ouroboros_mode, version=version,
                new_first=new_first, new_last=new_last,
            )

        # Issue #158 Phase 7: Hold write lock for full DELETE+INSERT (L1)
        from .write_lock import cache_write_lock

        with cache_write_lock(symbol, threshold_decimal_bps):
            return self._do_replace(
                symbol, threshold_decimal_bps, bars_df,
                ouroboros_mode=ouroboros_mode, version=version,
                new_first=new_first, new_last=new_last,
            )

    def _do_replace(
        self,
        symbol: str,
        threshold_decimal_bps: int,
        bars_df: object,
        *,
        ouroboros_mode: str,
        version: str | None,
        new_first: int,
        new_last: int,
    ) -> int:
        """Synchronous traditional DELETE-then-INSERT for authoritative bar replacement.

        Uses ALTER TABLE DELETE (traditional mutation) — NOT lightweight DELETE FROM.
        Traditional mutations physically rewrite parts, removing rows permanently.
        Lightweight DELETE FROM only sets _row_exists=0 masks that persist as ghost
        rows until CH background merge, causing false Stathera gap reports.

        mutations_sync=1 blocks until the DELETE completes — no async race
        where the mutation runs after INSERT and deletes the new bars.
        This was the root cause of bar loss in v13.39-v13.40 (#293).
        """
        # Issue #296: Serialize DELETE+INSERT pairs per partition.
        # Same-partition mutations must be serial (concurrent mutations_sync=1
        # DELETEs queue up). Different partitions mutate concurrently.
        with _get_partition_lock(symbol, threshold_decimal_bps):
            # Step 1: DELETE IN PARTITION (symbol, threshold) — partition key
            # no longer includes date. ORDER BY index prunes to matching
            # trade-ID granules. No date lookup query needed.
            self.client.command(
                "ALTER TABLE opendeviationbar_cache.open_deviation_bars "
                f"DELETE IN PARTITION ('{symbol}', {threshold_decimal_bps}) "
                f"WHERE first_agg_trade_id <= {new_last} "
                f"AND last_agg_trade_id >= {new_first} "
                "AND first_agg_trade_id > 0 "
                "SETTINGS mutations_sync = 1"
            )
            # Step 2: INSERT new bars (DELETE already completed synchronously).
            try:
                rows_inserted = self.store_bars_batch(
                    symbol, threshold_decimal_bps, bars_df,
                    version=version, ouroboros_mode=ouroboros_mode,
                    skip_dedup=True,
                    overlap_strategy="off",
                    _skip_lock=True,
                )
            except Exception:
                _dead_letter_replace_failure(
                    symbol, threshold_decimal_bps, bars_df, new_first, new_last,
                )
                raise

        return rows_inserted

    def _wait_for_mutations(
        self,
        timeout: int = 300,
        poll_interval: int = 5,
    ) -> None:
        """Poll system.mutations until all pending mutations complete.

        Issue #158: Used after ALTER TABLE DELETE to ensure bars are removed
        before inserting replacements.
        """
        import time

        deadline = time.monotonic() + timeout
        while time.monotonic() < deadline:
            result = self.client.query(
                "SELECT count() FROM system.mutations "
                "WHERE database = 'opendeviationbar_cache' "
                "AND table = 'open_deviation_bars' "
                "AND is_done = 0"
            )
            active = result.result_rows[0][0] if result.result_rows else 0
            if active == 0:
                return
            logger.debug("Waiting for %d active mutation(s)...", active)
            time.sleep(poll_interval)
        logger.warning(
            "Mutation wait timed out after %ds, mutations continue in background",
            timeout,
        )

    def _wait_for_merges(self, timeout: int = 600, poll_interval: int = 10) -> None:
        """Poll system.merges until no active merges for our table.

        Issue #90: After submitting OPTIMIZE, poll ClickHouse to wait for
        background merges to finish. If timeout is reached, log a warning
        and return — merges continue server-side.

        Parameters
        ----------
        timeout : int
            Maximum seconds to wait (default: 600).
        poll_interval : int
            Seconds between polls (default: 10).
        """
        import time

        deadline = time.monotonic() + timeout
        while time.monotonic() < deadline:
            result = self.client.query(
                "SELECT count() FROM system.merges "
                "WHERE database = 'opendeviationbar_cache' "
                "AND table = 'open_deviation_bars'"
            )
            active = result.result_rows[0][0] if result.result_rows else 0
            if active == 0:
                logger.info("All merges complete")
                return
            logger.debug("Waiting for %d active merge(s)...", active)
            time.sleep(poll_interval)
        logger.warning(
            "Merge wait timed out after %ds, merges continue in background",
            timeout,
        )
