# Issue #318: Bulk Parquet population for clean-slate sidecar startup
"""Populate ClickHouse from local Parquet tick cache on empty table.

When the sidecar detects an empty ClickHouse table (0 bars for a symbol),
this module runs repair_direct_parquet-style bulk processing inline during
startup. This eliminates the need for manual repair_direct_parquet.py
invocation and makes clean-slate deploys populate in one go.

Design:
- Sequential per-symbol (Aion requires continuous processor state)
- All thresholds from one Parquet read per day (threshold-agnostic ticks)
- Saves processor checkpoints for seamless sidecar handoff
- Respects symbols.toml as SSoT for symbol + threshold selection
"""

from __future__ import annotations

import gc
import json
import logging
import time
from datetime import UTC, date, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING

import polars as pl

from opendeviationbar._core import PyOpenDeviationBarProcessor

if TYPE_CHECKING:
    from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

logger = logging.getLogger("opendeviationbar.sidecar")

_TICK_COLS = ["agg_trade_id", "price", "quantity", "timestamp", "is_buyer_maker"]


def _bulk_populate_from_parquet(
    ch_cache: OpenDeviationBarCache,
    symbols: list[str],
    thresholds: list[int],
) -> dict[str, int]:
    """Populate ClickHouse for symbols with 0 bars from local Parquet ticks.

    Returns dict of {symbol: bars_written} for symbols that were populated.
    Symbols with existing bars are skipped (not a clean-slate symbol).
    """
    from opendeviationbar import __version__
    from opendeviationbar.symbol_registry import (
        get_effective_start_date,
        get_symbol_entries,
        get_thresholds_for_symbol,
    )

    source_version = f"{__version__}+sidecar-bulk"
    results: dict[str, int] = {}
    entries = get_symbol_entries()

    # Find symbols with 0 bars in ClickHouse
    empty_symbols: list[str] = []
    for sym in symbols:
        try:
            result = ch_cache.client.query(
                "SELECT count() FROM opendeviationbar_cache.open_deviation_bars FINAL"
                f" WHERE symbol = '{sym}' AND first_agg_trade_id > 0"
            )
            count = int(result.result_rows[0][0]) if result.result_rows else 0
            if count == 0:
                empty_symbols.append(sym)
        except (OSError, RuntimeError, ConnectionError):
            logger.warning("bulk-populate: count query failed for %s", sym, exc_info=True)

    if not empty_symbols:
        logger.info("bulk-populate: all %d symbols have existing bars — skipping", len(symbols))
        return results

    logger.info(
        "bulk-populate: %d/%d symbols empty — populating from Parquet: %s",
        len(empty_symbols), len(symbols), ", ".join(empty_symbols),
    )

    for sym in empty_symbols:
        entry = entries.get(sym)
        if entry is None:
            logger.warning("bulk-populate: %s not in registry — skipping", sym)
            continue

        sym_thresholds = list(get_thresholds_for_symbol(sym))
        if not sym_thresholds:
            sym_thresholds = thresholds

        mode = entry.ouroboros_mode.value

        start_str = get_effective_start_date(sym)
        start = date.fromisoformat(start_str) if start_str else date(2018, 1, 16)
        end_date = datetime.now(UTC).date()

        sym_dir = Path.home() / ".cache" / "opendeviationbar" / "ticks" / sym
        if not sym_dir.exists():
            logger.info("bulk-populate: %s — no Parquet ticks directory", sym)
            continue

        available_days = _collect_available_days(sym_dir, start, end_date)
        if not available_days:
            logger.info("bulk-populate: %s — no Parquet files found", sym)
            continue

        logger.info(
            "bulk-populate: %s — processing %d days (%s to %s) x %d thresholds",
            sym, len(available_days), available_days[0], available_days[-1],
            len(sym_thresholds),
        )

        total_bars = _process_symbol(
            sym, available_days, sym_dir, sym_thresholds, mode,
            ch_cache, source_version,
        )
        results[sym] = total_bars

    logger.info(
        "bulk-populate: completed — %d symbols, %d total bars",
        len(results), sum(results.values()),
    )
    return results


def _collect_available_days(
    sym_dir: Path, start: date, end_date: date,
) -> list[date]:
    """Collect sorted list of dates with existing Parquet files."""
    available: list[date] = []
    d = start
    while d <= end_date:
        if (sym_dir / f"{d.isoformat()}.parquet").exists():
            available.append(d)
        d += timedelta(days=1)
    available.sort()
    return available


def _read_day_arrow(sym_dir: Path, day: date) -> object | None:
    """Read one day's Parquet ticks as Arrow RecordBatch."""
    parquet_path = sym_dir / f"{day.isoformat()}.parquet"
    try:
        lf = pl.scan_parquet(parquet_path)
        select_cols = [c for c in _TICK_COLS if c in lf.collect_schema().names()]
        day_ticks = lf.select(select_cols).collect()
    except (OSError, RuntimeError):
        return None

    if day_ticks.is_empty():
        return None

    trades_arrow = day_ticks.to_arrow()
    del day_ticks
    batches = trades_arrow.to_batches()
    del trades_arrow
    if not batches:
        return None
    if len(batches) == 1:
        return batches[0]
    import pyarrow as pa
    return pa.Table.from_batches(batches).combine_chunks().to_batches()[0]


def _rename_arrow_columns(bars_df: pl.DataFrame) -> pl.DataFrame:
    """Rename Arrow open_time/close_time to CH _us suffix columns."""
    rename_exprs: list[pl.Expr] = []
    drop_cols: list[str] = []
    if "open_time" in bars_df.columns:
        rename_exprs.append(pl.col("open_time").alias("open_time_us"))
        drop_cols.append("open_time")
    if "close_time" in bars_df.columns:
        rename_exprs.append(pl.col("close_time").alias("close_time_us"))
        drop_cols.append("close_time")
    if rename_exprs:
        return bars_df.with_columns(rename_exprs).drop(drop_cols)
    return bars_df


def _process_symbol(
    symbol: str,
    days: list[date],
    sym_dir: Path,
    thresholds: list[int],
    mode: str,
    ch_cache: OpenDeviationBarCache,
    source_version: str,
) -> int:
    """Process all days for one symbol through continuous Aion processors."""
    processors = {thr: PyOpenDeviationBarProcessor(thr) for thr in thresholds}
    total_bars = 0
    t0 = time.monotonic()

    for day_idx, day in enumerate(days):
        trades_batch = _read_day_arrow(sym_dir, day)
        if trades_batch is None:
            continue

        total_bars += _process_day_all_thresholds(
            symbol, day, trades_batch, thresholds, processors,
            mode, ch_cache, source_version,
        )
        del trades_batch

        if (day_idx + 1) % 200 == 0 or (day_idx + 1) == len(days):
            elapsed = time.monotonic() - t0
            rate = (day_idx + 1) / elapsed * 60 if elapsed > 0 else 0
            logger.info(
                "bulk-populate: %s %d/%d days (%.0f/min) bars=%d",
                symbol, day_idx + 1, len(days), rate, total_bars,
            )

        if (day_idx + 1) % 200 == 0:
            gc.collect()

    _save_checkpoints(symbol, processors)
    del processors

    elapsed = time.monotonic() - t0
    logger.info(
        "bulk-populate: %s complete — %d bars from %d days in %.1fs (%.1f days/min)",
        symbol, total_bars, len(days), elapsed,
        len(days) / elapsed * 60 if elapsed > 0 else 0,
    )
    return total_bars


def _process_day_all_thresholds(
    symbol: str,
    day: date,
    trades_batch: object,
    thresholds: list[int],
    processors: dict[int, PyOpenDeviationBarProcessor],
    mode: str,
    ch_cache: OpenDeviationBarCache,
    source_version: str,
) -> int:
    """Process one day's trades through all threshold processors and write to CH."""
    day_bars = 0
    for thr in thresholds:
        bars_arrow = processors[thr].process_trades_arrow_native(trades_batch)
        bars_df = pl.from_arrow(bars_arrow)
        del bars_arrow

        if bars_df.is_empty():
            continue

        if "first_agg_trade_id" in bars_df.columns:
            bars_df = bars_df.filter(pl.col("first_agg_trade_id") > 0)
            if bars_df.is_empty():
                continue

        bars_df = _rename_arrow_columns(bars_df)

        try:
            written = ch_cache.store_bars_batch(
                symbol=symbol,
                threshold_decimal_bps=thr,
                bars=bars_df,
                version=source_version,
                overlap_strategy="warn",
                ouroboros_mode=mode,
                _skip_lock=True,
            )
            day_bars += written
        except Exception:
            logger.warning(
                "bulk-populate: store_bars_batch failed %s@%d day=%s",
                symbol, thr, day, exc_info=True,
            )
    return day_bars


def _save_checkpoints(
    symbol: str,
    processors: dict[int, PyOpenDeviationBarProcessor],
) -> None:
    """Save processor checkpoints for sidecar handoff."""
    try:
        from platformdirs import user_cache_dir

        ckpt_dir = Path(user_cache_dir("opendeviationbar")) / "repair_checkpoints"
        ckpt_dir.mkdir(parents=True, exist_ok=True)
        for thr, processor in processors.items():
            ckpt = processor.create_checkpoint(symbol)
            ckpt_path = ckpt_dir / f"{symbol}_{thr}.json"
            ckpt_path.write_text(json.dumps(ckpt, default=str))
        logger.info("bulk-populate: %s — saved %d checkpoints", symbol, len(processors))
    except Exception:
        logger.warning("bulk-populate: %s — checkpoint save failed", symbol, exc_info=True)
