"""Business logic for sidecar REST API endpoints (Issue #213).

Extracted from sidecar.py to keep HTTP handlers thin and logic testable.

Endpoints:
    GET /ariadne/{symbol}/{threshold}  — Cascading last_agg_trade_id lookup (#217)
    GET /trades/gap-fill               — Trade continuity gap-fill (#216)
    GET /catchup/{symbol}/{threshold}  — Server-side gap-fill (#224)
"""

from __future__ import annotations

import logging
import re
from collections.abc import Callable
from typing import TYPE_CHECKING, Any

if TYPE_CHECKING:
    from opendeviationbar.config.backfill import BackfillConfig

logger = logging.getLogger(__name__)

# Allowlist: symbols are uppercase alphanumeric (e.g., BTCUSDT, ETHBTC)
_SYMBOL_RE = re.compile(r"^[A-Z0-9]{2,20}$")

# Per-tier limit caps for gap-fill (#220).
# Parquet can serve up to 5000 (local I/O, no API budget).
# REST capped at Binance's per-request max of 1000.
# Overall cap = PARQUET + REST = 6000 max per call.
_PARQUET_LIMIT_CAP = 5000
_REST_LIMIT_CAP = 1000
_OVERALL_LIMIT_CAP = _PARQUET_LIMIT_CAP + _REST_LIMIT_CAP


def _ariadne_clickhouse(
    symbol: str,
    threshold: int,
    result: dict[str, Any],
    sources_tried: list[dict[str, str]],
) -> dict[str, Any] | None:
    """ClickHouse sources (3 + 4) for ariadne cascade.

    SAFETY: caller (ariadne_lookup) validates symbol via _SYMBOL_RE before calling.
    """
    ch_cache = None
    try:
        from opendeviationbar.clickhouse.cache import OpenDeviationBarCache

        ch_cache = OpenDeviationBarCache()
    except Exception as exc:
        logger.warning("ariadne: ClickHouse unreachable (%s) — skipping sources 3+4, cascade continues to Binance REST", exc)
        sources_tried.append({"source": "clickhouse_bars", "status": "error", "reason": "cache init failed"})
        sources_tried.append({"source": "clickhouse_checkpoints", "status": "skipped", "reason": "cache init failed"})
        return None

    # Source 3: open_deviation_bars
    try:
        from opendeviationbar.config import Settings

        ouroboros_mode = Settings.get().population.ouroboros_mode
        query = (
            "SELECT max(last_agg_trade_id) AS max_tid "
            "FROM opendeviationbar_cache.open_deviation_bars FINAL "
            "WHERE symbol = {symbol:String} "
            "AND threshold_decimal_bps = {threshold:UInt32} "
            "AND ouroboros_mode = {ouroboros:String} "
            "AND last_agg_trade_id > 0"
        )
        row = ch_cache.client.query(
            query,
            parameters={"symbol": symbol, "threshold": threshold, "ouroboros": ouroboros_mode},
        ).first_row
        if row and row[0] and row[0] > 0:
            result["last_agg_trade_id"] = row[0]
            result["source"] = "clickhouse_bars"
            sources_tried.append({"source": "clickhouse_bars", "status": "hit", "tid": str(row[0])})
            return result
        sources_tried.append({"source": "clickhouse_bars", "status": "skipped", "reason": "no rows"})
    except Exception as exc:
        logger.warning("ariadne: ClickHouse bars query failed (%s) — falling through to checkpoints", exc)
        sources_tried.append({"source": "clickhouse_bars", "status": "error"})

    # Source 4: population_checkpoints
    try:
        query = (
            "SELECT max(last_agg_trade_id) AS max_tid "
            "FROM opendeviationbar_cache.population_checkpoints FINAL "
            "WHERE symbol = {symbol:String} "
            "AND threshold_decimal_bps = {threshold:UInt32} "
            "AND last_agg_trade_id > 0"
        )
        row = ch_cache.client.query(
            query,
            parameters={"symbol": symbol, "threshold": threshold},
        ).first_row
        if row and row[0] and row[0] > 0:
            result["last_agg_trade_id"] = row[0]
            result["source"] = "clickhouse_checkpoints"
            sources_tried.append({"source": "clickhouse_checkpoints", "status": "hit", "tid": str(row[0])})
            return result
        sources_tried.append({"source": "clickhouse_checkpoints", "status": "skipped", "reason": "no rows"})
    except Exception as exc:
        logger.warning("ariadne: ClickHouse checkpoints query failed (%s) — falling through to Binance REST", exc)
        sources_tried.append({"source": "clickhouse_checkpoints", "status": "error"})

    return None


def _ariadne_source1_live_processor(
    symbol: str,
    threshold: int,
    result: dict[str, Any],
    sources_tried: list[dict[str, str]],
    engine: object | None,
) -> dict[str, Any] | None:
    """Check live processor for forming bar (Source 1)."""
    if engine is None:
        sources_tried.append({"source": "live_processor", "status": "skipped", "reason": "engine is None"})
        return None
    if not hasattr(engine, "get_forming_bars"):
        sources_tried.append({"source": "live_processor", "status": "skipped", "reason": "missing get_forming_bars"})
        return None

    try:
        forming_bars = engine.get_forming_bars()
        matched = False
        for fb in forming_bars:
            if fb.get("_symbol") == symbol and fb.get("_threshold") == threshold:
                tid = fb.get("last_agg_trade_id")
                if tid and tid > 0:
                    result["last_agg_trade_id"] = tid
                    result["source"] = "live_processor"
                    result["has_incomplete_bar"] = True
                    sources_tried.append({"source": "live_processor", "status": "hit", "tid": str(tid)})
                    result["sources_tried"] = sources_tried
                    return result
                sources_tried.append({"source": "live_processor", "status": "skipped", "reason": f"tid={tid}"})
                matched = True
                break
        if not matched:
            sources_tried.append({"source": "live_processor", "status": "skipped", "reason": "no match"})
            logger.info("ariadne: Source 1 no match for %s@%d", symbol, threshold)
    except Exception:
        sources_tried.append({"source": "live_processor", "status": "error"})
        logger.debug("ariadne: live processor busy (concurrent WS trade processing) — falling through to checkpoint/CH")
    return None


def _ariadne_source2_checkpoint_file(
    symbol: str,
    threshold: int,
    result: dict[str, Any],
    sources_tried: list[dict[str, str]],
) -> dict[str, Any] | None:
    """Check processor checkpoint file (Source 2)."""
    try:
        from opendeviationbar.sidecar import _load_checkpoint

        cp = _load_checkpoint(symbol, threshold)
        if cp and cp.get("processor_checkpoint"):
            pc = cp["processor_checkpoint"]
            incomplete = pc.get("incomplete_bar")
            if incomplete:
                tid = incomplete.get("last_agg_trade_id")
                if tid and tid > 0:
                    result["last_agg_trade_id"] = tid
                    result["source"] = "checkpoint_file"
                    result["has_incomplete_bar"] = True
                    sources_tried.append({"source": "checkpoint_file", "status": "hit", "tid": str(tid)})
                    result["sources_tried"] = sources_tried
                    return result
            last_tid = pc.get("last_agg_trade_id")
            if last_tid and last_tid > 0:
                result["last_agg_trade_id"] = last_tid
                result["source"] = "checkpoint_file"
                sources_tried.append({"source": "checkpoint_file", "status": "hit", "tid": str(last_tid)})
                result["sources_tried"] = sources_tried
                return result
            sources_tried.append({"source": "checkpoint_file", "status": "skipped", "reason": "no tid in checkpoint"})
        else:
            sources_tried.append({"source": "checkpoint_file", "status": "skipped", "reason": "no checkpoint"})
    except Exception as exc:
        sources_tried.append({"source": "checkpoint_file", "status": "error"})
        logger.debug("ariadne: checkpoint file not available (%s) — falling through to ClickHouse", exc)
    return None


def ariadne_lookup(
    symbol: str,
    threshold: int,
    engine: object | None = None,
    *,
    committed_only: bool = False,
) -> dict[str, Any]:
    """Cascading lookup for last_agg_trade_id (Issue #217).

    5-source cascade (most authoritative first):
    1. Live processor state (forming bar from engine)
    2. Processor checkpoint file
    3. ClickHouse open_deviation_bars (MAX last_agg_trade_id)
    4. ClickHouse population_checkpoints
    5. Binance REST API (latest aggTrade)

    When ``committed_only=True`` (#225), Sources 1-2 are skipped so only
    committed ClickHouse data is returned — used by /catchup/ endpoint.

    Returns a dict with last_agg_trade_id, source, sources_tried, and metadata.
    """
    if not _SYMBOL_RE.match(symbol):
        msg = f"invalid symbol: {symbol!r}"
        raise ValueError(msg)
    from opendeviationbar.symbol_registry import validate_symbol_registered

    validate_symbol_registered(symbol, operation="ariadne_lookup")
    if threshold <= 0:
        msg = f"threshold must be positive, got {threshold}"
        raise ValueError(msg)

    result: dict[str, Any] = {
        "symbol": symbol,
        "threshold_dbps": threshold,
        "last_agg_trade_id": None,
        "source": None,
        "has_incomplete_bar": False,
        "committed_only": committed_only,
    }
    sources_tried: list[dict[str, str]] = []

    if committed_only:
        sources_tried.append({"source": "live_processor", "status": "skipped", "reason": "committed_only"})
        sources_tried.append({"source": "checkpoint_file", "status": "skipped", "reason": "committed_only"})
    else:
        # Source 1: Live processor state (forming bars from engine)
        source1_result = _ariadne_source1_live_processor(symbol, threshold, result, sources_tried, engine)
        if source1_result is not None:
            return source1_result

        # Source 2: Processor checkpoint file
        source2_result = _ariadne_source2_checkpoint_file(symbol, threshold, result, sources_tried)
        if source2_result is not None:
            return source2_result

    # Sources 3 + 4: ClickHouse
    ch_result = _ariadne_clickhouse(symbol, threshold, result, sources_tried)
    if ch_result is not None:
        ch_result["sources_tried"] = sources_tried
        return ch_result

    # Source 5: Binance REST API (latest aggTrade)
    try:
        from opendeviationbar._core import fetch_latest_aggtrade

        trade = fetch_latest_aggtrade(symbol)
        if trade and "agg_trade_id" in trade:
            result["last_agg_trade_id"] = trade["agg_trade_id"]
            result["source"] = "binance_rest"
            sources_tried.append({"source": "binance_rest", "status": "hit", "tid": str(trade["agg_trade_id"])})
            result["sources_tried"] = sources_tried
            return result
        sources_tried.append({"source": "binance_rest", "status": "skipped", "reason": "no agg_trade_id"})
    except Exception as exc:
        sources_tried.append({"source": "binance_rest", "status": "error"})
        logger.warning("ariadne: Binance REST (source 5) failed: %s — no more fallbacks", exc)

    # All sources exhausted — flag degraded so consumer can distinguish from "no data"
    if result["last_agg_trade_id"] is None:
        logger.error(
            "ariadne: all 5 sources exhausted for %s@%d (live/checkpoint/CH bars/CH checkpoints/REST) — returning degraded response",
            symbol, threshold,
        )
        result["degraded"] = True

    result["sources_tried"] = sources_tried
    return result


def _to_trade_dict(row: dict[str, Any]) -> dict[str, Any]:
    """Convert a raw trade row to Binance-compatible format."""
    return {
        "a": row.get("agg_trade_id", 0),
        "p": str(row.get("price", "0")),
        "q": str(row.get("quantity", row.get("volume", "0"))),
        "T": row.get("timestamp", 0),
        "m": row.get("is_buyer_maker", False),
    }


def _parquet_scan(symbol: str, from_agg_id: int, limit: int) -> list[dict[str, Any]]:
    """Scan Parquet tick cache for trades starting from a specific aggTrade ID.

    Accumulates across multiple monthly Parquet files (#227) to handle
    agg_trade_id ranges spanning file boundaries.
    """
    import polars as pl

    from opendeviationbar.storage.parquet import TickStorage

    storage = TickStorage()
    tick_dir = storage._get_symbol_dir(symbol)
    if not tick_dir.exists():
        logger.debug("gap-fill: tick directory does not exist for %s: %s", symbol, tick_dir)
        return []

    parquet_files = sorted(tick_dir.glob("*.parquet"))
    accumulated: list[dict[str, Any]] = []
    seen_ids: set[int] = set()
    failures = 0

    for pf in parquet_files[:5]:
        if len(accumulated) >= limit:
            break
        try:
            lf = pl.scan_parquet(pf)
            if "agg_trade_id" not in lf.collect_schema().names():
                continue
            remaining = limit - len(accumulated)
            filtered = (
                lf.filter(pl.col("agg_trade_id") >= from_agg_id)
                .sort("agg_trade_id")
                .head(remaining + 100)  # Over-fetch for dedup margin
                .collect()
            )
            for row in filtered.iter_rows(named=True):
                tid = row.get("agg_trade_id", 0)
                if tid not in seen_ids:
                    seen_ids.add(tid)
                    accumulated.append(_to_trade_dict(row))
                    if len(accumulated) >= limit:
                        break
        except MemoryError:
            raise
        except Exception:
            failures += 1
            logger.debug("gap-fill: failed to read %s", pf, exc_info=True)

    if failures > 0 and failures == min(len(parquet_files), 5):
        logger.warning("gap-fill: all %d Parquet files failed for %s", failures, symbol)

    # Sort ascending — files may overlap at boundaries
    accumulated.sort(key=lambda t: t["a"])
    if accumulated:
        logger.debug("gap-fill: %d trades from Parquet cache for %s", len(accumulated), symbol)
    return accumulated[:limit]


def _rest_fetch(symbol: str, from_agg_id: int, limit: int) -> list[dict[str, Any]]:
    """Fetch trades from Binance REST API starting from a specific aggTrade ID."""
    from opendeviationbar._core import fetch_aggtrades_by_id

    trades_raw = fetch_aggtrades_by_id(symbol, from_agg_id, limit)
    trades = [_to_trade_dict(t) for t in trades_raw]
    logger.debug(
        "gap-fill: %d trades from Binance REST for %s",
        len(trades),
        symbol,
    )
    return trades


def gap_fill_trades(
    symbol: str,
    from_agg_id: int,
    limit: int = 1000,
) -> dict[str, Any]:
    """Fetch trades for gap-fill starting from a specific aggTrade ID (Issue #216).

    Two-tier lookup with bridging (#220):
    1. Fast path: Local Parquet tick cache (TickStorage)
    2. Bridge: If Parquet returned partial data, continue via Binance REST

    Returns a dict with ``trades`` list, ``partial`` flag, and metadata.
    Max effective limit is 6000 (5000 Parquet + 1000 REST).
    """
    if not _SYMBOL_RE.match(symbol):
        msg = f"invalid symbol: {symbol!r}"
        raise ValueError(msg)
    from opendeviationbar.symbol_registry import validate_symbol_registered

    validate_symbol_registered(symbol, operation="gap_fill_trades")
    if from_agg_id < 0:
        msg = f"from_agg_id must be non-negative, got {from_agg_id}"
        raise ValueError(msg)
    if limit <= 0:
        return {"trades": [], "partial": False, "count": 0}

    # Defense-in-depth: cap at overall max regardless of caller
    limit = min(limit, _OVERALL_LIMIT_CAP)

    parquet_limit = min(limit, _PARQUET_LIMIT_CAP)
    trades: list[dict[str, Any]] = []
    partial = False

    # Tier 1: Parquet tick cache
    try:
        parquet_trades = _parquet_scan(symbol, from_agg_id, parquet_limit)
        trades.extend(parquet_trades)
    except MemoryError:
        raise
    except Exception as exc:
        logger.info("gap-fill: Parquet cache unavailable for %s (%s) — falling through to REST", symbol, exc)

    # Bridge: if Parquet returned partial data, continue via REST (#220)
    if len(trades) < limit:
        continuation_id = trades[-1]["a"] + 1 if trades else from_agg_id
        remaining = limit - len(trades)
        rest_limit = min(remaining, _REST_LIMIT_CAP)
        try:
            rest_trades = _rest_fetch(symbol, continuation_id, rest_limit)
            trades.extend(rest_trades)
        except Exception:
            if not trades:
                raise  # No Parquet data either — propagate
            logger.warning(
                "gap-fill: REST bridge failed after %d Parquet trades for %s — returning partial result",
                len(trades), symbol,
            )
            partial = True

    return {"trades": trades, "partial": partial, "count": len(trades)}


# ---------------------------------------------------------------------------
# Server-side paginated gap-fill (#226)
# ---------------------------------------------------------------------------

def _backfill_cfg() -> BackfillConfig:
    from opendeviationbar.config.backfill import BackfillConfig
    return BackfillConfig()


def _paginate_trades_loop(
    symbol: str,
    fetch_fn: Callable[[str, int, int], list[dict[str, Any]]],
    cursor: int,
    max_trades: int,
    limit_cap: int,
    phase_name: str,
) -> tuple[list[dict[str, Any]], int, int, int, bool]:
    """Generic pagination loop for Parquet or REST.

    Returns (trades, count, batches, cursor, failed).
    """
    trades: list[dict[str, Any]] = []
    count = 0
    batches = 0
    failed = False
    max_batches = (max_trades // max(limit_cap, 1)) + 10  # Safety cap
    try:
        while len(trades) < max_trades and batches < max_batches:
            remaining = max_trades - len(trades)
            batch_limit = min(remaining, limit_cap)
            batch = fetch_fn(symbol, cursor, batch_limit)
            if not batch:
                break
            batches += 1
            count += len(batch)
            trades.extend(batch)
            new_cursor = batch[-1]["a"] + 1
            if new_cursor <= cursor:
                logger.warning("gap-fill-paginated: cursor did not advance (%d → %d), aborting %s for %s",
                               cursor, new_cursor, phase_name, symbol)
                break
            cursor = new_cursor
            if len(batch) < batch_limit:  # Short batch = data exhausted
                break
    except MemoryError:
        raise
    except Exception as exc:
        logger.warning("gap-fill-paginated: %s batch failed for %s (%s) — returning %d trades collected so far", phase_name, symbol, exc, len(trades))
        failed = True
    return trades, count, batches, cursor, failed


def gap_fill_trades_paginated(
    symbol: str,
    from_agg_id: int,
    max_trades: int = 50_000,
) -> dict[str, Any]:
    """Paginated gap-fill across Parquet + REST with server-side orchestration (#226).

    Unlike ``gap_fill_trades()`` which caps at 6000, this loops internally:
    1. Drain Parquet via repeated ``_parquet_scan()`` calls
    2. Continue via REST ``_rest_fetch()`` until data exhausted or cap hit

    Memory safety: 100K trades * ~100 bytes = ~10MB hard cap.
    """
    if not _SYMBOL_RE.match(symbol):
        msg = f"invalid symbol: {symbol!r}"
        raise ValueError(msg)
    from opendeviationbar.symbol_registry import validate_symbol_registered

    validate_symbol_registered(symbol, operation="gap_fill_trades_paginated")
    if from_agg_id < 0:
        msg = f"from_agg_id must be non-negative, got {from_agg_id}"
        raise ValueError(msg)

    max_trades = min(max_trades, _backfill_cfg().paginated_max_trades)
    if max_trades <= 0:
        return {
            "trades": [],
            "count": 0,
            "partial": False,
            "from_agg_id": from_agg_id,
            "through_agg_id": None,
            "source_breakdown": {"parquet": 0, "rest": 0},
            "batches_fetched": 0,
        }

    cursor = from_agg_id
    batches_total = 0

    # Phase 1: Parquet
    parquet_trades, parquet_count, pq_batches, cursor, _ = _paginate_trades_loop(
        symbol, _parquet_scan, cursor, max_trades, _PARQUET_LIMIT_CAP, "Parquet"
    )
    batches_total += pq_batches
    all_trades = parquet_trades

    # Phase 2: REST (only if haven't hit max yet)
    rest_count = 0
    rest_batches = 0
    rest_failed = False
    partial = False
    if len(all_trades) < max_trades:
        remaining = max_trades - len(all_trades)
        rest_trades, rest_count, rest_batches, cursor, rest_failed = _paginate_trades_loop(
            symbol, _rest_fetch, cursor, remaining, _REST_LIMIT_CAP, "REST"
        )
        batches_total += rest_batches
        all_trades.extend(rest_trades)
        # REST failure with existing trades marks partial
        if rest_failed and all_trades:
            partial = True

    # Also set partial if we hit the cap
    if len(all_trades) >= max_trades:
        partial = True

    # Truncate before computing through_agg_id so it matches returned trades
    all_trades = all_trades[:max_trades]
    through_agg_id = all_trades[-1]["a"] if all_trades else None

    return {
        "trades": all_trades,
        "count": len(all_trades),
        "partial": partial,
        "from_agg_id": from_agg_id,
        "through_agg_id": through_agg_id,
        "source_breakdown": {"parquet": parquet_count, "rest": rest_count},
        "batches_fetched": batches_total,
    }


# ---------------------------------------------------------------------------
# Catchup composite endpoint (#228)
# ---------------------------------------------------------------------------


def catchup_lookup(
    symbol: str,
    threshold: int,
    max_trades: int = 50_000,
    engine: object | None = None,
) -> dict[str, Any]:
    """Server-side gap-fill: ariadne (committed) → paginated trades (#228).

    Composes ``ariadne_lookup(committed_only=True)`` with
    ``gap_fill_trades_paginated()`` into a single call, replacing 3-step
    client orchestration in flowsurface.
    """
    ariadne_result = ariadne_lookup(symbol, threshold, engine=engine, committed_only=True)

    last_tid = ariadne_result.get("last_agg_trade_id")
    if last_tid is None:
        return {
            "symbol": symbol,
            "threshold_dbps": threshold,
            "last_committed_bar": {"last_agg_trade_id": None, "source": None},
            "trades": [],
            "from_agg_id": None,
            "through_agg_id": None,
            "count": 0,
            "partial": False,
            "source_breakdown": {"parquet": 0, "rest": 0},
            "batches_fetched": 0,
            "ariadne_sources_tried": ariadne_result.get("sources_tried", []),
            "error": "no committed bar found — all ariadne sources exhausted",
        }

    from_id = last_tid + 1
    gap_result = gap_fill_trades_paginated(symbol, from_id, max_trades=max_trades)

    return {
        "symbol": symbol,
        "threshold_dbps": threshold,
        "last_committed_bar": {
            "last_agg_trade_id": last_tid,
            "source": ariadne_result.get("source"),
        },
        "trades": gap_result["trades"],
        "from_agg_id": from_id,
        "through_agg_id": gap_result["through_agg_id"],
        "count": gap_result["count"],
        "partial": gap_result["partial"],
        "source_breakdown": gap_result["source_breakdown"],
        "batches_fetched": gap_result["batches_fetched"],
        "ariadne_sources_tried": ariadne_result.get("sources_tried", []),
    }
