"""Tests for sidecar_api.py — gap-fill bridge (#220), token bucket (#222), sources_tried (#221)."""

from __future__ import annotations

import threading
import time
from unittest.mock import MagicMock, patch

import pytest

# ---------------------------------------------------------------------------
# Fix 2: Token bucket rate limiter (#222)
# ---------------------------------------------------------------------------

class TestTokenBucket:
    """Tests for _TokenBucket in sidecar.py."""

    def _make_bucket(self, rate: float = 1.0, burst: int = 5):
        from opendeviationbar.sidecar import _TokenBucket

        return _TokenBucket(rate=rate, burst=burst)

    def test_burst(self):
        """5 rapid acquires succeed, 6th fails."""
        bucket = self._make_bucket(rate=1.0, burst=5)
        for _ in range(5):
            assert bucket.try_acquire()
        assert not bucket.try_acquire()

    def test_refill(self):
        """After exhaustion + 1s sleep, one more succeeds."""
        bucket = self._make_bucket(rate=1.0, burst=5)
        for _ in range(5):
            bucket.try_acquire()
        assert not bucket.try_acquire()
        time.sleep(1.1)
        assert bucket.try_acquire()

    def test_thread_safe(self):
        """Concurrent access from multiple threads doesn't over-issue tokens."""
        bucket = self._make_bucket(rate=0.0, burst=10)  # No refill
        results: list[bool] = []
        lock = threading.Lock()

        def acquire_many():
            local = []
            for _ in range(5):
                local.append(bucket.try_acquire())
            with lock:
                results.extend(local)

        threads = [threading.Thread(target=acquire_many) for _ in range(4)]
        for t in threads:
            t.start()
        for t in threads:
            t.join()

        assert sum(results) == 10  # Exactly burst count

    def test_no_refill_at_zero_rate(self):
        """Rate=0 means no refill ever."""
        bucket = self._make_bucket(rate=0.0, burst=1)
        assert bucket.try_acquire()
        time.sleep(0.1)
        assert not bucket.try_acquire()

    def test_partial_refill(self):
        """Partial refill — 0.5s at rate=2.0 refills 1 token."""
        bucket = self._make_bucket(rate=2.0, burst=5)
        for _ in range(5):
            bucket.try_acquire()
        time.sleep(0.55)
        assert bucket.try_acquire()
        assert not bucket.try_acquire()  # Only ~1 token refilled


# ---------------------------------------------------------------------------
# Fix 1: Parquet → REST bridge (#220)
# ---------------------------------------------------------------------------

def _make_trade_dict(agg_id: int) -> dict:
    return {"a": agg_id, "p": "42000", "q": "0.5", "T": 1700000000000, "m": False}


class TestGapFillBridge:
    """Tests for gap_fill_trades() Parquet→REST bridging."""

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_parquet_full_no_rest(self, mock_parquet, mock_rest):
        """When Parquet returns limit trades, REST is not called."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.return_value = [_make_trade_dict(i) for i in range(1000)]
        result = gap_fill_trades("BTCUSDT", 100, limit=1000)
        assert result["count"] == 1000
        assert result["partial"] is False
        mock_rest.assert_not_called()

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_parquet_partial_bridges_rest(self, mock_parquet, mock_rest):
        """Parquet returns 300/1000, REST bridges remaining."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        parquet_trades = [_make_trade_dict(100 + i) for i in range(300)]
        rest_trades = [_make_trade_dict(400 + i) for i in range(700)]
        mock_parquet.return_value = parquet_trades
        mock_rest.return_value = rest_trades

        result = gap_fill_trades("BTCUSDT", 100, limit=1000)
        assert result["count"] == 1000
        assert result["partial"] is False
        # REST should continue from last Parquet trade + 1
        mock_rest.assert_called_once_with("BTCUSDT", 400, 700)

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_parquet_empty_falls_to_rest(self, mock_parquet, mock_rest):
        """Parquet returns 0, REST handles entire request."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.return_value = []
        mock_rest.return_value = [_make_trade_dict(100 + i) for i in range(500)]

        result = gap_fill_trades("BTCUSDT", 100, limit=500)
        assert result["count"] == 500
        mock_rest.assert_called_once_with("BTCUSDT", 100, 500)

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_rest_failure_after_partial_signals_partial(self, mock_parquet, mock_rest):
        """Parquet returns 300, REST raises → partial=True, not crash."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.return_value = [_make_trade_dict(100 + i) for i in range(300)]
        mock_rest.side_effect = ConnectionError("Binance API down")

        result = gap_fill_trades("BTCUSDT", 100, limit=1000)
        assert result["count"] == 300
        assert result["partial"] is True  # Consumer knows data is incomplete

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_both_fail_propagates(self, mock_parquet, mock_rest):
        """Both Parquet and REST fail → exception propagates."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.return_value = []
        mock_rest.side_effect = ConnectionError("Binance API down")

        with pytest.raises(ConnectionError):
            gap_fill_trades("BTCUSDT", 100, limit=1000)

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_limit_cap_rest_at_1000(self, mock_parquet, mock_rest):
        """REST limit capped at 1000 even when remaining > 1000."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.return_value = []
        mock_rest.return_value = []

        gap_fill_trades("BTCUSDT", 100, limit=3000)
        mock_rest.assert_called_once_with("BTCUSDT", 100, 1000)

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_overall_limit_cap(self, mock_parquet, mock_rest):
        """Limit capped at _OVERALL_LIMIT_CAP (6000) for direct callers."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.return_value = [_make_trade_dict(i) for i in range(5000)]
        mock_rest.return_value = [_make_trade_dict(5000 + i) for i in range(1000)]

        result = gap_fill_trades("BTCUSDT", 0, limit=50000)
        assert result["count"] == 6000
        # Parquet capped at 5000, REST capped at 1000
        mock_parquet.assert_called_once_with("BTCUSDT", 0, 5000)
        mock_rest.assert_called_once_with("BTCUSDT", 5000, 1000)

    def test_limit_zero_returns_empty(self):
        """limit=0 returns empty result without calling any tier."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        result = gap_fill_trades("BTCUSDT", 100, limit=0)
        assert result["trades"] == []
        assert result["count"] == 0
        assert result["partial"] is False

    def test_limit_negative_returns_empty(self):
        """Negative limit returns empty result."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        result = gap_fill_trades("BTCUSDT", 100, limit=-5)
        assert result["trades"] == []
        assert result["count"] == 0

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_response_format(self, mock_parquet, mock_rest):
        """Response is a dict with trades, partial, and count keys."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.return_value = [_make_trade_dict(1)]
        result = gap_fill_trades("BTCUSDT", 1, limit=1)
        assert "trades" in result
        assert "partial" in result
        assert "count" in result
        assert isinstance(result["trades"], list)
        assert isinstance(result["partial"], bool)
        assert isinstance(result["count"], int)

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_parquet_memory_error_propagates(self, mock_parquet, mock_rest):
        """MemoryError from Parquet scan propagates instead of being swallowed."""
        from opendeviationbar.sidecar_api import gap_fill_trades

        mock_parquet.side_effect = MemoryError("out of memory")
        with pytest.raises(MemoryError):
            gap_fill_trades("BTCUSDT", 100, limit=1000)


# ---------------------------------------------------------------------------
# Fix 3: Ariadne sources_tried diagnostics (#221)
# ---------------------------------------------------------------------------

class TestAriadneSourcesTried:
    """Tests for sources_tried in ariadne_lookup()."""

    def test_sources_tried_present(self):
        """sources_tried always present in response."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=None)
        assert "sources_tried" in result
        assert isinstance(result["sources_tried"], list)

    def test_source1_engine_none(self):
        """When engine is None, source 1 shows reason."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=None)

        source1 = next(s for s in result["sources_tried"] if s["source"] == "live_processor")
        assert source1["status"] == "skipped"
        assert source1["reason"] == "engine is None"

    def test_source1_no_match(self):
        """When engine has no matching forming bar, reason is 'no match'."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        engine = MagicMock()
        engine.get_forming_bars.return_value = [
            {"_symbol": "ETHUSDT", "_threshold": 250, "last_agg_trade_id": 999},
        ]

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=engine)

        source1 = next(s for s in result["sources_tried"] if s["source"] == "live_processor")
        assert source1["status"] == "skipped"
        assert source1["reason"] == "no match"

    def test_source1_hit(self):
        """When engine has matching forming bar, status is 'hit' with tid."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        engine = MagicMock()
        engine.get_forming_bars.return_value = [
            {"_symbol": "BTCUSDT", "_threshold": 250, "last_agg_trade_id": 12345},
        ]

        result = ariadne_lookup("BTCUSDT", 250, engine=engine)
        assert result["source"] == "live_processor"
        assert result["last_agg_trade_id"] == 12345

        source1 = next(s for s in result["sources_tried"] if s["source"] == "live_processor")
        assert source1["status"] == "hit"
        assert source1["tid"] == "12345"

    def test_source1_missing_get_forming_bars(self):
        """When engine lacks get_forming_bars, reason explains."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        engine = object()  # No get_forming_bars attribute

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=engine)

        source1 = next(s for s in result["sources_tried"] if s["source"] == "live_processor")
        assert source1["status"] == "skipped"
        assert source1["reason"] == "missing get_forming_bars"

    def test_source1_tid_zero(self):
        """When forming bar has tid=0, it's skipped with reason."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        engine = MagicMock()
        engine.get_forming_bars.return_value = [
            {"_symbol": "BTCUSDT", "_threshold": 250, "last_agg_trade_id": 0},
        ]

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=engine)

        source1 = next(s for s in result["sources_tried"] if s["source"] == "live_processor")
        assert source1["status"] == "skipped"
        assert "tid=" in source1["reason"]

    def test_source1_exception(self):
        """When engine.get_forming_bars() raises, source 1 records error."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        engine = MagicMock()
        engine.get_forming_bars.side_effect = RuntimeError("engine crashed")

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=engine)

        source1 = next(s for s in result["sources_tried"] if s["source"] == "live_processor")
        assert source1["status"] == "error"

    def test_source2_no_checkpoint(self):
        """When no checkpoint file exists, source 2 records 'no checkpoint'."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar.logger"):
                with patch("opendeviationbar.sidecar._load_checkpoint", return_value=None):
                    with patch("opendeviationbar.sidecar_api.logger"):
                        result = ariadne_lookup("BTCUSDT", 250, engine=None)

        source2 = next(s for s in result["sources_tried"] if s["source"] == "checkpoint_file")
        assert source2["status"] == "skipped"
        assert source2["reason"] == "no checkpoint"

    def test_source2_checkpoint_hit(self):
        """When checkpoint has valid tid, source 2 returns hit."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        cp = {"processor_checkpoint": {"last_agg_trade_id": 99999}}
        with patch("opendeviationbar.sidecar.logger"):
            with patch("opendeviationbar.sidecar._load_checkpoint", return_value=cp):
                result = ariadne_lookup("BTCUSDT", 250, engine=None)

        assert result["source"] == "checkpoint_file"
        assert result["last_agg_trade_id"] == 99999
        source2 = next(s for s in result["sources_tried"] if s["source"] == "checkpoint_file")
        assert source2["status"] == "hit"
        assert source2["tid"] == "99999"

    def test_degraded_has_all_sources(self):
        """When all sources fail, degraded response includes all 5+ sources_tried entries."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse") as mock_ch:
            # Mock to add CH sources_tried entries
            def side_effect(symbol, threshold, result, sources_tried):
                sources_tried.append({"source": "clickhouse_bars", "status": "error"})
                sources_tried.append({"source": "clickhouse_checkpoints", "status": "error"})
            mock_ch.side_effect = side_effect

            with patch("opendeviationbar.sidecar.logger"):
                with patch("opendeviationbar.sidecar._load_checkpoint", return_value=None):
                    with patch("opendeviationbar._core.fetch_latest_aggtrade", side_effect=ConnectionError("down")):
                        with patch("opendeviationbar.sidecar_api.logger"):
                            result = ariadne_lookup("BTCUSDT", 250, engine=None)

        assert result.get("degraded") is True
        sources = [s["source"] for s in result["sources_tried"]]
        assert "live_processor" in sources
        assert "checkpoint_file" in sources
        assert "clickhouse_bars" in sources
        assert "clickhouse_checkpoints" in sources
        assert "binance_rest" in sources


# ---------------------------------------------------------------------------
# Fix 4: Cross-file Parquet accumulation (#227)
# ---------------------------------------------------------------------------

class TestParquetCrossFileAccumulation:
    """Tests for _parquet_scan() cross-file accumulation."""

    def _make_parquet_file(self, tmp_path, name, trades):
        """Create a Parquet file with the given trades."""
        import polars as pl

        df = pl.DataFrame(
            {
                "agg_trade_id": [t[0] for t in trades],
                "price": [t[1] for t in trades],
                "quantity": [t[2] for t in trades],
                "timestamp": [t[3] for t in trades],
                "is_buyer_maker": [t[4] for t in trades],
            }
        )
        path = tmp_path / name
        df.write_parquet(path)
        return path

    @patch("opendeviationbar.storage.parquet.TickStorage")
    def test_cross_file_accumulation(self, mock_storage_cls, tmp_path):
        """File 1 has 3 trades, file 2 has 7 → returns all 10."""
        from opendeviationbar.sidecar_api import _parquet_scan

        # File 2 (older, sorted first reversed) has agg_ids 100-106
        self._make_parquet_file(tmp_path, "2024-01.parquet",
                                [(100 + i, 42000.0, 0.5, 1700000000000, False) for i in range(7)])
        # File 1 (newer) has agg_ids 107-109
        self._make_parquet_file(tmp_path, "2024-02.parquet",
                                [(107 + i, 42000.0, 0.5, 1700000000000, False) for i in range(3)])

        mock_storage = MagicMock()
        mock_storage._get_symbol_dir.return_value = tmp_path
        mock_storage_cls.return_value = mock_storage

        result = _parquet_scan("BTCUSDT", 100, 10)
        assert len(result) == 10
        assert result[0]["a"] == 100
        assert result[-1]["a"] == 109

    @patch("opendeviationbar.storage.parquet.TickStorage")
    def test_single_file_sufficient(self, mock_storage_cls, tmp_path):
        """First file has enough → returns from it."""
        from opendeviationbar.sidecar_api import _parquet_scan

        self._make_parquet_file(tmp_path, "2024-02.parquet",
                                [(100 + i, 42000.0, 0.5, 1700000000000, False) for i in range(20)])

        mock_storage = MagicMock()
        mock_storage._get_symbol_dir.return_value = tmp_path
        mock_storage_cls.return_value = mock_storage

        result = _parquet_scan("BTCUSDT", 100, 10)
        assert len(result) == 10

    @patch("opendeviationbar.storage.parquet.TickStorage")
    def test_dedup_at_boundary(self, mock_storage_cls, tmp_path):
        """Overlapping agg_trade_ids across files are deduplicated."""
        from opendeviationbar.sidecar_api import _parquet_scan

        # Both files have agg_id 105
        self._make_parquet_file(tmp_path, "2024-01.parquet",
                                [(100 + i, 42000.0, 0.5, 1700000000000, False) for i in range(6)])  # 100-105
        self._make_parquet_file(tmp_path, "2024-02.parquet",
                                [(105 + i, 42000.0, 0.5, 1700000000000, False) for i in range(5)])  # 105-109

        mock_storage = MagicMock()
        mock_storage._get_symbol_dir.return_value = tmp_path
        mock_storage_cls.return_value = mock_storage

        result = _parquet_scan("BTCUSDT", 100, 20)
        agg_ids = [t["a"] for t in result]
        assert len(agg_ids) == len(set(agg_ids)), "Duplicate agg_trade_ids found"
        assert agg_ids == sorted(agg_ids), "Not sorted ascending"

    @patch("opendeviationbar.storage.parquet.TickStorage")
    def test_sort_order(self, mock_storage_cls, tmp_path):
        """Trades from multiple files in ascending agg_trade_id order."""
        from opendeviationbar.sidecar_api import _parquet_scan

        self._make_parquet_file(tmp_path, "2024-01.parquet",
                                [(100 + i, 42000.0, 0.5, 1700000000000, False) for i in range(3)])
        self._make_parquet_file(tmp_path, "2024-02.parquet",
                                [(103 + i, 42000.0, 0.5, 1700000000000, False) for i in range(3)])

        mock_storage = MagicMock()
        mock_storage._get_symbol_dir.return_value = tmp_path
        mock_storage_cls.return_value = mock_storage

        result = _parquet_scan("BTCUSDT", 100, 10)
        agg_ids = [t["a"] for t in result]
        assert agg_ids == sorted(agg_ids)

    @patch("opendeviationbar.storage.parquet.TickStorage")
    def test_limit_respected(self, mock_storage_cls, tmp_path):
        """limit=5 with 10 available → exactly 5 returned."""
        from opendeviationbar.sidecar_api import _parquet_scan

        self._make_parquet_file(tmp_path, "2024-01.parquet",
                                [(100 + i, 42000.0, 0.5, 1700000000000, False) for i in range(10)])

        mock_storage = MagicMock()
        mock_storage._get_symbol_dir.return_value = tmp_path
        mock_storage_cls.return_value = mock_storage

        result = _parquet_scan("BTCUSDT", 100, 5)
        assert len(result) == 5

    @patch("opendeviationbar.storage.parquet.TickStorage")
    def test_all_files_empty(self, mock_storage_cls, tmp_path):
        """No matching trades → empty list."""
        from opendeviationbar.sidecar_api import _parquet_scan

        self._make_parquet_file(tmp_path, "2024-01.parquet",
                                [(1, 42000.0, 0.5, 1700000000000, False)])

        mock_storage = MagicMock()
        mock_storage._get_symbol_dir.return_value = tmp_path
        mock_storage_cls.return_value = mock_storage

        result = _parquet_scan("BTCUSDT", 999999, 10)
        assert result == []

    @patch("opendeviationbar.storage.parquet.TickStorage")
    def test_old_from_id_with_newer_files(self, mock_storage_cls, tmp_path):
        """from_agg_id=100 with newer file having IDs 500000+ returns trades from 100, not 500000."""
        from opendeviationbar.sidecar_api import _parquet_scan

        # Old file has IDs 100-199
        self._make_parquet_file(tmp_path, "2024-01.parquet",
                                [(100 + i, 42000.0, 0.5, 1700000000000, False) for i in range(100)])
        # Newer file has IDs 500000-500099
        self._make_parquet_file(tmp_path, "2024-06.parquet",
                                [(500000 + i, 42000.0, 0.5, 1700000000000, False) for i in range(100)])

        mock_storage = MagicMock()
        mock_storage._get_symbol_dir.return_value = tmp_path
        mock_storage_cls.return_value = mock_storage

        result = _parquet_scan("BTCUSDT", 100, 50)
        assert len(result) == 50
        # Must start from 100, NOT from 500000
        assert result[0]["a"] == 100
        assert result[-1]["a"] == 149

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_through_agg_id_matches_truncated_trades(self, mock_parquet, mock_rest):
        """through_agg_id matches last trade AFTER truncation."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.side_effect = [
            [_make_trade_dict(100 + i) for i in range(5000)],
            [],
        ]
        mock_rest.return_value = []

        result = gap_fill_trades_paginated("BTCUSDT", 100, max_trades=3000)
        assert result["count"] == 3000
        assert result["through_agg_id"] == result["trades"][-1]["a"]


# ---------------------------------------------------------------------------
# Fix 5: committed_only mode for ariadne (#225)
# ---------------------------------------------------------------------------

class TestAriadneCommittedOnly:
    """Tests for committed_only parameter in ariadne_lookup()."""

    def test_committed_only_skips_sources_1_2(self):
        """Sources 1-2 skipped with 'committed_only' reason."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=None, committed_only=True)

        for src_name in ("live_processor", "checkpoint_file"):
            src = next(s for s in result["sources_tried"] if s["source"] == src_name)
            assert src["status"] == "skipped"
            assert src["reason"] == "committed_only"

    def test_committed_only_false_default(self):
        """Default behavior unchanged (Sources 1-2 checked when engine=None)."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, engine=None)

        source1 = next(s for s in result["sources_tried"] if s["source"] == "live_processor")
        assert source1["reason"] != "committed_only"

    def test_committed_only_in_response(self):
        """committed_only key present in response dict."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result = ariadne_lookup("BTCUSDT", 250, committed_only=True)
        assert result["committed_only"] is True

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", return_value=None):
            with patch("opendeviationbar.sidecar_api.logger"):
                result2 = ariadne_lookup("BTCUSDT", 250, committed_only=False)
        assert result2["committed_only"] is False

    def test_committed_only_clickhouse_hit(self):
        """With committed_only=True, returns CH bar's tid."""
        from opendeviationbar.sidecar_api import ariadne_lookup

        def ch_side_effect(symbol, threshold, result, sources_tried):
            result["last_agg_trade_id"] = 77777
            result["source"] = "clickhouse_bars"
            sources_tried.append({"source": "clickhouse_bars", "status": "hit", "tid": "77777"})
            return result

        with patch("opendeviationbar.sidecar_api._ariadne_clickhouse", side_effect=ch_side_effect):
            result = ariadne_lookup("BTCUSDT", 250, committed_only=True)

        assert result["last_agg_trade_id"] == 77777
        assert result["source"] == "clickhouse_bars"
        assert result["committed_only"] is True


# ---------------------------------------------------------------------------
# Fix 6: Server-side paginated gap-fill (#226)
# ---------------------------------------------------------------------------

class TestGapFillPaginated:
    """Tests for gap_fill_trades_paginated()."""

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_parquet_then_rest(self, mock_parquet, mock_rest):
        """Parquet exhausted → REST continues."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.side_effect = [
            [_make_trade_dict(100 + i) for i in range(3000)],
            [],  # Parquet exhausted
        ]
        mock_rest.side_effect = [
            [_make_trade_dict(3100 + i) for i in range(1000)],
            [],  # REST exhausted
        ]

        result = gap_fill_trades_paginated("BTCUSDT", 100, max_trades=5000)
        assert result["count"] == 4000
        assert result["source_breakdown"]["parquet"] == 3000
        assert result["source_breakdown"]["rest"] == 1000

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_max_trades_cap(self, mock_parquet, mock_rest):
        """Stops at max_trades."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.return_value = [_make_trade_dict(100 + i) for i in range(5000)]

        result = gap_fill_trades_paginated("BTCUSDT", 100, max_trades=3000)
        assert result["count"] == 3000
        assert result["partial"] is True

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_hard_ceiling(self, mock_parquet, mock_rest):
        """max_trades clamped to _PAGINATED_MAX_TRADES."""
        from opendeviationbar.sidecar_api import (
            gap_fill_trades_paginated,
        )

        mock_parquet.return_value = []
        mock_rest.return_value = []

        result = gap_fill_trades_paginated("BTCUSDT", 100, max_trades=999999)
        # Should not crash; max_trades internally clamped
        assert result["count"] == 0

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_source_breakdown(self, mock_parquet, mock_rest):
        """Correct parquet/rest counts."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.side_effect = [
            [_make_trade_dict(100 + i) for i in range(200)],
            [],
        ]
        mock_rest.side_effect = [
            [_make_trade_dict(300 + i) for i in range(150)],
            [],
        ]

        result = gap_fill_trades_paginated("BTCUSDT", 100)
        assert result["source_breakdown"]["parquet"] == 200
        assert result["source_breakdown"]["rest"] == 150

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_batches_fetched(self, mock_parquet, mock_rest):
        """Correct batch count."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.side_effect = [
            [_make_trade_dict(100 + i) for i in range(100)],
            [],
        ]
        mock_rest.side_effect = [
            [_make_trade_dict(200 + i) for i in range(100)],
            [],
        ]

        result = gap_fill_trades_paginated("BTCUSDT", 100)
        assert result["batches_fetched"] == 2  # 1 parquet + 1 rest

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_through_agg_id(self, mock_parquet, mock_rest):
        """Last trade's agg_trade_id in response."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.side_effect = [
            [_make_trade_dict(100 + i) for i in range(5)],
            [],
        ]
        mock_rest.return_value = []

        result = gap_fill_trades_paginated("BTCUSDT", 100)
        assert result["through_agg_id"] == 104

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_empty_data(self, mock_parquet, mock_rest):
        """Both sources empty → count=0."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.return_value = []
        mock_rest.return_value = []

        result = gap_fill_trades_paginated("BTCUSDT", 100)
        assert result["count"] == 0
        assert result["through_agg_id"] is None
        assert result["partial"] is False

    @patch("opendeviationbar.sidecar_api._rest_fetch")
    @patch("opendeviationbar.sidecar_api._parquet_scan")
    def test_rest_failure_partial(self, mock_parquet, mock_rest):
        """REST fails mid-pagination → partial=True, Parquet data preserved."""
        from opendeviationbar.sidecar_api import gap_fill_trades_paginated

        mock_parquet.side_effect = [
            [_make_trade_dict(100 + i) for i in range(500)],
            [],
        ]
        mock_rest.side_effect = ConnectionError("Binance down")

        result = gap_fill_trades_paginated("BTCUSDT", 100)
        assert result["count"] == 500
        assert result["partial"] is True
        assert result["source_breakdown"]["parquet"] == 500


# ---------------------------------------------------------------------------
# Fix 7: Catchup composite endpoint (#228)
# ---------------------------------------------------------------------------

class TestCatchupLookup:
    """Tests for catchup_lookup()."""

    @patch("opendeviationbar.sidecar_api.gap_fill_trades_paginated")
    @patch("opendeviationbar.sidecar_api.ariadne_lookup")
    def test_happy_path(self, mock_ariadne, mock_gap_fill):
        """Ariadne returns tid → paginated gap-fill → composite response."""
        from opendeviationbar.sidecar_api import catchup_lookup

        mock_ariadne.return_value = {
            "last_agg_trade_id": 50000,
            "source": "clickhouse_bars",
            "sources_tried": [{"source": "clickhouse_bars", "status": "hit"}],
        }
        mock_gap_fill.return_value = {
            "trades": [_make_trade_dict(50001 + i) for i in range(100)],
            "count": 100,
            "partial": False,
            "from_agg_id": 50001,
            "through_agg_id": 50100,
            "source_breakdown": {"parquet": 100, "rest": 0},
            "batches_fetched": 1,
        }

        result = catchup_lookup("BTCUSDT", 250)
        assert result["last_committed_bar"]["last_agg_trade_id"] == 50000
        assert result["count"] == 100
        assert result["from_agg_id"] == 50001
        assert result["through_agg_id"] == 50100
        assert len(result["ariadne_sources_tried"]) > 0

    @patch("opendeviationbar.sidecar_api.ariadne_lookup")
    def test_ariadne_degraded(self, mock_ariadne):
        """All sources fail → error with diagnostics."""
        from opendeviationbar.sidecar_api import catchup_lookup

        mock_ariadne.return_value = {
            "last_agg_trade_id": None,
            "source": None,
            "degraded": True,
            "sources_tried": [{"source": "clickhouse_bars", "status": "error"}],
        }

        result = catchup_lookup("BTCUSDT", 250)
        assert result["count"] == 0
        assert "error" in result
        assert result["last_committed_bar"]["last_agg_trade_id"] is None
        assert len(result["ariadne_sources_tried"]) > 0

    @patch("opendeviationbar.sidecar_api.gap_fill_trades_paginated")
    @patch("opendeviationbar.sidecar_api.ariadne_lookup")
    def test_max_trades_passthrough(self, mock_ariadne, mock_gap_fill):
        """max_trades correctly forwarded."""
        from opendeviationbar.sidecar_api import catchup_lookup

        mock_ariadne.return_value = {
            "last_agg_trade_id": 1000,
            "source": "clickhouse_bars",
            "sources_tried": [],
        }
        mock_gap_fill.return_value = {
            "trades": [], "count": 0, "partial": False,
            "from_agg_id": 1001, "through_agg_id": None,
            "source_breakdown": {"parquet": 0, "rest": 0}, "batches_fetched": 0,
        }

        catchup_lookup("BTCUSDT", 250, max_trades=10000)
        mock_gap_fill.assert_called_once_with("BTCUSDT", 1001, max_trades=10000)

    @patch("opendeviationbar.sidecar_api.ariadne_lookup")
    def test_committed_only_enforced(self, mock_ariadne):
        """Ariadne called with committed_only=True."""
        from opendeviationbar.sidecar_api import catchup_lookup

        mock_ariadne.return_value = {
            "last_agg_trade_id": None,
            "source": None,
            "sources_tried": [],
        }

        catchup_lookup("BTCUSDT", 250)
        mock_ariadne.assert_called_once_with("BTCUSDT", 250, engine=None, committed_only=True)

    def test_validation_errors(self):
        """Invalid symbol/threshold → ValueError."""
        from opendeviationbar.sidecar_api import catchup_lookup

        with pytest.raises(ValueError, match="invalid symbol"):
            catchup_lookup("invalid!", 250)
        with pytest.raises(ValueError, match="threshold must be positive"):
            catchup_lookup("BTCUSDT", -1)

    @patch("opendeviationbar.sidecar_api.gap_fill_trades_paginated")
    @patch("opendeviationbar.sidecar_api.ariadne_lookup")
    def test_response_shape(self, mock_ariadne, mock_gap_fill):
        """Response has all expected keys."""
        from opendeviationbar.sidecar_api import catchup_lookup

        mock_ariadne.return_value = {
            "last_agg_trade_id": 5000, "source": "clickhouse_bars", "sources_tried": [],
        }
        mock_gap_fill.return_value = {
            "trades": [], "count": 0, "partial": False,
            "from_agg_id": 5001, "through_agg_id": None,
            "source_breakdown": {"parquet": 0, "rest": 0}, "batches_fetched": 0,
        }

        result = catchup_lookup("BTCUSDT", 250)
        expected_keys = {
            "symbol", "threshold_dbps", "last_committed_bar", "trades",
            "from_agg_id", "through_agg_id", "count", "partial",
            "source_breakdown", "batches_fetched", "ariadne_sources_tried",
        }
        assert expected_keys.issubset(result.keys())
