"""
tests/test_circuit_breaker.py — Circuit breaker tests (#347).

Tests cover:
    - Error classification (classify_error)
    - Breaker opens after threshold consecutive quota/rate-limit failures
    - Breaker does NOT open on transient/model errors
    - Auto-recovery: OPEN → HALF_OPEN after cooldown
    - Read operations are ALWAYS allowed (degraded mode)
    - Writes fail explicitly (CircuitBreakerOpenError) when breaker is open
    - Writes are allowed again after cooldown + success
    - Integration: exhaustion + recovery cycle
    - Config fields loaded from defaults.toml
"""
from __future__ import annotations

import time
from unittest.mock import patch

import pytest

from cortex.llm.circuit_breaker import (
    BreakerState,
    CircuitBreaker,
    CircuitBreakerOpenError,
    FailureType,
    classify_error,
)


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


class _FakeException(Exception):
    """Exception with a configurable status_code attribute."""

    def __init__(self, msg: str, status_code: int | None = None) -> None:
        super().__init__(msg)
        self.status_code = status_code


def _make_429(msg: str = "rate limit exceeded") -> Exception:
    return _FakeException(msg, status_code=429)


def _make_quota() -> Exception:
    return _FakeException("You exceeded your quota (insufficient_quota)", status_code=429)


def _make_500() -> Exception:
    return _FakeException("internal server error", status_code=500)


def _make_404() -> Exception:
    return _FakeException("model not found", status_code=404)


def _make_connection_error() -> Exception:
    return ConnectionError("connection reset by peer")


# ---------------------------------------------------------------------------
# classify_error
# ---------------------------------------------------------------------------


class TestClassifyError:
    def test_429_is_rate_limited(self):
        assert classify_error(_make_429()) == FailureType.RATE_LIMITED

    def test_429_with_quota_keywords_is_quota_exhausted(self):
        exc = _FakeException("rate limit: you exceeded your quota", status_code=429)
        assert classify_error(exc) == FailureType.QUOTA_EXHAUSTED

    def test_insufficient_quota_is_quota_exhausted(self):
        exc = _FakeException("insufficient_quota: billing limit reached")
        assert classify_error(exc) == FailureType.QUOTA_EXHAUSTED

    def test_billing_keyword_is_quota_exhausted(self):
        exc = _FakeException("your billing limit has been exceeded")
        assert classify_error(exc) == FailureType.QUOTA_EXHAUSTED

    def test_usage_limit_keyword_is_quota_exhausted(self):
        exc = _FakeException("usage limit exceeded for this month")
        assert classify_error(exc) == FailureType.QUOTA_EXHAUSTED

    def test_500_is_transient(self):
        assert classify_error(_make_500()) == FailureType.TRANSIENT

    def test_502_is_transient(self):
        exc = _FakeException("bad gateway", status_code=502)
        assert classify_error(exc) == FailureType.TRANSIENT

    def test_503_is_transient(self):
        exc = _FakeException("service unavailable", status_code=503)
        assert classify_error(exc) == FailureType.TRANSIENT

    def test_connection_error_is_transient(self):
        assert classify_error(_make_connection_error()) == FailureType.TRANSIENT

    def test_timeout_error_is_transient(self):
        assert classify_error(TimeoutError("request timed out")) == FailureType.TRANSIENT

    def test_timeout_keyword_is_transient(self):
        assert classify_error(Exception("request timeout after 30s")) == FailureType.TRANSIENT

    def test_404_model_not_found_is_model_error(self):
        assert classify_error(_make_404()) == FailureType.MODEL_ERROR

    def test_400_is_model_error(self):
        exc = _FakeException("invalid model specification", status_code=400)
        assert classify_error(exc) == FailureType.MODEL_ERROR

    def test_model_not_found_keyword_is_model_error(self):
        exc = _FakeException("model not found: gpt-5-turbo")
        assert classify_error(exc) == FailureType.MODEL_ERROR

    def test_unknown_exception_defaults_to_transient(self):
        assert classify_error(ValueError("something unexpected")) == FailureType.TRANSIENT

    def test_rate_limit_in_message_without_status_code(self):
        exc = Exception("error: rate limit reached, please wait")
        assert classify_error(exc) == FailureType.RATE_LIMITED


# ---------------------------------------------------------------------------
# CircuitBreaker — basic state machine
# ---------------------------------------------------------------------------


class TestCircuitBreakerBasic:
    def test_initial_state_is_closed(self):
        cb = CircuitBreaker(threshold=3)
        assert cb.get_state("openai") == BreakerState.CLOSED

    def test_unknown_provider_starts_closed(self):
        cb = CircuitBreaker()
        assert not cb.is_open("anthropic")
        assert cb.get_state("anthropic") == BreakerState.CLOSED

    def test_allow_write_when_closed(self):
        cb = CircuitBreaker()
        assert cb.allow_write("openai") is True

    def test_allow_read_always_true(self):
        cb = CircuitBreaker()
        assert cb.allow_read("openai") is True

    def test_disabled_breaker_never_opens(self):
        cb = CircuitBreaker(enabled=False, threshold=1)
        exc = _make_429()
        for _ in range(10):
            cb.record_failure("openai", exc)
        assert not cb.is_open("openai")
        assert cb.allow_write("openai") is True

    def test_assert_write_allowed_passes_when_closed(self):
        cb = CircuitBreaker()
        # Should not raise
        cb.assert_write_allowed("openai")

    def test_record_success_on_fresh_provider_is_noop(self):
        cb = CircuitBreaker()
        cb.record_success("openai")
        assert cb.get_state("openai") == BreakerState.CLOSED


# ---------------------------------------------------------------------------
# Breaker opens after threshold
# ---------------------------------------------------------------------------


class TestBreakerOpensAfterThreshold:
    def test_opens_after_threshold_rate_limit_failures(self):
        cb = CircuitBreaker(threshold=5)
        exc = _make_429()
        for i in range(4):
            cb.record_failure("openai", exc)
            assert cb.get_state("openai") == BreakerState.CLOSED, f"opened too early at {i+1}"
        cb.record_failure("openai", exc)
        assert cb.get_state("openai") == BreakerState.OPEN

    def test_opens_after_threshold_quota_failures(self):
        cb = CircuitBreaker(threshold=3)
        exc = _make_quota()
        for _ in range(3):
            cb.record_failure("anthropic", exc)
        assert cb.get_state("anthropic") == BreakerState.OPEN

    def test_opens_at_threshold_exactly(self):
        cb = CircuitBreaker(threshold=2)
        exc = _make_429()
        cb.record_failure("openai", exc)
        assert not cb.is_open("openai")
        cb.record_failure("openai", exc)
        assert cb.is_open("openai")

    def test_transient_failures_do_not_trip_breaker(self):
        cb = CircuitBreaker(threshold=3)
        exc = _make_500()
        for _ in range(100):
            cb.record_failure("openai", exc)
        assert not cb.is_open("openai")
        assert cb.get_state("openai") == BreakerState.CLOSED

    def test_model_errors_do_not_trip_breaker(self):
        cb = CircuitBreaker(threshold=3)
        exc = _make_404()
        for _ in range(50):
            cb.record_failure("openai", exc)
        assert not cb.is_open("openai")

    def test_connection_errors_do_not_trip_breaker(self):
        cb = CircuitBreaker(threshold=3)
        exc = _make_connection_error()
        for _ in range(20):
            cb.record_failure("openai", exc)
        assert not cb.is_open("openai")

    def test_breakers_are_isolated_per_provider(self):
        cb = CircuitBreaker(threshold=3)
        exc = _make_429()
        for _ in range(3):
            cb.record_failure("openai", exc)
        # openai is open, anthropic should still be closed
        assert cb.is_open("openai")
        assert not cb.is_open("anthropic")
        assert cb.get_state("anthropic") == BreakerState.CLOSED

    def test_threshold_one_opens_immediately(self):
        cb = CircuitBreaker(threshold=1)
        cb.record_failure("openai", _make_429())
        assert cb.is_open("openai")

    def test_mixed_failures_only_trip_types_count(self):
        """Transient failures interspersed with rate-limit failures.
        Only the rate-limit ones count toward the threshold.
        """
        cb = CircuitBreaker(threshold=3)
        for _ in range(10):
            cb.record_failure("openai", _make_500())   # transient, ignored
        assert not cb.is_open("openai")
        for _ in range(2):
            cb.record_failure("openai", _make_429())   # 2 rate-limit
        assert not cb.is_open("openai")
        cb.record_failure("openai", _make_429())        # 3rd rate-limit → OPEN
        assert cb.is_open("openai")


# ---------------------------------------------------------------------------
# Writes blocked when open, reads still work
# ---------------------------------------------------------------------------


class TestDegradedMode:
    def _open_breaker(self, cb: CircuitBreaker, provider: str = "openai") -> None:
        """Helper: force breaker open by hitting threshold."""
        exc = _make_429()
        for _ in range(cb.threshold):
            cb.record_failure(provider, exc)

    def test_writes_blocked_when_open(self):
        cb = CircuitBreaker(threshold=3)
        self._open_breaker(cb)
        assert cb.allow_write("openai") is False

    def test_reads_still_allowed_when_open(self):
        cb = CircuitBreaker(threshold=3)
        self._open_breaker(cb)
        assert cb.allow_read("openai") is True

    def test_reads_allowed_for_all_providers_always(self):
        """Reads are always True regardless of breaker state."""
        cb = CircuitBreaker(threshold=1)
        cb.record_failure("openai", _make_429())
        assert cb.allow_read("openai") is True
        assert cb.allow_read("anthropic") is True
        assert cb.allow_read("ollama") is True

    def test_assert_write_raises_when_open(self):
        cb = CircuitBreaker(threshold=3)
        self._open_breaker(cb)
        with pytest.raises(CircuitBreakerOpenError) as exc_info:
            cb.assert_write_allowed("openai")
        err = str(exc_info.value)
        assert "openai" in err
        assert "OPEN" in err
        assert "retrieval" in err.lower() or "read" in err.lower()

    def test_write_error_message_content(self):
        cb = CircuitBreaker(threshold=3, cooldown_sec=300)
        self._open_breaker(cb)
        msg = cb.write_error("openai")
        assert "openai" in msg
        assert "OPEN" in msg
        assert "rate_limited" in msg or "quota" in msg
        assert "retrieval" in msg.lower() or "read" in msg.lower()
        assert "300" in msg or "s" in msg  # cooldown mentioned

    def test_write_error_message_contains_failure_count(self):
        cb = CircuitBreaker(threshold=3)
        self._open_breaker(cb)
        msg = cb.write_error("openai")
        assert "3" in msg

    def test_circuit_breaker_open_error_is_exception(self):
        """CircuitBreakerOpenError must be catchable as Exception."""
        with pytest.raises(Exception):
            raise CircuitBreakerOpenError("test error")

    def test_write_blocked_for_open_provider_only(self):
        cb = CircuitBreaker(threshold=3)
        self._open_breaker(cb, provider="openai")
        # openai is blocked, anthropic is not
        assert cb.allow_write("openai") is False
        assert cb.allow_write("anthropic") is True


# ---------------------------------------------------------------------------
# Auto-recovery after cooldown
# ---------------------------------------------------------------------------


class TestAutoRecovery:
    def _open_breaker(self, cb: CircuitBreaker, provider: str = "openai") -> None:
        exc = _make_429()
        for _ in range(cb.threshold):
            cb.record_failure(provider, exc)

    def test_stays_open_before_cooldown(self):
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)
        self._open_breaker(cb)
        assert cb.get_state("openai") == BreakerState.OPEN
        # Time hasn't advanced — still open
        with patch("time.monotonic", return_value=time.monotonic() + 30):
            assert cb.get_state("openai") == BreakerState.OPEN

    def test_transitions_to_half_open_after_cooldown(self):
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)
        self._open_breaker(cb)
        assert cb.get_state("openai") == BreakerState.OPEN

        # Simulate cooldown elapsed by patching time.monotonic
        future_time = time.monotonic() + 61
        with patch("time.monotonic", return_value=future_time):
            state = cb.get_state("openai")
        assert state == BreakerState.HALF_OPEN

    def test_writes_allowed_in_half_open(self):
        """HALF_OPEN allows one probe write (is_open returns False)."""
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)
        self._open_breaker(cb)

        future_time = time.monotonic() + 61
        with patch("time.monotonic", return_value=future_time):
            assert cb.allow_write("openai") is True
            assert not cb.is_open("openai")

    def test_success_in_half_open_closes_breaker(self):
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)
        self._open_breaker(cb)

        # Advance time past cooldown
        future_time = time.monotonic() + 61
        with patch("time.monotonic", return_value=future_time):
            cb.get_state("openai")  # trigger HALF_OPEN transition

        assert cb.get_state("openai") == BreakerState.HALF_OPEN
        cb.record_success("openai")
        assert cb.get_state("openai") == BreakerState.CLOSED

    def test_failure_in_half_open_reopens_breaker(self):
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)
        self._open_breaker(cb)

        future_time = time.monotonic() + 61
        with patch("time.monotonic", return_value=future_time):
            cb.get_state("openai")  # trigger HALF_OPEN

        # A quota failure in HALF_OPEN should re-open immediately
        cb.record_failure("openai", _make_429())
        # consecutive_failures is now 1 (reset would need to hit threshold again)
        # After 1 failure in half-open + threshold=3, not immediately open
        # But if we hit threshold again:
        for _ in range(2):
            cb.record_failure("openai", _make_429())
        assert cb.get_state("openai") == BreakerState.OPEN

    def test_success_resets_failure_count(self):
        cb = CircuitBreaker(threshold=5)
        exc = _make_429()
        # Hit 4 failures (one short of threshold)
        for _ in range(4):
            cb.record_failure("openai", exc)
        # Success resets count
        cb.record_success("openai")
        assert cb.get_state("openai") == BreakerState.CLOSED
        # Now 4 more failures should NOT open (count was reset)
        for _ in range(4):
            cb.record_failure("openai", exc)
        assert not cb.is_open("openai")

    def test_is_open_false_after_half_open_transition(self):
        cb = CircuitBreaker(threshold=3, cooldown_sec=1)
        self._open_breaker(cb)

        future_time = time.monotonic() + 2
        with patch("time.monotonic", return_value=future_time):
            assert cb.is_open("openai") is False  # HALF_OPEN ≠ OPEN


# ---------------------------------------------------------------------------
# Integration: exhaustion + recovery cycle
# ---------------------------------------------------------------------------


class TestIntegrationExhaustionRecovery:
    def test_full_cycle_quota_exhaustion_to_recovery(self):
        """Full cycle: quota exhausted → breaker opens → cooldown → recovery."""
        cb = CircuitBreaker(threshold=5, cooldown_sec=300)

        # Phase 1: Normal writes work
        assert cb.allow_write("openai") is True

        # Phase 2: Quota errors accumulate
        quota_err = _FakeException("insufficient_quota billing limit", status_code=429)
        for _ in range(5):
            cb.record_failure("openai", quota_err)

        # Phase 3: Breaker open — writes blocked
        assert cb.is_open("openai")
        assert cb.allow_write("openai") is False
        assert cb.allow_read("openai") is True  # degraded mode: reads work

        with pytest.raises(CircuitBreakerOpenError):
            cb.assert_write_allowed("openai")

        # Phase 4: Cooldown elapses
        future_time = time.monotonic() + 301
        with patch("time.monotonic", return_value=future_time):
            assert cb.get_state("openai") == BreakerState.HALF_OPEN
            assert cb.allow_write("openai") is True  # probe allowed

        # Phase 5: Successful probe → fully recovered
        cb.record_success("openai")
        assert cb.get_state("openai") == BreakerState.CLOSED
        assert cb.allow_write("openai") is True

    def test_reads_uninterrupted_throughout_full_cycle(self):
        """Reads should work in ALL states: CLOSED, OPEN, HALF_OPEN."""
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)
        provider = "anthropic"

        # CLOSED
        assert cb.allow_read(provider) is True

        # OPEN
        for _ in range(3):
            cb.record_failure(provider, _make_quota())
        assert cb.get_state(provider) == BreakerState.OPEN
        assert cb.allow_read(provider) is True

        # HALF_OPEN
        future_time = time.monotonic() + 61
        with patch("time.monotonic", return_value=future_time):
            assert cb.get_state(provider) == BreakerState.HALF_OPEN
            assert cb.allow_read(provider) is True

        # CLOSED again
        cb.record_success(provider)
        assert cb.allow_read(provider) is True

    def test_multiple_providers_independent_cycles(self):
        """Three providers: one open, one recovering, one healthy."""
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)

        # openai: hit threshold → OPEN
        for _ in range(3):
            cb.record_failure("openai", _make_429())

        # anthropic: 2 failures (below threshold)
        for _ in range(2):
            cb.record_failure("anthropic", _make_quota())

        # ollama: healthy

        assert cb.is_open("openai")
        assert not cb.is_open("anthropic")
        assert not cb.is_open("ollama")

        # Reads work for all three
        assert cb.allow_read("openai") is True
        assert cb.allow_read("anthropic") is True
        assert cb.allow_read("ollama") is True

        # Writes blocked only for openai
        assert cb.allow_write("openai") is False
        assert cb.allow_write("anthropic") is True
        assert cb.allow_write("ollama") is True

    def test_repeated_failures_after_recovery_reopens_breaker(self):
        """After recovery, new quota errors should re-open the breaker."""
        cb = CircuitBreaker(threshold=3, cooldown_sec=60)

        # First cycle: open → recover
        for _ in range(3):
            cb.record_failure("openai", _make_429())
        assert cb.is_open("openai")

        future_time = time.monotonic() + 61
        with patch("time.monotonic", return_value=future_time):
            cb.get_state("openai")  # transition to HALF_OPEN
        cb.record_success("openai")
        assert cb.get_state("openai") == BreakerState.CLOSED

        # Second cycle: quota hits again
        for _ in range(3):
            cb.record_failure("openai", _make_quota())
        assert cb.is_open("openai")


# ---------------------------------------------------------------------------
# Reset / maintenance
# ---------------------------------------------------------------------------


class TestReset:
    def test_reset_closes_open_breaker(self):
        cb = CircuitBreaker(threshold=3)
        for _ in range(3):
            cb.record_failure("openai", _make_429())
        assert cb.is_open("openai")
        cb.reset("openai")
        assert not cb.is_open("openai")
        assert cb.get_state("openai") == BreakerState.CLOSED

    def test_reset_all_closes_all_breakers(self):
        cb = CircuitBreaker(threshold=3)
        for provider in ("openai", "anthropic"):
            for _ in range(3):
                cb.record_failure(provider, _make_429())
        assert cb.is_open("openai")
        assert cb.is_open("anthropic")
        cb.reset_all()
        assert not cb.is_open("openai")
        assert not cb.is_open("anthropic")

    def test_reset_unknown_provider_is_noop(self):
        cb = CircuitBreaker()
        cb.reset("nonexistent")  # should not raise

    def test_status_returns_correct_info(self):
        cb = CircuitBreaker(threshold=3)
        for _ in range(3):
            cb.record_failure("openai", _make_quota())
        status = cb.status()
        assert "openai" in status
        assert status["openai"]["state"] == "open"
        assert status["openai"]["consecutive_failures"] == 3
        assert status["openai"]["last_failure_type"] == "quota_exhausted"

    def test_status_empty_when_no_providers_tracked(self):
        cb = CircuitBreaker()
        assert cb.status() == {}


# ---------------------------------------------------------------------------
# Config fields from defaults.toml
# ---------------------------------------------------------------------------


class TestConfigFields:
    def test_circuit_breaker_fields_in_config(self):
        """CortexConfig must load circuit breaker fields from defaults.toml."""
        from cortex.config import CortexConfig

        cfg = CortexConfig.defaults()
        assert hasattr(cfg, "circuit_breaker_enabled")
        assert hasattr(cfg, "circuit_breaker_threshold")
        assert hasattr(cfg, "circuit_breaker_cooldown_sec")

    def test_circuit_breaker_defaults(self):
        """Verify the default values are sensible."""
        from cortex.config import CortexConfig

        cfg = CortexConfig.defaults()
        assert cfg.circuit_breaker_enabled is True
        assert cfg.circuit_breaker_threshold == 5
        assert cfg.circuit_breaker_cooldown_sec == 300

    def test_circuit_breaker_config_overlay(self):
        """Config overlay correctly overrides circuit breaker settings."""
        from cortex.config import CortexConfig

        cfg = CortexConfig.from_toml(overlay={
            "cortex": {
                "circuit_breaker": {
                    "enabled": False,
                    "threshold": 10,
                    "cooldown_sec": 600,
                }
            }
        })
        assert cfg.circuit_breaker_enabled is False
        assert cfg.circuit_breaker_threshold == 10
        assert cfg.circuit_breaker_cooldown_sec == 600

    def test_circuit_breaker_from_config(self):
        """Can build a CircuitBreaker directly from CortexConfig fields."""
        from cortex.config import CortexConfig

        cfg = CortexConfig.defaults()
        cb = CircuitBreaker(
            enabled=cfg.circuit_breaker_enabled,
            threshold=cfg.circuit_breaker_threshold,
            cooldown_sec=cfg.circuit_breaker_cooldown_sec,
        )
        assert cb.enabled is True
        assert cb.threshold == 5
        assert cb.cooldown_sec == 300
