"""Tests for rule-based failure pattern detectors."""

from __future__ import annotations

from evalkit.models import EvaluationResult, MetricScore, ToolCall
from evalkit.models import TestCase as EvalTestCase
from evalkit.reports.patterns import (
    detect_empty_context,
    detect_long_context,
    detect_score_cliff,
    detect_short_answer,
    detect_tool_name_mismatch,
    run_all_detectors,
)


def _tc(**kwargs) -> EvalTestCase:
    defaults: dict = {"question": "Q?", "context": ["chunk"], "answer": "A."}
    defaults.update(kwargs)
    return EvalTestCase(**defaults)


def _fail(tc: EvalTestCase, score: float = 0.2) -> tuple[float, EvaluationResult]:
    ms = MetricScore(metric_name="test", score=score, reason="r", passed=False)
    return (score, EvaluationResult(test_case=tc, scores=[ms]))


class TestDetectLongContext:
    def test_fires_when_majority_have_long_context(self):
        failures = [_fail(_tc(context=[f"c{i}" for i in range(7)])) for _ in range(4)] + [_fail(_tc(context=["short"]))]
        result = detect_long_context("faithfulness", failures, 0.5)
        assert result is not None
        assert ">5 chunks" in result

    def test_does_not_fire_below_threshold(self):
        failures = [
            _fail(_tc(context=[f"c{i}" for i in range(7)])),
            _fail(_tc(context=["short"])),
            _fail(_tc(context=["short"])),
            _fail(_tc(context=["short"])),
        ]
        result = detect_long_context("faithfulness", failures, 0.5)
        assert result is None

    def test_returns_none_for_empty_list(self):
        result = detect_long_context("faithfulness", [], 0.5)
        assert result is None

    def test_exactly_5_chunks_does_not_count(self):
        failures = [_fail(_tc(context=[f"c{i}" for i in range(5)])) for _ in range(5)]
        result = detect_long_context("faithfulness", failures, 0.5)
        assert result is None


class TestDetectShortAnswer:
    def test_fires_when_answers_are_terse(self):
        failures = [
            _fail(_tc(answer="Yes", expected_answer="The capital of France is Paris, a large city.")),
            _fail(_tc(answer="No", expected_answer="The economy has been growing steadily over years.")),
            _fail(_tc(answer="OK", expected_answer="Machine learning is a subset of artificial intelligence.")),
        ]
        result = detect_short_answer("completeness", failures, 0.5)
        assert result is not None
        assert "terse" in result

    def test_does_not_fire_with_long_answers(self):
        long_answer = "This is a reasonably detailed answer that covers the topic."
        failures = [
            _fail(_tc(answer=long_answer, expected_answer="This is the expected answer text.")),
            _fail(_tc(answer=long_answer, expected_answer="Another expected answer here.")),
            _fail(_tc(answer=long_answer, expected_answer="Yet another expected one.")),
        ]
        result = detect_short_answer("completeness", failures, 0.5)
        assert result is None

    def test_skips_cases_without_expected_answer(self):
        failures = [
            _fail(_tc(answer="Yes", expected_answer=None)),
            _fail(_tc(answer="No", expected_answer=None)),
            _fail(_tc(answer="OK", expected_answer="A long expected answer for comparison.")),
        ]
        result = detect_short_answer("completeness", failures, 0.5)
        assert result is None

    def test_requires_at_least_3_cases(self):
        failures = [
            _fail(_tc(answer="Y", expected_answer="A very long expected answer text.")),
            _fail(_tc(answer="N", expected_answer="Another very long expected answer.")),
        ]
        result = detect_short_answer("completeness", failures, 0.5)
        assert result is None


class TestDetectEmptyContext:
    def test_fires_when_majority_have_empty_context(self):
        failures = [
            _fail(_tc(context=[])),
            _fail(_tc(context=[])),
            _fail(_tc(context=[])),
            _fail(_tc(context=["has context"])),
        ]
        result = detect_empty_context("faithfulness", failures, 0.5)
        assert result is not None
        assert "empty context" in result

    def test_does_not_fire_when_most_have_context(self):
        failures = [
            _fail(_tc(context=[])),
            _fail(_tc(context=["c1"])),
            _fail(_tc(context=["c2"])),
            _fail(_tc(context=["c3"])),
        ]
        result = detect_empty_context("faithfulness", failures, 0.5)
        assert result is None

    def test_returns_none_for_empty_list(self):
        result = detect_empty_context("faithfulness", [], 0.5)
        assert result is None

    def test_fires_at_exactly_above_50_percent(self):
        failures = [
            _fail(_tc(context=[])),
            _fail(_tc(context=[])),
            _fail(_tc(context=["c"])),
        ]
        result = detect_empty_context("faithfulness", failures, 0.5)
        assert result is not None


class TestDetectScoreCliff:
    def test_fires_on_bimodal_distribution(self):
        failures = [
            _fail(_tc(), score=0.1),
            _fail(_tc(), score=0.05),
            _fail(_tc(), score=0.15),
        ]
        result = detect_score_cliff("faithfulness", failures, 0.8)
        assert result is not None
        assert "Bimodal" in result
        assert "0.80" in result

    def test_does_not_fire_when_overall_avg_low(self):
        failures = [_fail(_tc(), score=0.1)]
        result = detect_score_cliff("faithfulness", failures, 0.5)
        assert result is None

    def test_does_not_fire_when_failure_avg_not_low_enough(self):
        failures = [
            _fail(_tc(), score=0.3),
            _fail(_tc(), score=0.4),
        ]
        result = detect_score_cliff("faithfulness", failures, 0.8)
        assert result is None

    def test_returns_none_for_empty_failures(self):
        result = detect_score_cliff("faithfulness", [], 0.9)
        assert result is None


class TestDetectToolNameMismatch:
    def test_fires_for_tool_selection_metric(self):
        tc = _tc(
            expected_tool_calls=[ToolCall(name="search", parameters={})],
            actual_tool_calls=[ToolCall(name="lookup", parameters={})],
        )
        failures = [_fail(tc)]
        result = detect_tool_name_mismatch("tool_selection", failures, 0.5)
        assert result is not None
        assert "lookup instead of search" in result

    def test_ignores_non_tool_selection_metric(self):
        tc = _tc(
            expected_tool_calls=[ToolCall(name="search", parameters={})],
            actual_tool_calls=[ToolCall(name="lookup", parameters={})],
        )
        failures = [_fail(tc)]
        result = detect_tool_name_mismatch("faithfulness", failures, 0.5)
        assert result is None

    def test_no_mismatches_when_names_match(self):
        tc = _tc(
            expected_tool_calls=[ToolCall(name="search", parameters={})],
            actual_tool_calls=[ToolCall(name="search", parameters={})],
        )
        failures = [_fail(tc)]
        result = detect_tool_name_mismatch("tool_selection", failures, 0.5)
        assert result is None

    def test_handles_missing_tool_calls(self):
        tc = _tc(expected_tool_calls=None, actual_tool_calls=None)
        failures = [_fail(tc)]
        result = detect_tool_name_mismatch("tool_selection", failures, 0.5)
        assert result is None

    def test_limits_to_top_3_mismatches(self):
        failures = []
        for i in range(5):
            tc = _tc(
                expected_tool_calls=[ToolCall(name=f"expected_{i}", parameters={})],
                actual_tool_calls=[ToolCall(name=f"actual_{i}", parameters={})],
            )
            failures.append(_fail(tc))
        result = detect_tool_name_mismatch("tool_selection", failures, 0.5)
        assert result is not None
        assert result.count("instead of") <= 3


class TestRunAllDetectors:
    def test_returns_empty_for_no_patterns(self):
        failures = [_fail(_tc(), score=0.4)]
        patterns = run_all_detectors("faithfulness", failures, 0.5)
        assert patterns == []

    def test_returns_multiple_patterns(self):
        failures = [
            _fail(_tc(context=[]), score=0.1),
            _fail(_tc(context=[]), score=0.05),
            _fail(_tc(context=[]), score=0.15),
        ]
        patterns = run_all_detectors("faithfulness", failures, 0.8)
        assert len(patterns) >= 2
        has_empty = any("empty context" in p for p in patterns)
        has_cliff = any("Bimodal" in p for p in patterns)
        assert has_empty
        assert has_cliff

    def test_returns_empty_for_no_failures(self):
        patterns = run_all_detectors("faithfulness", [], 0.9)
        assert patterns == []
