"""Tests for LLM-powered failure analysis."""

from __future__ import annotations

from unittest.mock import MagicMock

from evalkit.models import (
    EvaluationResult,
    JudgeResponse,
    MetricScore,
)
from evalkit.models import TestCase as EvalTestCase
from evalkit.reports.analyzer import analyze_failure_group, parse_analysis


class TestParseAnalysis:
    def test_parses_both_sections(self):
        reason = "ANALYSIS: The model hallucinates numbers.\nPRIORITY FIX: Add anti-hallucination prompt."
        result = parse_analysis(reason)
        assert "hallucinates numbers" in result
        assert "anti-hallucination" in result

    def test_handles_missing_priority_fix(self):
        reason = "ANALYSIS: Some pattern found."
        result = parse_analysis(reason)
        assert "Some pattern found" in result

    def test_handles_no_markers(self):
        reason = "The model is just bad at this."
        result = parse_analysis(reason)
        assert "just bad" in result


class TestAnalyzeFailureGroup:
    def _make_failure(self, question="Q?", score=0.2):
        tc = EvalTestCase(question=question, context=["ctx"], answer="A.")
        ms = MetricScore(metric_name="faithfulness", score=score, reason="r", passed=False)
        return (score, EvaluationResult(test_case=tc, scores=[ms]))

    def test_returns_analysis_string(self):
        judge = MagicMock()
        judge.evaluate.return_value = JudgeResponse(
            score=0.0,
            reason="ANALYSIS: Pattern found.\nPRIORITY FIX: Fix the prompt.",
        )
        failures = [self._make_failure()]
        result = analyze_failure_group("faithfulness", failures, judge)
        assert result is not None
        assert "Pattern found" in result
        judge.evaluate.assert_called_once()

    def test_returns_none_when_no_judge(self):
        result = analyze_failure_group("faithfulness", [], None)
        assert result is None

    def test_returns_none_on_judge_error(self):
        judge = MagicMock()
        judge.evaluate.side_effect = Exception("API error")
        failures = [self._make_failure()]
        result = analyze_failure_group("faithfulness", failures, judge)
        assert result is None

    def test_caps_failures_at_10(self):
        judge = MagicMock()
        judge.evaluate.return_value = JudgeResponse(score=0.0, reason="ANALYSIS: ok.\nPRIORITY FIX: fix.")
        failures = [self._make_failure(question=f"Q{i}?", score=0.1) for i in range(15)]
        analyze_failure_group("faithfulness", failures, judge)
        prompt = judge.evaluate.call_args[0][0]
        assert prompt.count("Question:") <= 10

    def test_returns_none_when_failures_empty(self):
        judge = MagicMock()
        result = analyze_failure_group("faithfulness", [], judge)
        assert result is None
        judge.evaluate.assert_not_called()
