"""Unit tests for evalkit.reports.generator.ReportGenerator."""

from __future__ import annotations

import pytest

from evalkit.models import (
    EvaluationResult,
    MetricScore,
)
from evalkit.models import (
    TestCase as EvalTestCase,
)
from evalkit.reports.generator import ReportGenerator

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_test_case(question: str = "What is X?") -> EvalTestCase:
    return EvalTestCase(
        question=question,
        context=["Context chunk."],
        answer="The answer is X.",
    )


def _make_metric_score(
    name: str = "faithfulness",
    score: float = 0.8,
    passed: bool = True,
) -> MetricScore:
    return MetricScore(
        metric_name=name,
        score=score,
        reason="Test reason.",
        passed=passed,
    )


def _make_result(scores: list[MetricScore], question: str = "What is X?") -> EvaluationResult:
    return EvaluationResult(
        test_case=_make_test_case(question=question),
        scores=scores,
    )


# ---------------------------------------------------------------------------
# _compute_summary
# ---------------------------------------------------------------------------


class TestComputeSummary:
    def setup_method(self):
        self.gen = ReportGenerator()

    def test_empty_results_all_zeros(self):
        summary = self.gen._compute_summary([])
        assert summary.total_cases == 0
        assert summary.passed == 0
        assert summary.failed == 0
        assert summary.average_score == pytest.approx(0.0)
        assert summary.score_by_metric == {}

    def test_all_passing(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
            _make_result([_make_metric_score("faithfulness", 0.9, True)]),
        ]
        summary = self.gen._compute_summary(results)
        assert summary.total_cases == 2
        assert summary.passed == 2
        assert summary.failed == 0

    def test_all_failing(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
            _make_result([_make_metric_score("faithfulness", 0.2, False)]),
        ]
        summary = self.gen._compute_summary(results)
        assert summary.total_cases == 2
        assert summary.passed == 0
        assert summary.failed == 2

    def test_mixed_pass_fail(self):
        # overall_score = mean of MetricScores per result
        # result1: overall=0.8 (passes >= 0.5)
        # result2: overall=0.3 (fails < 0.5)
        results = [
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
        ]
        summary = self.gen._compute_summary(results)
        assert summary.total_cases == 2
        assert summary.passed == 1
        assert summary.failed == 1

    def test_average_score_correct(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.6, True)]),
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
        ]
        summary = self.gen._compute_summary(results)
        # overall scores are 0.6 and 0.8 → mean = 0.7
        assert summary.average_score == pytest.approx(0.7)

    def test_score_by_metric_single_metric(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.6, True)]),
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
            _make_result([_make_metric_score("faithfulness", 1.0, True)]),
        ]
        summary = self.gen._compute_summary(results)
        assert "faithfulness" in summary.score_by_metric
        assert summary.score_by_metric["faithfulness"] == pytest.approx((0.6 + 0.8 + 1.0) / 3)

    def test_score_by_metric_multiple_metrics(self):
        results = [
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.6, True),
                    _make_metric_score("hallucination", 0.4, False),
                ]
            ),
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.8, True),
                    _make_metric_score("hallucination", 0.6, True),
                ]
            ),
        ]
        summary = self.gen._compute_summary(results)
        assert summary.score_by_metric["faithfulness"] == pytest.approx(0.7)
        assert summary.score_by_metric["hallucination"] == pytest.approx(0.5)

    def test_pass_threshold_exactly_0_5(self):
        # result with overall_score == 0.5 should pass
        results = [
            _make_result([_make_metric_score("faithfulness", 0.5, True)]),
        ]
        summary = self.gen._compute_summary(results)
        assert summary.passed == 1
        assert summary.failed == 0

    def test_pass_threshold_just_below_0_5(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.49, False)]),
        ]
        summary = self.gen._compute_summary(results)
        assert summary.passed == 0
        assert summary.failed == 1


# ---------------------------------------------------------------------------
# _group_failures
# ---------------------------------------------------------------------------


class TestGroupFailures:
    def setup_method(self):
        self.gen = ReportGenerator()

    def test_no_failures_returns_empty_list(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.9, True)]),
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
        ]
        failures = self.gen._group_failures(results)
        assert failures == []

    def test_groups_by_metric_correctly(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
            _make_result([_make_metric_score("hallucination", 0.2, False)]),
        ]
        failures = self.gen._group_failures(results)
        failure_types = {fg.failure_type for fg in failures}
        assert "faithfulness" in failure_types
        assert "hallucination" in failure_types

    def test_failure_type_matches_metric_name(self):
        results = [
            _make_result([_make_metric_score("answer_relevancy", 0.3, False)]),
        ]
        failures = self.gen._group_failures(results)
        assert len(failures) == 1
        assert failures[0].failure_type == "answer_relevancy"

    def test_count_is_correct(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
            _make_result([_make_metric_score("faithfulness", 0.4, False)]),
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
        ]
        failures = self.gen._group_failures(results)
        assert len(failures) == 1
        assert failures[0].count == 2

    def test_limits_examples_to_3_worst(self):
        # 5 failures for faithfulness — examples should be the 3 worst (lowest scores)
        results = [
            _make_result([_make_metric_score("faithfulness", score, False)], question=f"Q{i}")
            for i, score in enumerate([0.1, 0.2, 0.3, 0.4, 0.45])
        ]
        failures = self.gen._group_failures(results)
        assert len(failures) == 1
        fg = failures[0]
        assert fg.count == 5
        assert len(fg.examples) == 3
        # The 3 worst scores should be 0.1, 0.2, 0.3
        example_questions = {er.test_case.question for er in fg.examples}
        assert "Q0" in example_questions  # score 0.1 (worst)
        assert "Q1" in example_questions  # score 0.2
        assert "Q2" in example_questions  # score 0.3

    def test_examples_sorted_worst_first(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.4, False)], question="Q_med"),
            _make_result([_make_metric_score("faithfulness", 0.1, False)], question="Q_low"),
            _make_result([_make_metric_score("faithfulness", 0.3, False)], question="Q_mid"),
        ]
        failures = self.gen._group_failures(results)
        assert len(failures) == 1
        examples = failures[0].examples
        # All 3 fit within limit; first example should be the worst (0.1)
        assert examples[0].test_case.question == "Q_low"

    def test_sorts_by_count_descending(self):
        results = [
            # faithfulness fails once
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
            # hallucination fails three times
            _make_result([_make_metric_score("hallucination", 0.2, False)]),
            _make_result([_make_metric_score("hallucination", 0.3, False)]),
            _make_result([_make_metric_score("hallucination", 0.4, False)]),
            # completeness fails twice
            _make_result([_make_metric_score("completeness", 0.1, False)]),
            _make_result([_make_metric_score("completeness", 0.2, False)]),
        ]
        failures = self.gen._group_failures(results)
        counts = [fg.count for fg in failures]
        assert counts == sorted(counts, reverse=True)
        assert failures[0].failure_type == "hallucination"

    def test_attaches_correct_suggestion_from_dict(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
        ]
        failures = self.gen._group_failures(results)
        # suggestion should be one of the tiered suggestions for faithfulness
        assert failures[0].suggestion in ReportGenerator.SUGGESTIONS["faithfulness"].values()

    def test_attaches_hallucination_suggestion(self):
        results = [
            _make_result([_make_metric_score("hallucination", 0.2, False)]),
        ]
        failures = self.gen._group_failures(results)
        assert failures[0].suggestion in ReportGenerator.SUGGESTIONS["hallucination"].values()

    def test_fallback_suggestion_for_unknown_metric(self):
        results = [
            _make_result([_make_metric_score("custom_metric", 0.3, False)]),
        ]
        failures = self.gen._group_failures(results)
        assert failures[0].suggestion == "Review your custom_metric metric scores."

    def test_mixed_pass_fail_per_metric(self):
        # Some results pass faithfulness, some fail — only failures are grouped
        results = [
            _make_result([_make_metric_score("faithfulness", 0.9, True)]),
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
        ]
        failures = self.gen._group_failures(results)
        assert len(failures) == 1
        assert failures[0].count == 1


# ---------------------------------------------------------------------------
# generate() (end-to-end)
# ---------------------------------------------------------------------------


class TestGenerate:
    def setup_method(self):
        self.gen = ReportGenerator()

    def test_returns_evaluation_report_with_correct_suite_name(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
        ]
        report = self.gen.generate(results, suite_name="my-suite")
        assert report.suite_name == "my-suite"

    def test_report_results_equal_input(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
        ]
        report = self.gen.generate(results, suite_name="test")
        assert report.results == results

    def test_end_to_end_five_cases_mixed_scores(self):
        """5 test cases with mixed scores — verify summary + failures structure."""
        results = [
            # Case 1: faithfulness passes, hallucination fails
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.8, True),
                    _make_metric_score("hallucination", 0.3, False),
                ],
                question="Q1",
            ),
            # Case 2: faithfulness fails, hallucination passes
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.2, False),
                    _make_metric_score("hallucination", 0.9, True),
                ],
                question="Q2",
            ),
            # Case 3: both pass
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.7, True),
                    _make_metric_score("hallucination", 0.8, True),
                ],
                question="Q3",
            ),
            # Case 4: faithfulness fails, hallucination fails
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.1, False),
                    _make_metric_score("hallucination", 0.2, False),
                ],
                question="Q4",
            ),
            # Case 5: both pass but low scores
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.6, True),
                    _make_metric_score("hallucination", 0.6, True),
                ],
                question="Q5",
            ),
        ]

        report = self.gen.generate(results, suite_name="end-to-end-suite")

        # suite_name
        assert report.suite_name == "end-to-end-suite"

        # summary: total_cases
        assert report.summary.total_cases == 5

        # overall_score per result:
        # Q1: (0.8+0.3)/2 = 0.55 → passes
        # Q2: (0.2+0.9)/2 = 0.55 → passes
        # Q3: (0.7+0.8)/2 = 0.75 → passes
        # Q4: (0.1+0.2)/2 = 0.15 → fails
        # Q5: (0.6+0.6)/2 = 0.60 → passes
        assert report.summary.passed == 4
        assert report.summary.failed == 1

        # average_score = mean of [0.55, 0.55, 0.75, 0.15, 0.60]
        expected_avg = (0.55 + 0.55 + 0.75 + 0.15 + 0.60) / 5
        assert report.summary.average_score == pytest.approx(expected_avg)

        # score_by_metric
        # faithfulness: (0.8+0.2+0.7+0.1+0.6)/5 = 2.4/5 = 0.48
        # hallucination: (0.3+0.9+0.8+0.2+0.6)/5 = 2.8/5 = 0.56
        assert report.summary.score_by_metric["faithfulness"] == pytest.approx(0.48)
        assert report.summary.score_by_metric["hallucination"] == pytest.approx(0.56)

        # failures: faithfulness failed in Q2 and Q4 (count=2), hallucination in Q1 and Q4 (count=2)
        failure_types = {fg.failure_type for fg in report.failures}
        assert "faithfulness" in failure_types
        assert "hallucination" in failure_types

        faith_group = next(fg for fg in report.failures if fg.failure_type == "faithfulness")
        assert faith_group.count == 2

        halluc_group = next(fg for fg in report.failures if fg.failure_type == "hallucination")
        assert halluc_group.count == 2

        # suggestions are attached and come from the tiered dict
        assert faith_group.suggestion in ReportGenerator.SUGGESTIONS["faithfulness"].values()
        assert halluc_group.suggestion in ReportGenerator.SUGGESTIONS["hallucination"].values()

    def test_generate_empty_results(self):
        report = self.gen.generate([], suite_name="empty-suite")
        assert report.suite_name == "empty-suite"
        assert report.summary.total_cases == 0
        assert report.summary.average_score == pytest.approx(0.0)
        assert report.failures == []


# ---------------------------------------------------------------------------
# Severity classification
# ---------------------------------------------------------------------------


class TestSeverityClassification:
    def setup_method(self):
        self.gen = ReportGenerator()

    def test_critical_when_avg_below_half_threshold(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.2, False)]),
            _make_result([_make_metric_score("faithfulness", 0.2, False)]),
        ]
        failures = self.gen._group_failures(results, thresholds={"faithfulness": 0.5})
        assert failures[0].severity == "critical"

    def test_warning_when_avg_between_half_and_threshold(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.35, False)]),
        ]
        failures = self.gen._group_failures(results, thresholds={"faithfulness": 0.5})
        assert failures[0].severity == "warning"

    def test_mild_when_avg_near_threshold(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.45, False)]),
        ]
        failures = self.gen._group_failures(results, thresholds={"faithfulness": 0.5})
        assert failures[0].severity == "mild"

    def test_default_threshold_when_not_provided(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.1, False)]),
        ]
        failures = self.gen._group_failures(results)
        assert failures[0].severity == "critical"

    def test_average_score_set_on_failure_group(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.1, False)]),
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
        ]
        failures = self.gen._group_failures(results)
        assert failures[0].average_score == pytest.approx(0.2)

    def test_unknown_metric_gets_generic_fallback(self):
        results = [
            _make_result([_make_metric_score("custom_metric", 0.1, False)]),
        ]
        failures = self.gen._group_failures(results)
        assert "Review your custom_metric" in failures[0].suggestion

    def test_all_13_metrics_have_suggestions(self):
        known_metrics = [
            "faithfulness",
            "answer_relevancy",
            "contextual_relevancy",
            "hallucination",
            "completeness",
            "tool_selection",
            "tool_parameter",
            "tool_parameter_similarity",
            "exact_match",
            "string_containment",
            "string_similarity",
            "bleu",
            "rouge",
        ]
        for metric in known_metrics:
            for severity in ("critical", "warning", "mild"):
                suggestion = self.gen._get_suggestion(metric, severity)
                assert "Review your" not in suggestion, f"Metric {metric!r} severity {severity!r} uses generic fallback"


# ---------------------------------------------------------------------------
# _compute_summary with MetricSummaries
# ---------------------------------------------------------------------------


class TestComputeSummaryWithMetricSummaries:
    def setup_method(self):
        self.gen = ReportGenerator()

    def test_metric_summaries_populated(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
            _make_result([_make_metric_score("faithfulness", 0.3, False)]),
        ]
        summary = self.gen._compute_summary(results, thresholds={"faithfulness": 0.5})
        ms = summary.metric_summaries["faithfulness"]
        assert ms.total_count == 2
        assert ms.failed_count == 1
        assert ms.pass_rate == pytest.approx(0.5)
        assert ms.min_score == pytest.approx(0.3)
        assert ms.max_score == pytest.approx(0.8)

    def test_uses_metric_specific_threshold(self):
        results = [
            _make_result([_make_metric_score("hallucination", 0.6, True)]),
            _make_result([_make_metric_score("hallucination", 0.65, True)]),
        ]
        summary = self.gen._compute_summary(results, thresholds={"hallucination": 0.7})
        ms = summary.metric_summaries["hallucination"]
        assert ms.failed_count == 2
        assert ms.pass_rate == pytest.approx(0.0)

    def test_defaults_to_0_5_threshold(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.6, True)]),
        ]
        summary = self.gen._compute_summary(results)
        ms = summary.metric_summaries["faithfulness"]
        assert ms.pass_rate == pytest.approx(1.0)


class TestGenerateWithMetrics:
    def setup_method(self):
        self.gen = ReportGenerator()

    def test_metric_summaries_populated(self):
        results = [
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.8, True),
                    _make_metric_score("hallucination", 0.3, False),
                ]
            ),
            _make_result(
                [
                    _make_metric_score("faithfulness", 0.9, True),
                    _make_metric_score("hallucination", 0.2, False),
                ]
            ),
        ]
        report = self.gen.generate(results, "test")
        assert "faithfulness" in report.summary.metric_summaries
        assert "hallucination" in report.summary.metric_summaries
        ms = report.summary.metric_summaries["hallucination"]
        assert ms.failed_count == 2
        assert ms.total_count == 2
        assert ms.pass_rate == pytest.approx(0.0)

    def test_severity_set_on_failure_groups(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.1, False)]),
        ]
        report = self.gen.generate(results, "test")
        assert report.failures[0].severity == "critical"

    def test_patterns_populated_on_failure_groups(self):
        from unittest.mock import patch

        results = [
            _make_result([_make_metric_score("faithfulness", 0.1, False)]),
            _make_result([_make_metric_score("faithfulness", 0.2, False)]),
        ]
        with patch("evalkit.reports.generator.run_all_detectors", return_value=["test pattern"]):
            report = self.gen.generate(results, "test")
        assert "test pattern" in report.failures[0].patterns

    def test_llm_analysis_when_judge_provided(self):
        from unittest.mock import MagicMock

        judge = MagicMock()
        from evalkit.models import JudgeResponse

        judge.evaluate.return_value = JudgeResponse(score=0.0, reason="ANALYSIS: insight.\nPRIORITY FIX: fix.")
        results = [
            _make_result([_make_metric_score("faithfulness", 0.1, False)]),
        ]
        report = self.gen.generate(results, "test", judge=judge)
        assert report.failures[0].llm_analysis is not None
        assert "insight" in report.failures[0].llm_analysis

    def test_no_llm_analysis_without_judge(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.1, False)]),
        ]
        report = self.gen.generate(results, "test")
        assert report.failures[0].llm_analysis is None

    def test_backwards_compat_no_new_args(self):
        results = [
            _make_result([_make_metric_score("faithfulness", 0.8, True)]),
        ]
        report = self.gen.generate(results, "test")
        assert report.suite_name == "test"
        assert report.summary.metric_summaries != {}
