"""Unit tests for EvaluationEngine."""

from __future__ import annotations

import asyncio

import pytest

from evalkit.engine import EvaluationEngine
from evalkit.metrics.base import BaseMetric
from evalkit.models import EvaluationReport, ToolCall
from evalkit.models import TestCase as EvalTestCase
from evalkit.suites import AgentToolCallSuite, DocumentSearchSuite

# Re-use fixtures from conftest: mock_judge, sample_test_case


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


class _AlwaysPassMetric(BaseMetric):
    """Deterministic stub metric that always returns score=0.8."""

    name = "always_pass"
    threshold = 0.5

    def _build_prompt(self, test_case: EvalTestCase) -> str:
        return f"Evaluate: {test_case.question}"


class _AlwaysFailMetric(BaseMetric):
    """Deterministic stub metric that always returns score=0.2."""

    name = "always_fail"
    threshold = 0.5

    def _build_prompt(self, test_case: EvalTestCase) -> str:
        return f"Evaluate: {test_case.question}"


# ---------------------------------------------------------------------------
# _resolve_metrics
# ---------------------------------------------------------------------------


class TestResolveMetrics:
    def test_suite_returns_suite_name_and_metrics(self, mock_judge):
        engine = EvaluationEngine(mock_judge)
        suite = DocumentSearchSuite()
        resolved_metrics, suite_name = engine._resolve_metrics(suite, None)
        assert suite_name == "document_search"
        assert resolved_metrics is suite.metrics

    def test_metrics_list_returns_custom_name(self, mock_judge):
        engine = EvaluationEngine(mock_judge)
        custom = [_AlwaysPassMetric()]
        resolved_metrics, suite_name = engine._resolve_metrics(None, custom)
        assert suite_name == "custom"
        assert resolved_metrics is custom

    def test_both_raises_value_error(self, mock_judge):
        engine = EvaluationEngine(mock_judge)
        suite = DocumentSearchSuite()
        with pytest.raises(ValueError, match="not both"):
            engine._resolve_metrics(suite, [_AlwaysPassMetric()])

    def test_neither_raises_value_error(self, mock_judge):
        engine = EvaluationEngine(mock_judge)
        with pytest.raises(ValueError, match="Must provide"):
            engine._resolve_metrics(None, None)


# ---------------------------------------------------------------------------
# run() — synchronous
# ---------------------------------------------------------------------------


class TestRunSync:
    def test_run_with_suite_returns_report(self, mock_judge, sample_test_case):
        engine = EvaluationEngine(mock_judge)
        suite = DocumentSearchSuite()
        report = engine.run([sample_test_case], suite=suite)
        assert isinstance(report, EvaluationReport)
        assert report.suite_name == "document_search"
        assert len(report.results) == 1

    def test_run_with_custom_metrics_returns_report(self, mock_judge, sample_test_case):
        engine = EvaluationEngine(mock_judge)
        custom_metrics = [_AlwaysPassMetric()]
        report = engine.run([sample_test_case], metrics=custom_metrics)
        assert isinstance(report, EvaluationReport)
        assert report.suite_name == "custom"
        assert len(report.results) == 1

    def test_run_calls_metric_for_each_test_case(self, mock_judge):
        """judge.call_count should equal len(test_cases) * len(metrics)."""
        engine = EvaluationEngine(mock_judge)
        test_cases = [
            EvalTestCase(
                question=f"Question {i}",
                context=[f"Context {i}"],
                answer=f"Answer {i}",
            )
            for i in range(3)
        ]
        custom_metrics = [_AlwaysPassMetric(), _AlwaysFailMetric()]
        engine.run(test_cases, metrics=custom_metrics)
        # 3 test cases × 2 metrics = 6 judge calls
        assert mock_judge.call_count == 6

    def test_run_raises_when_both_suite_and_metrics(self, mock_judge, sample_test_case):
        engine = EvaluationEngine(mock_judge)
        suite = DocumentSearchSuite()
        with pytest.raises(ValueError, match="not both"):
            engine.run([sample_test_case], suite=suite, metrics=[_AlwaysPassMetric()])

    def test_run_raises_when_neither_suite_nor_metrics(self, mock_judge, sample_test_case):
        engine = EvaluationEngine(mock_judge)
        with pytest.raises(ValueError, match="Must provide"):
            engine.run([sample_test_case])

    def test_run_with_multiple_test_cases_returns_correct_count(self, mock_judge):
        engine = EvaluationEngine(mock_judge)
        test_cases = [
            EvalTestCase(
                question=f"Q{i}",
                context=[f"ctx {i}"],
                answer=f"A{i}",
            )
            for i in range(5)
        ]
        report = engine.run(test_cases, metrics=[_AlwaysPassMetric()])
        assert len(report.results) == 5

    def test_run_concurrency_produces_correct_results(self, mock_judge):
        """Engine with concurrency=2 and 4 test cases still returns all 4 results."""
        engine = EvaluationEngine(mock_judge, concurrency=2)
        test_cases = [
            EvalTestCase(
                question=f"Q{i}",
                context=[f"ctx {i}"],
                answer=f"A{i}",
            )
            for i in range(4)
        ]
        report = engine.run(test_cases, metrics=[_AlwaysPassMetric()])
        assert len(report.results) == 4
        # All results should be present and scores parseable
        for result in report.results:
            assert len(result.scores) == 1
            assert 0.0 <= result.overall_score <= 1.0


# ---------------------------------------------------------------------------
# run_async() — asynchronous
# ---------------------------------------------------------------------------


class TestRunAsync:
    def test_run_async_returns_same_structure_as_run(self, mock_judge, sample_test_case):
        engine = EvaluationEngine(mock_judge)
        suite = DocumentSearchSuite()

        sync_report = engine.run([sample_test_case], suite=suite)

        # Reset call count so async run has a fresh mock
        mock_judge.call_count = 0
        async_report = asyncio.run(engine.run_async([sample_test_case], suite=suite))

        assert isinstance(async_report, EvaluationReport)
        assert async_report.suite_name == sync_report.suite_name
        assert len(async_report.results) == len(sync_report.results)
        assert async_report.summary.total_cases == sync_report.summary.total_cases

    def test_run_async_with_custom_metrics(self, mock_judge, sample_test_case):
        engine = EvaluationEngine(mock_judge)
        custom_metrics = [_AlwaysPassMetric()]
        report = asyncio.run(engine.run_async([sample_test_case], metrics=custom_metrics))
        assert isinstance(report, EvaluationReport)
        assert report.suite_name == "custom"
        assert len(report.results) == 1

    def test_run_async_raises_when_both_provided(self, mock_judge, sample_test_case):
        engine = EvaluationEngine(mock_judge)
        suite = DocumentSearchSuite()
        with pytest.raises(ValueError, match="not both"):
            asyncio.run(
                engine.run_async(
                    [sample_test_case],
                    suite=suite,
                    metrics=[_AlwaysPassMetric()],
                )
            )

    def test_run_async_multiple_test_cases(self, mock_judge):
        engine = EvaluationEngine(mock_judge)
        test_cases = [EvalTestCase(question=f"Q{i}", context=[f"ctx {i}"], answer=f"A{i}") for i in range(4)]
        report = asyncio.run(engine.run_async(test_cases, metrics=[_AlwaysPassMetric()]))
        assert len(report.results) == 4


# ---------------------------------------------------------------------------
# AgentToolCallSuite integration
# ---------------------------------------------------------------------------


class TestEngineWithAgentSuite:
    def test_agent_suite_deterministic_metrics_skip_judge(self, mock_judge):
        """ToolSelectionMetric and ToolParameterMetric are deterministic; only
        ToolParameterSimilarityMetric calls the judge (when tool names match)."""
        test_case = EvalTestCase(
            question="Book a flight from NYC to LA",
            expected_tool_calls=[
                ToolCall(name="search_flights", parameters={"origin": "NYC", "destination": "LA"}),
            ],
            actual_tool_calls=[
                ToolCall(name="search_flights", parameters={"origin": "NYC", "destination": "LA"}),
            ],
            answer="Found flights from NYC to LA.",
        )
        engine = EvaluationEngine(mock_judge)
        report = engine.run([test_case], suite=AgentToolCallSuite())

        result = report.results[0]
        assert len(result.scores) == 3
        score_names = [s.metric_name for s in result.scores]
        assert "tool_selection" in score_names
        assert "tool_parameter" in score_names
        assert "tool_parameter_similarity" in score_names
        # Only ToolParameterSimilarityMetric calls the judge
        assert mock_judge.call_count == 1

    def test_agent_suite_with_mismatched_tools(self, mock_judge):
        """When tool names differ, tool_selection=0.0 and tool_parameter_similarity=0.0."""
        test_case = EvalTestCase(
            question="Get the weather in Paris",
            expected_tool_calls=[
                ToolCall(name="get_weather", parameters={"city": "Paris"}),
            ],
            actual_tool_calls=[
                ToolCall(name="search_web", parameters={"query": "Paris weather"}),
            ],
            answer="It is sunny in Paris.",
        )
        engine = EvaluationEngine(mock_judge)
        report = engine.run([test_case], suite=AgentToolCallSuite())

        result = report.results[0]
        scores_by_name = {s.metric_name: s.score for s in result.scores}
        assert scores_by_name["tool_selection"] == 0.0
        assert scores_by_name["tool_parameter_similarity"] == 0.0
