"""Tests for prompt eval suite runner."""
from __future__ import annotations

import json
from pathlib import Path

from abi_evals.prompt_eval import (
    PromptMockProvider,
    _compute_input_hash,
    run_prompt_eval_suite,
)


def _write_suite(tmp_path: Path, suite: dict) -> Path:
    """Helper: write a suite dict to a temp JSON file."""
    p = tmp_path / "suite.json"
    p.write_text(json.dumps(suite, sort_keys=True, ensure_ascii=False), encoding="utf-8")
    return p


def _make_suite(
    cases: list[dict] | None = None,
    suite_id: str = "test-suite",
    prompt_id: str = "test-prompt",
) -> dict:
    """Helper: build a minimal valid prompt-eval-suite/1.0."""
    if cases is None:
        cases = [
            {
                "case_id": "case-001",
                "description": "Basic case",
                "input": {"task": "test"},
                "expected": {
                    "required_keys": ["implementation", "explanation"],
                    "contains_patterns": ["mock"],
                },
                "comparison": "prompt_structural",
            }
        ]
    return {
        "schema_version": "prompt-eval-suite/1.0",
        "suite_id": suite_id,
        "prompt_id": prompt_id,
        "description": "Test suite",
        "cases": cases,
    }


class TestMockProvider:
    def test_mock_provider_determinism(self):
        """Same input always produces same output."""
        provider = PromptMockProvider()
        r1 = provider.invoke("prompt-a", {"key": "value"})
        r2 = provider.invoke("prompt-a", {"key": "value"})
        assert r1.output == r2.output
        assert r1.tokens_used == r2.tokens_used
        assert r1.prompt_id == r2.prompt_id

    def test_mock_provider_custom_response(self):
        """set_response overrides the default output."""
        provider = PromptMockProvider()
        input_hash = _compute_input_hash("my-prompt", {"x": 1})
        provider.set_response(input_hash, '{"custom": true}')

        result = provider.invoke("my-prompt", {"x": 1})
        assert result.output == '{"custom": true}'
        parsed = json.loads(result.output)
        assert parsed["custom"] is True

    def test_no_network_calls(self):
        """Mock provider makes no network calls — just verify it returns immediately."""
        provider = PromptMockProvider()
        result = provider.invoke("offline-prompt", {"data": "test"})
        assert isinstance(result.output, str)
        assert result.tokens_used > 0
        assert result.prompt_id == "offline-prompt"


class TestRunPromptEvalSuite:
    def test_run_suite_all_pass(self, tmp_path: Path):
        """Clean suite with all default mock outputs should pass."""
        suite = _make_suite()
        suite_path = _write_suite(tmp_path, suite)

        results = run_prompt_eval_suite(suite_path)

        assert results["summary"]["passed"] == 1
        assert results["summary"]["failed"] == 0
        assert results["summary"]["errors"] == 0
        assert results["summary"]["total"] == 1

    def test_run_suite_budget_check(self, tmp_path: Path):
        """Budget enforcement: case with max_tokens=1 should fail."""
        cases = [
            {
                "case_id": "case-budget",
                "description": "Tiny budget",
                "input": {"task": "something"},
                "expected": {
                    "required_keys": ["implementation"],
                },
                "comparison": "prompt_structural",
                "budget_expectation": {"max_tokens": 1},
            }
        ]
        suite = _make_suite(cases=cases)
        suite_path = _write_suite(tmp_path, suite)

        results = run_prompt_eval_suite(suite_path)

        assert results["summary"]["failed"] == 1
        budget_result = results["results"][0]
        assert budget_result["status"] == "fail"
        assert "Budget exceeded" in budget_result["message"]

    def test_run_suite_missing_keys(self, tmp_path: Path):
        """Fails when output is missing required keys."""
        cases = [
            {
                "case_id": "case-missing-keys",
                "description": "Requires a key the mock will not produce",
                "input": {"task": "test"},
                "expected": {
                    "required_keys": ["nonexistent_key_xyz"],
                },
                "comparison": "prompt_structural",
            }
        ]
        suite = _make_suite(cases=cases)
        suite_path = _write_suite(tmp_path, suite)

        results = run_prompt_eval_suite(suite_path)

        assert results["summary"]["failed"] == 1
        assert results["results"][0]["status"] == "fail"
        assert "Missing keys" in results["results"][0]["message"]

    def test_run_suite_missing_patterns(self, tmp_path: Path):
        """Fails when output does not contain required patterns."""
        cases = [
            {
                "case_id": "case-missing-patterns",
                "description": "Requires a pattern the mock will not produce",
                "input": {"task": "test"},
                "expected": {
                    "required_keys": ["implementation"],
                    "contains_patterns": ["XYZZY_NEVER_IN_OUTPUT"],
                },
                "comparison": "prompt_structural",
            }
        ]
        suite = _make_suite(cases=cases)
        suite_path = _write_suite(tmp_path, suite)

        results = run_prompt_eval_suite(suite_path)

        assert results["summary"]["failed"] == 1
        assert results["results"][0]["status"] == "fail"
        assert "Missing patterns" in results["results"][0]["message"]

    def test_result_schema_compliance(self, tmp_path: Path):
        """Output matches eval-result/1.0 required structure."""
        suite = _make_suite()
        suite_path = _write_suite(tmp_path, suite)

        results = run_prompt_eval_suite(suite_path)

        # Top-level keys
        assert results["schema_version"] == "eval-result/1.0"
        assert "suite_id" in results
        assert "run_id" in results
        assert "results" in results
        assert "summary" in results

        # Summary keys
        summary = results["summary"]
        for key in ("total", "passed", "failed", "errors", "skipped"):
            assert key in summary
            assert isinstance(summary[key], int)

        # Each result
        for r in results["results"]:
            assert "case_id" in r
            assert "status" in r
            assert r["status"] in ("pass", "fail", "error")
            assert "actual" in r
            assert "message" in r
            assert "diff" in r

    def test_stable_ordering(self, tmp_path: Path):
        """Cases are sorted by case_id in output regardless of input order."""
        cases = [
            {
                "case_id": "case-z",
                "description": "Z comes last",
                "input": {"task": "z"},
                "expected": {"required_keys": ["implementation"]},
                "comparison": "prompt_structural",
            },
            {
                "case_id": "case-a",
                "description": "A comes first",
                "input": {"task": "a"},
                "expected": {"required_keys": ["implementation"]},
                "comparison": "prompt_structural",
            },
            {
                "case_id": "case-m",
                "description": "M is in the middle",
                "input": {"task": "m"},
                "expected": {"required_keys": ["implementation"]},
                "comparison": "prompt_structural",
            },
        ]
        suite = _make_suite(cases=cases)
        suite_path = _write_suite(tmp_path, suite)

        results = run_prompt_eval_suite(suite_path)

        case_ids = [r["case_id"] for r in results["results"]]
        assert case_ids == sorted(case_ids)
        assert case_ids == ["case-a", "case-m", "case-z"]

    def test_deterministic_run_id(self, tmp_path: Path):
        """Same suite always produces the same run_id."""
        suite = _make_suite()
        p1 = tmp_path / "suite1.json"
        p2 = tmp_path / "suite2.json"
        content = json.dumps(suite, sort_keys=True, ensure_ascii=False)
        p1.write_text(content, encoding="utf-8")
        p2.write_text(content, encoding="utf-8")

        r1 = run_prompt_eval_suite(p1)
        r2 = run_prompt_eval_suite(p2)

        assert r1["run_id"] == r2["run_id"]
        # run_id is 8 hex chars
        assert len(r1["run_id"]) == 8
        assert all(c in "0123456789abcdef" for c in r1["run_id"])
