"""Tests for eval suite runner."""
from __future__ import annotations

import json
from pathlib import Path

import pytest

from abi_evals.runner import load_eval_suite, parse_cases, run_suite


class TestRunner:
    def test_load_suite(self, sample_suite_path):
        suite = load_eval_suite(sample_suite_path)
        assert suite["suite_id"] == "sample-suite-001"
        assert len(suite["cases"]) >= 5

    def test_parse_cases(self, sample_suite):
        cases = parse_cases(sample_suite)
        assert len(cases) >= 5
        assert cases[0].case_id is not None

    def test_run_suite_deterministic(self, sample_suite):
        """Same input always produces same output."""
        r1 = run_suite(sample_suite, run_id="test0001", timestamp="2025-01-01T00:00:00Z")
        r2 = run_suite(sample_suite, run_id="test0001", timestamp="2025-01-01T00:00:00Z")
        assert r1 == r2

    def test_run_suite_stable_ordering(self, sample_suite):
        """Results are sorted by case_id."""
        results = run_suite(sample_suite, run_id="test0001", timestamp="2025-01-01T00:00:00Z")
        case_ids = [r["case_id"] for r in results["results"]]
        assert case_ids == sorted(case_ids)

    def test_run_suite_stable_json(self, sample_suite):
        """JSON serialization is stable."""
        results = run_suite(sample_suite, run_id="test0001", timestamp="2025-01-01T00:00:00Z")
        s1 = json.dumps(results, sort_keys=True, ensure_ascii=False)
        s2 = json.dumps(results, sort_keys=True, ensure_ascii=False)
        assert s1 == s2

    def test_run_suite_all_pass(self, sample_suite):
        """All cases in sample suite should pass."""
        results = run_suite(sample_suite, run_id="test0001", timestamp="2025-01-01T00:00:00Z")
        assert results["summary"]["failed"] == 0
        assert results["summary"]["errors"] == 0
        assert results["summary"]["passed"] == results["summary"]["total"]

    def test_run_suite_with_evidence_pack(self, sample_suite, sample_evidence_pack):
        results = run_suite(
            sample_suite,
            evidence_pack=sample_evidence_pack,
            run_id="test0001",
            timestamp="2025-01-01T00:00:00Z",
        )
        assert results["schema_version"] == "eval-result/1.0"

    def test_run_suite_output_format(self, sample_suite):
        results = run_suite(sample_suite, run_id="abcd1234", timestamp="2025-01-01T00:00:00Z")
        assert "schema_version" in results
        assert "suite_id" in results
        assert "run_id" in results
        assert "evaluated_at" in results
        assert "results" in results
        assert "summary" in results


@pytest.fixture()
def _cost_suite_path(tmp_path: Path) -> Path:
    """Create a minimal eval suite for cost tracking tests."""
    suite = {
        "schema_version": "eval-suite/1.0",
        "suite_id": "cost-test-suite",
        "description": "Suite for cost tracking verification",
        "cases": [
            {
                "case_id": "cost-001",
                "description": "Exact match A",
                "input": "hello",
                "expected": "hello",
                "comparison": "exact",
            },
            {
                "case_id": "cost-002",
                "description": "Exact match B",
                "input": {"key": "value", "num": 42},
                "expected": {"key": "value", "num": 42},
                "comparison": "structural",
            },
            {
                "case_id": "cost-003",
                "description": "Numeric match",
                "input": 3.14,
                "expected": 3.14,
                "comparison": "exact",
            },
        ],
    }
    path = tmp_path / "cost_suite.json"
    path.write_text(json.dumps(suite, indent=2), encoding="utf-8")
    return path


class TestCostTracking:
    def test_results_include_tokens_used(self, _cost_suite_path: Path):
        """Each result dict contains a tokens_used integer > 0."""
        suite = load_eval_suite(_cost_suite_path)
        output = run_suite(suite, run_id="cost0001", timestamp="2025-01-01T00:00:00Z")
        for r in output["results"]:
            assert "tokens_used" in r, f"Missing tokens_used in case {r['case_id']}"
            assert isinstance(r["tokens_used"], int)
            assert r["tokens_used"] > 0

    def test_cost_summary_present(self, _cost_suite_path: Path):
        """Suite output includes a cost_summary dict."""
        suite = load_eval_suite(_cost_suite_path)
        output = run_suite(suite, run_id="cost0001", timestamp="2025-01-01T00:00:00Z")
        assert "cost_summary" in output
        cs = output["cost_summary"]
        assert "total_tokens" in cs
        assert "case_token_map" in cs

    def test_cost_summary_consistent(self, _cost_suite_path: Path):
        """total_tokens equals the sum of individual tokens_used values."""
        suite = load_eval_suite(_cost_suite_path)
        output = run_suite(suite, run_id="cost0001", timestamp="2025-01-01T00:00:00Z")
        individual_sum = sum(r["tokens_used"] for r in output["results"])
        assert output["cost_summary"]["total_tokens"] == individual_sum
        # Also verify the case_token_map sums match
        map_sum = sum(output["cost_summary"]["case_token_map"].values())
        assert map_sum == individual_sum
