"""Tests for eval case validation (expected exceptions, file structure, metadata)."""

from __future__ import annotations

from evals import run_eval, validate_cases
from evals.validate_cases import expected_exception_in_output
from evals.validate_cases import main as validate_main
from llmdebug import _json as json


def test_validate_cases_succeeds_for_synthetic_case(tmp_path) -> None:
    case_dir = tmp_path / "idx_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    (case_dir / "buggy.py").write_text(
        "from __future__ import annotations\n\n\n"
        "def bad_index(values: list[int]) -> int:\n"
        "    return values[2]\n",
        encoding="utf-8",
    )
    (case_dir / "test_buggy.py").write_text(
        "from __future__ import annotations\n\n"
        "from buggy import bad_index\n\n\n"
        "def test_bad_index() -> None:\n"
        "    assert bad_index([7, 8]) == 8\n",
        encoding="utf-8",
    )
    (case_dir / "case.json").write_text(
        json.dumps(
            {
                "description": "Mini validator fixture.",
                "category": "traceback_controls",
                "difficulty": "easy",
                "snapshot_dependency": "none",
                "expected_exception": "IndexError",
                "tags": ["control", "index"],
                "python_args": ["-m", "pytest", "-q"],
                "timeout_sec": 30,
            }
        ),
        encoding="utf-8",
    )
    (case_dir / "oracle.patch").write_text(
        "--- a/buggy.py\n"
        "+++ b/buggy.py\n"
        "@@ -2,5 +2,5 @@\n"
        " \n"
        " \n"
        " def bad_index(values: list[int]) -> int:\n"
        "-    return values[2]\n"
        "+    return values[1]\n",
        encoding="utf-8",
    )

    exit_code = validate_main(["--cases-dir", str(tmp_path), "--schema-only"])
    assert exit_code == 0


def test_validate_cases_schema_only_detects_missing_required_file(tmp_path) -> None:
    case_dir = tmp_path / "bad_case"
    case_dir.mkdir(parents=True, exist_ok=True)
    (case_dir / "buggy.py").write_text("def f():\n    return 1\n", encoding="utf-8")
    (case_dir / "test_buggy.py").write_text("def test_f():\n    assert True\n", encoding="utf-8")
    (case_dir / "case.json").write_text(
        json.dumps(
            {
                "description": "Missing oracle patch should fail schema check.",
                "category": "traceback_controls",
                "difficulty": "easy",
                "snapshot_dependency": "none",
                "expected_exception": "AssertionError",
                "tags": [],
            }
        ),
        encoding="utf-8",
    )

    exit_code = validate_main(["--cases-dir", str(tmp_path), "--schema-only"])
    assert exit_code == 1


def test_validate_cases_schema_only_detects_missing_implementation_file(tmp_path) -> None:
    case_dir = tmp_path / "bad_case_impl"
    case_dir.mkdir(parents=True, exist_ok=True)
    (case_dir / "test_buggy.py").write_text("def test_f():\n    assert True\n", encoding="utf-8")
    (case_dir / "case.json").write_text(
        json.dumps(
            {
                "description": "Missing implementation file should fail schema check.",
                "category": "traceback_controls",
                "difficulty": "easy",
                "snapshot_dependency": "none",
                "expected_exception": "AssertionError",
                "tags": [],
            }
        ),
        encoding="utf-8",
    )
    (case_dir / "oracle.patch").write_text(
        "--- a/buggy.py\n+++ b/buggy.py\n@@ -1 +1 @@\n-def f():\n+def f():\n",
        encoding="utf-8",
    )

    exit_code = validate_main(["--cases-dir", str(tmp_path), "--schema-only"])
    assert exit_code == 1


def test_expected_exception_matching_handles_qualified_names() -> None:
    output = (
        "E       asyncio.exceptions.InvalidStateError: Result is not set.\n"
        "FAILED test_buggy.py::test_cancelled_task_is_handled - "
        "asyncio.exceptions.InvalidStateError: Result is not set.\n"
    )
    assert expected_exception_in_output("InvalidStateError", output)
    assert expected_exception_in_output("asyncio.exceptions.InvalidStateError", output)
    assert not expected_exception_in_output("CancelledError", output)


def test_validate_one_case_reports_timeout_cleanly(tmp_path, monkeypatch) -> None:
    case_dir = tmp_path / "timeout_case"
    case_dir.mkdir(parents=True, exist_ok=True)
    (case_dir / "buggy.py").write_text("def f():\n    return 1\n", encoding="utf-8")
    (case_dir / "test_buggy.py").write_text("def test_f():\n    assert False\n", encoding="utf-8")
    (case_dir / "oracle.patch").write_text(
        "--- a/buggy.py\n+++ b/buggy.py\n@@ -1 +1 @@\n-def f():\n+def f():\n",
        encoding="utf-8",
    )

    timed_out = run_eval.RunResult(
        returncode=run_eval.PYTEST_TIMEOUT_RETURNCODE,
        duration_sec=0.1,
        stdout="",
        stderr="pytest timed out after 1s",
        timed_out=True,
        timeout_sec=1,
    )
    monkeypatch.setattr(validate_cases, "run_pytest", lambda *args, **kwargs: timed_out)

    ok, message = validate_cases.validate_one_case(
        case_dir,
        expected_exception="AssertionError",
        python_args=["-m", "pytest", "-q"],
        timeout_sec=1,
        output_format="json_compact",
    )
    assert not ok
    assert message == "baseline run timed out after 1s"
