"""Tests for SWE-bench scoring command construction and prerequisite checks."""

from __future__ import annotations

from pathlib import Path

import pytest

from evals.swebench.io import write_instance_ids, write_predictions
from evals.swebench.score_official import (
    build_run_evaluation_command,
    check_scoring_prerequisites,
    run_official_scoring,
)
from evals.swebench.types import PredictionRecord


def _write_predictions(path: Path, ids: list[str], *, model_tag: str) -> None:
    write_predictions(
        path,
        [
            PredictionRecord(
                instance_id=instance_id,
                model_patch="diff --git a/a.py b/a.py\n",
                model_name_or_path=model_tag,
            )
            for instance_id in ids
        ],
    )


def test_build_run_evaluation_command_handles_clean_flag() -> None:
    base_kwargs = {
        "predictions_path": Path("/tmp/predictions.jsonl"),
        "run_id": "run-baseline",
        "dataset_name": "princeton-nlp/SWE-bench_Lite",
        "split": "test",
        "workers": 4,
        "cache_level": "env",
        "python_executable": "python",
        "module": "swebench.harness.run_evaluation",
    }

    without_clean = build_run_evaluation_command(clean=False, **base_kwargs)
    with_clean = build_run_evaluation_command(clean=True, **base_kwargs)

    assert "--clean" not in without_clean
    assert "--clean" in with_clean


def test_run_official_scoring_routes_conditions_and_outputs(monkeypatch, tmp_path) -> None:
    run_dir = tmp_path / "demo-run"
    (run_dir / "baseline").mkdir(parents=True, exist_ok=True)
    (run_dir / "llmdebug").mkdir(parents=True, exist_ok=True)
    write_instance_ids(run_dir / "instance_ids.txt", ["inst-1", "inst-2"])
    _write_predictions(
        run_dir / "baseline" / "predictions.jsonl",
        ["inst-1", "inst-2"],
        model_tag="baseline-model",
    )
    _write_predictions(
        run_dir / "llmdebug" / "predictions.jsonl",
        ["inst-1", "inst-2"],
        model_tag="llmdebug-model",
    )

    calls: list[tuple[list[str], Path]] = []

    class _Proc:
        def __init__(self, returncode: int = 0) -> None:
            self.returncode = returncode

    def _fake_run(command, *, cwd, stdout=None, stderr=None, text=True, check=False):
        del stdout, stderr, text, check
        calls.append((list(command), cwd))
        return _Proc(returncode=0)

    monkeypatch.setattr("evals.swebench.score_official.subprocess.run", _fake_run)
    summary = run_official_scoring(
        run_dir=run_dir,
        conditions="baseline,llmdebug",
        dataset_name="princeton-nlp/SWE-bench_Lite",
        split="test",
        workers=2,
        cache_level="env",
        clean=False,
        check_prereqs=False,
    )

    assert len(calls) == 2
    assert summary.run_dir == run_dir
    assert {item.condition for item in summary.condition_summaries} == {"baseline", "llmdebug"}
    assert any("--run_id" in cmd and "demo-run-baseline" in cmd for cmd, _ in calls)
    assert any("--run_id" in cmd and "demo-run-llmdebug" in cmd for cmd, _ in calls)
    assert any(cwd == run_dir / "official_eval" / "baseline" for _, cwd in calls)
    assert any(cwd == run_dir / "official_eval" / "llmdebug" for _, cwd in calls)


def test_run_official_scoring_rejects_single_condition(tmp_path) -> None:
    run_dir = tmp_path / "demo-run"
    run_dir.mkdir(parents=True, exist_ok=True)
    with pytest.raises(ValueError, match="conditions must include exactly baseline,llmdebug"):
        run_official_scoring(
            run_dir=run_dir,
            conditions="baseline",
            dataset_name="princeton-nlp/SWE-bench_Lite",
            split="test",
            workers=2,
            cache_level="env",
            clean=False,
            check_prereqs=False,
        )


def test_check_scoring_prerequisites_rejects_version_mismatch(monkeypatch) -> None:
    monkeypatch.setattr(
        "evals.swebench.score_official.shutil.which", lambda _name: "/usr/bin/docker"
    )
    monkeypatch.setattr(
        "evals.swebench.score_official.importlib.metadata.version", lambda _pkg: "4.0.5"
    )

    with pytest.raises(RuntimeError, match="version mismatch"):
        check_scoring_prerequisites(
            python_executable="python",
            module="swebench.harness.run_evaluation",
        )
