"""Tests for eval execution configuration parsing and argument defaults."""

from __future__ import annotations

import argparse
from pathlib import Path
from types import SimpleNamespace

import pytest

from evals import run_eval
from llmdebug import _json as json


def test_execution_config_defaults_to_testcontainers() -> None:
    parser = argparse.ArgumentParser()
    run_eval.add_execution_arguments(parser)
    args = parser.parse_args([])
    config = run_eval.build_execution_config_from_args(args)
    assert config.executor == "testcontainers"
    assert config.container_image


def test_execution_config_rejects_invalid_container_env() -> None:
    parser = argparse.ArgumentParser()
    run_eval.add_execution_arguments(parser)
    args = parser.parse_args(["--container-env", "INVALID"])
    with pytest.raises(ValueError, match="expected KEY=VALUE"):
        run_eval.build_execution_config_from_args(args)


def test_run_pytest_dispatches_local_executor(tmp_path, monkeypatch) -> None:
    marker = run_eval.RunResult(returncode=0, duration_sec=0.0, stdout="ok", stderr="")
    monkeypatch.setattr(run_eval, "_run_pytest_local", lambda **kwargs: marker)
    result = run_eval.run_pytest(
        cwd=tmp_path,
        python_args=["-m", "pytest", "-q"],
        condition=run_eval.Condition.TRACEBACK_ONLY,
        timeout_sec=1,
        output_format="json_compact",
        execution=run_eval.ExecutionConfig(executor="local"),
    )
    assert result is marker


def test_run_pytest_dispatches_testcontainers_executor(tmp_path, monkeypatch) -> None:
    marker = run_eval.RunResult(returncode=1, duration_sec=0.0, stdout="", stderr="x")
    monkeypatch.setattr(run_eval, "_run_pytest_testcontainers", lambda **kwargs: marker)
    result = run_eval.run_pytest(
        cwd=tmp_path,
        python_args=["-m", "pytest", "-q"],
        condition=run_eval.Condition.TRACEBACK_ONLY,
        timeout_sec=1,
        output_format="json_compact",
        execution=run_eval.ExecutionConfig(executor="testcontainers"),
    )
    assert result is marker


def test_write_run_outputs_truncates_to_latest_lines(tmp_path) -> None:
    run = run_eval.RunResult(
        returncode=1,
        duration_sec=0.0,
        stdout="line-1\nline-2\nline-3\nline-4\n",
        stderr="err-1\nerr-2\n",
    )
    run_eval._write_run_outputs(tmp_path, run, prefix="pytest_before", tail_lines=2)

    stdout_text = (tmp_path / "pytest_before_stdout.txt").read_text(encoding="utf-8")
    stderr_text = (tmp_path / "pytest_before_stderr.txt").read_text(encoding="utf-8")
    combined_text = (tmp_path / "pytest_before_combined.txt").read_text(encoding="utf-8")

    assert "line-1" not in stdout_text
    assert "line-2" not in stdout_text
    assert "line-3\nline-4\n" in stdout_text
    assert "truncated 2 earlier line(s)" in stdout_text
    assert stderr_text == "err-1\nerr-2\n"
    assert "line-3\nline-4\n" in combined_text
    assert "err-1\nerr-2\n" in combined_text


def test_write_run_outputs_applies_hard_char_cap(tmp_path, monkeypatch) -> None:
    monkeypatch.setattr(run_eval, "PERSISTED_PYTEST_OUTPUT_HARD_CHAR_CAP", 64)
    huge_line = "prefix\n" + ("x" * 500)
    run = run_eval.RunResult(returncode=1, duration_sec=0.0, stdout=huge_line, stderr="")
    run_eval._write_run_outputs(tmp_path, run, prefix="pytest_before", tail_lines=0)

    stdout_text = (tmp_path / "pytest_before_stdout.txt").read_text(encoding="utf-8")
    assert "character(s)" in stdout_text
    assert stdout_text.endswith("x" * 64)
    assert len(stdout_text) < 200


def test_truncate_snapshot_for_artifact_caps_nested_strings() -> None:
    huge = "X" * 1000
    snapshot = {
        "exception": {"type": "AssertionError", "message": huge},
        "frames": [{"code": huge, "locals": {"value": huge}}],
    }

    truncated = run_eval._truncate_snapshot_for_artifact(snapshot, max_string_chars=80)

    assert truncated is not snapshot
    assert "snapshot string truncated" in truncated["exception"]["message"]
    assert len(truncated["exception"]["message"]) <= 80
    assert len(truncated["frames"][0]["code"]) <= 80
    assert len(truncated["frames"][0]["locals"]["value"]) <= 80
    assert snapshot["exception"]["message"] == huge


def test_truncate_snapshot_for_artifact_zero_disables_truncation() -> None:
    huge = "Y" * 1000
    snapshot = {"exception": {"message": huge}}
    truncated = run_eval._truncate_snapshot_for_artifact(snapshot, max_string_chars=0)
    assert truncated["exception"]["message"] == huge


def test_run_eval_preflights_testcontainers_runtime(tmp_path, monkeypatch, capsys) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "preflight_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="preflight_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=["preflight"],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(
        run_eval,
        "check_testcontainers_runtime_available",
        lambda: (False, "docker daemon is not reachable"),
    )

    exit_code = run_eval.main(
        [
            "--run-id",
            "preflight-run",
            "--patcher",
            "null",
            "--conditions",
            "traceback_only",
            "--k",
            "1",
        ]
    )
    assert exit_code == 2
    assert "Environment check failed: docker daemon is not reachable" in capsys.readouterr().out


def test_run_eval_prunes_stale_containers_before_testcontainers_run(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "cleanup_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="cleanup_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(
        run_eval,
        "check_testcontainers_runtime_available",
        lambda: (True, "ok"),
    )
    cleanup_calls: dict[str, int] = {}

    def _cleanup(*, grace_sec: int, now_utc=None):
        cleanup_calls["grace_sec"] = grace_sec
        return 0, 0, None

    monkeypatch.setattr(run_eval, "cleanup_stale_eval_containers", _cleanup)
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="stub"))

    def _run(*, case, condition, run_id, run_metadata, **_kwargs):
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--run-id",
            "cleanup-run",
            "--patcher",
            "null",
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "testcontainers",
        ]
    )

    assert exit_code == 0
    assert cleanup_calls == {"grace_sec": run_eval.DEFAULT_CONTAINER_CLEANUP_STALE_GRACE_SEC}


def test_run_eval_openai_max_propose_default_in_patcher_config(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "openai_default_budget_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="openai_default_budget_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="openai"))

    captured_run_metadata: dict[str, object] = {}

    def _run(*, case, condition, run_id, run_metadata, **_kwargs):
        captured_run_metadata.update(dict(run_metadata))
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--run-id",
            "openai-default-budget",
            "--patcher",
            "openai",
            "--openai-model",
            "dummy-model",
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
        ]
    )

    assert exit_code == 0
    patcher_config = captured_run_metadata.get("patcher_config")
    assert isinstance(patcher_config, dict)
    assert patcher_config["openai_max_propose_sec"] == 300.0
    assert patcher_config["openai_http2"] is True
    assert patcher_config["openai_transport_max_retries"] == 6
    assert patcher_config["openai_transport_retry_backoff_base_sec"] == 1.0
    assert patcher_config["openai_transport_retry_backoff_max_sec"] == 8.0


def test_run_eval_openai_transport_retry_settings_in_patcher_config(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "openai_transport_budget_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="openai_transport_budget_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="openai"))

    captured_run_metadata: dict[str, object] = {}

    def _run(*, case, condition, run_id, run_metadata, **_kwargs):
        captured_run_metadata.update(dict(run_metadata))
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--run-id",
            "openai-transport-settings",
            "--patcher",
            "openai",
            "--openai-model",
            "dummy-model",
            "--openai-transport-max-retries",
            "9",
            "--openai-transport-retry-backoff-base-sec",
            "1.5",
            "--openai-transport-retry-backoff-max-sec",
            "12.0",
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
        ]
    )

    assert exit_code == 0
    patcher_config = captured_run_metadata.get("patcher_config")
    assert isinstance(patcher_config, dict)
    assert patcher_config["openai_transport_max_retries"] == 9
    assert patcher_config["openai_http2"] is True
    assert patcher_config["openai_transport_retry_backoff_base_sec"] == 1.5
    assert patcher_config["openai_transport_retry_backoff_max_sec"] == 12.0


def test_run_eval_openai_http2_disable_in_patcher_config(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "openai_http2_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="openai_http2_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="openai"))

    captured_run_metadata: dict[str, object] = {}

    def _run(*, case, condition, run_id, run_metadata, **_kwargs):
        captured_run_metadata.update(dict(run_metadata))
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--run-id",
            "openai-http2-off",
            "--patcher",
            "openai",
            "--openai-model",
            "dummy-model",
            "--no-openai-http2",
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
        ]
    )

    assert exit_code == 0
    patcher_config = captured_run_metadata.get("patcher_config")
    assert isinstance(patcher_config, dict)
    assert patcher_config["openai_http2"] is False


def test_run_eval_case_list_file_filters_selected_cases(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_root = repo_root / "synthetic_cases"
    case_root.mkdir(parents=True, exist_ok=True)
    (case_root / "case_a").mkdir(parents=True, exist_ok=True)
    (case_root / "case_b").mkdir(parents=True, exist_ok=True)
    (case_root / "case_c").mkdir(parents=True, exist_ok=True)

    case_a = run_eval.CaseSpec(
        case_id="case_a",
        path=case_root / "case_a",
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )
    case_b = run_eval.CaseSpec(
        case_id="case_b",
        path=case_root / "case_b",
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )
    case_c = run_eval.CaseSpec(
        case_id="case_c",
        path=case_root / "case_c",
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    case_list_file = tmp_path / "selected_cases.txt"
    case_list_file.write_text("# comment\n\ncase_b\n", encoding="utf-8")

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case_a, case_b, case_c])
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="stub"))

    executed_case_ids: list[str] = []

    def _run(*, case, condition, run_id, run_metadata, **_kwargs):
        executed_case_ids.append(case.case_id)
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--run-id",
            "case-list-filter",
            "--patcher",
            "null",
            "--case",
            "case_a",
            "--case-list-file",
            str(case_list_file),
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
        ]
    )

    assert exit_code == 0
    assert sorted(executed_case_ids) == ["case_a", "case_b"]


def test_run_eval_case_list_file_rejects_unknown_case_id(tmp_path, monkeypatch, capsys) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_root = repo_root / "synthetic_cases"
    case_root.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="known_case",
        path=case_root / "known_case",
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    case_list_file = tmp_path / "selected_cases.txt"
    case_list_file.write_text("unknown_case\n", encoding="utf-8")

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])

    exit_code = run_eval.main(
        [
            "--run-id",
            "case-list-unknown",
            "--patcher",
            "null",
            "--case-list-file",
            str(case_list_file),
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
        ]
    )

    assert exit_code == 2
    assert "Unknown case id(s) requested via --case/--case-list-file" in capsys.readouterr().out


def test_run_eval_case_list_file_read_error_returns_cli_error(
    tmp_path, monkeypatch, capsys
) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_root = repo_root / "synthetic_cases"
    case_root.mkdir(parents=True, exist_ok=True)
    case_dir = case_root / "known_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="known_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    case_list_file = tmp_path / "selected_cases.txt"
    case_list_file.write_text("known_case\n", encoding="utf-8")

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])

    original_read_text = Path.read_text

    def _fake_read_text(path: Path, *args, **kwargs) -> str:
        if path == case_list_file:
            raise OSError("permission denied")
        return original_read_text(path, *args, **kwargs)

    monkeypatch.setattr(Path, "read_text", _fake_read_text)

    exit_code = run_eval.main(
        [
            "--run-id",
            "case-list-read-error",
            "--patcher",
            "null",
            "--case-list-file",
            str(case_list_file),
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
        ]
    )

    assert exit_code == 2
    assert "Failed to read case-list file" in capsys.readouterr().out


def test_run_eval_forwards_pytest_output_tail_lines(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "tail_lines_forward_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="tail_lines_forward_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="stub"))

    observed_tail_lines: dict[str, int] = {}

    def _run(*, case, condition, run_id, run_metadata, pytest_output_tail_lines, **_kwargs):
        observed_tail_lines["value"] = int(pytest_output_tail_lines)
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--run-id",
            "tail-lines-forward",
            "--patcher",
            "null",
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
            "--pytest-output-tail-lines",
            "123",
        ]
    )

    assert exit_code == 0
    assert observed_tail_lines["value"] == 123


def test_run_eval_forwards_snapshot_artifact_max_string_chars(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "snapshot_cap_forward_case"
    case_dir.mkdir(parents=True, exist_ok=True)

    case = run_eval.CaseSpec(
        case_id="snapshot_cap_forward_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="stub"))

    observed_snapshot_cap: dict[str, int] = {}

    def _run(
        *,
        case,
        condition,
        run_id,
        run_metadata,
        snapshot_artifact_max_string_chars,
        **_kwargs,
    ):
        observed_snapshot_cap["value"] = int(snapshot_artifact_max_string_chars)
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--run-id",
            "snapshot-cap-forward",
            "--patcher",
            "null",
            "--conditions",
            "traceback_only",
            "--k",
            "1",
            "--executor",
            "local",
            "--snapshot-artifact-max-string-chars",
            "321",
        ]
    )

    assert exit_code == 0
    assert observed_snapshot_cap["value"] == 321


def test_run_eval_config_precedence_and_resolved_lockfile(tmp_path, monkeypatch) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "config_precedence_case"
    case_dir.mkdir(parents=True, exist_ok=True)
    config_path = tmp_path / "run_eval.toml"
    config_path.write_text(
        """
[run_eval]
patcher = "null"
conditions = "traceback_only"
k = 3
run_id = "run-from-config"
executor = "local"
""".strip()
        + "\n",
        encoding="utf-8",
    )

    case = run_eval.CaseSpec(
        case_id="config_precedence_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    monkeypatch.setattr(run_eval, "make_patcher", lambda _args: SimpleNamespace(name="stub"))
    observed: dict[str, object] = {}

    def _run(*, case, condition, run_id, run_metadata, k, **_kwargs):
        observed["run_id"] = run_id
        observed["k"] = int(k)
        observed["run_metadata"] = dict(run_metadata)
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(
        [
            "--config",
            str(config_path),
            "--run-id",
            "run-from-cli",
            "--k",
            "1",
            "--executor",
            "local",
        ]
    )

    assert exit_code == 0
    assert observed["run_id"] == "run-from-cli"
    assert observed["k"] == 1
    lock_path = repo_root / "evals" / "artifacts" / "configs" / "run-from-cli.resolved.json"
    assert lock_path.exists()
    lock_payload = json.loads(lock_path.read_text(encoding="utf-8"))
    assert lock_payload["run_id"] == "run-from-cli"
    assert lock_payload["config_files"] == [str(config_path.resolve())]
    assert lock_payload["resolved_args"]["k"] == 1
    assert lock_payload["value_sources"]["k"] == "cli"
    assert str(lock_payload["value_sources"]["patcher"]).startswith("config:")


def test_run_eval_rejects_plaintext_openai_api_key_in_config(tmp_path, capsys) -> None:
    config_path = tmp_path / "run_eval.toml"
    config_path.write_text(
        """
[run_eval]
patcher = "openai"
openai_model = "demo"
openai_api_key = "sk-test-literal"
""".strip()
        + "\n",
        encoding="utf-8",
    )

    exit_code = run_eval.main(["--config", str(config_path)])
    assert exit_code == 2
    assert "openai_api_key" in capsys.readouterr().out


def test_run_eval_resolves_openai_api_key_env_from_config_and_redacts_lock(
    tmp_path, monkeypatch
) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "config_env_secret_case"
    case_dir.mkdir(parents=True, exist_ok=True)
    config_path = tmp_path / "run_eval.toml"
    config_path.write_text(
        """
[run_eval]
run_id = "config-env-secret-run"
patcher = "openai"
conditions = "traceback_only"
executor = "local"
k = 1

[run_eval.openai]
model = "demo"
api_key_env = "LLMDEBUG_TEST_OPENAI_KEY"
""".strip()
        + "\n",
        encoding="utf-8",
    )
    monkeypatch.setenv("LLMDEBUG_TEST_OPENAI_KEY", "super-secret-token")

    case = run_eval.CaseSpec(
        case_id="config_env_secret_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    observed_openai_key: dict[str, str] = {}

    def _make_patcher(_args):
        observed_openai_key["value"] = str(_args.openai_api_key)
        return SimpleNamespace(name="openai")

    monkeypatch.setattr(run_eval, "make_patcher", _make_patcher)

    def _run(*, case, condition, run_id, run_metadata, **_kwargs):
        return [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ]

    monkeypatch.setattr(run_eval, "run_one_condition", _run)

    exit_code = run_eval.main(["--config", str(config_path)])
    assert exit_code == 0
    assert observed_openai_key["value"] == "super-secret-token"

    lock_path = (
        repo_root / "evals" / "artifacts" / "configs" / "config-env-secret-run.resolved.json"
    )
    lock_payload = json.loads(lock_path.read_text(encoding="utf-8"))
    assert lock_payload["resolved_args"]["openai_api_key"] == run_eval.RUN_EVAL_LOCKFILE_REDACTED


def test_run_eval_config_openai_api_key_env_uses_standard_fallback_chain(
    tmp_path, monkeypatch
) -> None:
    repo_root = tmp_path / "repo"
    repo_root.mkdir(parents=True, exist_ok=True)
    case_dir = repo_root / "synthetic_cases" / "config_openai_env_fallback_case"
    case_dir.mkdir(parents=True, exist_ok=True)
    config_path = tmp_path / "run_eval.toml"
    config_path.write_text(
        """
[run_eval]
run_id = "config-openai-env-fallback-run"
patcher = "openai"
conditions = "traceback_only"
executor = "local"
k = 1
openai_api_key_env = "OPENAI_API_KEY"

[run_eval.openai]
model = "demo"
""".strip()
        + "\n",
        encoding="utf-8",
    )
    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
    monkeypatch.delenv("LLMDEBUG_EVAL_OPENAI_API_KEY", raising=False)
    monkeypatch.setenv("Z_API", "z-secret-token")

    case = run_eval.CaseSpec(
        case_id="config_openai_env_fallback_case",
        path=case_dir,
        category="traceback_controls",
        difficulty="easy",
        snapshot_dependency="none",
        expected_exception="AssertionError",
        tags=[],
        python_args=["-m", "pytest", "-q"],
        timeout_sec=30,
    )

    monkeypatch.setattr(run_eval, "_REPO_ROOT", repo_root)
    monkeypatch.setattr(run_eval, "load_cases", lambda _cases_dir: [case])
    observed_openai_key: dict[str, str] = {}

    def _make_patcher(_args):
        observed_openai_key["value"] = str(_args.openai_api_key)
        return SimpleNamespace(name="openai")

    monkeypatch.setattr(run_eval, "make_patcher", _make_patcher)
    monkeypatch.setattr(
        run_eval,
        "run_one_condition",
        lambda *, case, condition, run_id, run_metadata, **_kwargs: [
            {
                "run_id": run_id,
                "case_id": case.case_id,
                "condition": condition.value,
                "attempt": 1,
                "success": False,
                "run_metadata": dict(run_metadata),
            }
        ],
    )

    exit_code = run_eval.main(["--config", str(config_path)])
    assert exit_code == 0
    assert observed_openai_key["value"] == "z-secret-token"


def test_run_eval_config_openai_api_key_env_still_errors_when_all_standard_sources_missing(
    tmp_path, monkeypatch, capsys
) -> None:
    config_path = tmp_path / "run_eval.toml"
    config_path.write_text(
        """
[run_eval]
patcher = "openai"
openai_model = "demo"
openai_api_key_env = "OPENAI_API_KEY"
""".strip()
        + "\n",
        encoding="utf-8",
    )
    monkeypatch.delenv("Z_API", raising=False)
    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
    monkeypatch.delenv("LLMDEBUG_EVAL_OPENAI_API_KEY", raising=False)

    exit_code = run_eval.main(["--config", str(config_path)])
    assert exit_code == 2
    assert "OPENAI_API_KEY" in capsys.readouterr().out
