"""Tests for run-metrics reporting (effectiveness, efficiency, failure diagnostics)."""

from __future__ import annotations

from evals.report_run_metrics import (
    _compute_effectiveness_summary,
    _compute_efficiency_summaries,
    _compute_failure_diagnostics,
    _filter_rows_by_snapshot_dependency,
    _paired_sign_test,
    _parse_job_feedback_block,
    main,
)
from llmdebug import _json as json


def _record(
    *,
    run_id: str,
    case_id: str,
    condition: str,
    snapshot_dependency: str,
    category: str = "logic_error",
    difficulty: str = "easy",
    attempt: int = 1,
    success: bool = False,
    tokens_prompt: int = 10,
    tokens_completion: int = 2,
    tokens_total: int = 12,
    llm_call_count: int = 1,
    cumulative_runtime_sec: float = 1.0,
) -> dict[str, object]:
    return {
        "schema_version": 5,
        "run_id": run_id,
        "case_id": case_id,
        "condition": condition,
        "attempt": attempt,
        "success": success,
        "category": category,
        "difficulty": difficulty,
        "snapshot_dependency": snapshot_dependency,
        "run_metadata": {
            "run_id": run_id,
            "started_at_utc": "2026-02-18T00:00:00Z",
            "deterministic": False,
            "seed": None,
            "python_version": "3.11.0",
            "platform": "test",
            "llmdebug_version": "2.11.4",
            "patcher_name": "openai",
            "patcher_config_fingerprint": "fp",
            "run_settings_fingerprint": "runfp",
        },
        "attempt_metadata": {
            "attempt": attempt,
            "max_attempts": 3,
            "attempt_ordinal_label": f"{attempt}/3",
            "llm_call_count": llm_call_count,
            "tokens_prompt": tokens_prompt,
            "tokens_completion": tokens_completion,
            "tokens_total": tokens_total,
            "retry_history_budget_chars": 12000,
            "retry_history_budget_applied": False,
            "prompt_sha256": "p",
            "prompt_path": None,
            "evidence_sha256": "e",
            "evidence_path": None,
        },
        "model_metadata": {
            "backend": "openai",
            "name": "openai",
            "base_url": "http://127.0.0.1:8000/v1",
            "model": "dummy",
            "temperature": 0.0,
            "max_tokens": 2048,
            "response_mode": "edits",
            "api_compat_seed_requested": None,
            "api_compat_seed_acknowledged": False,
            "api_compat_seed_response": None,
        },
        "cumulative_runtime_sec": cumulative_runtime_sec,
        "note": "tests_still_failing" if not success else "success",
    }


def test_parse_job_feedback_block_extracts_energy_and_power() -> None:
    text = """
============================= JOB FEEDBACK =============================
Job Wall-clock time: 03:43:16
Energy Consumed: 8982304 Joule / 2495.08444444444 Watthours
Average node power draw: 670.521349656614 Watt
"""
    parsed = _parse_job_feedback_block(text)
    assert parsed["job wall-clock time"] == "03:43:16"
    assert parsed["energy_joule"] == 8982304.0
    assert parsed["energy_watthours"] == 2495.08444444444
    assert parsed["average_node_power_watt"] == 670.521349656614


def test_failure_diagnostics_detects_context_overflow_rates() -> None:
    records = [
        {
            "run_id": "r1",
            "case_id": "c1",
            "condition": "traceback_only",
            "success": False,
            "note": "tests_still_failing",
            "patch_meta": {
                "error": "http_error",
                "status": 400,
                "body": "request exceeds the available context size",
            },
        },
        {
            "run_id": "r1",
            "case_id": "c1",
            "condition": "traceback_only",
            "success": False,
            "note": "tests_still_failing",
            "patch_meta": {"error": "no_json_payload"},
        },
        {
            "run_id": "r1",
            "case_id": "c2",
            "condition": "with_snapshot",
            "success": False,
            "note": "tests_still_failing",
            "patch_meta": {
                "error": "http_error",
                "status": 400,
                "body": "request exceeds the available context size",
            },
        },
    ]
    out = _compute_failure_diagnostics(
        raw_records=records,
        condition_a="traceback_only",
        condition_b="with_snapshot",
        top_n=5,
    )
    tb = out["per_condition"]["traceback_only"]
    ws = out["per_condition"]["with_snapshot"]
    assert tb["attempts"] == 2
    assert tb["context_overflow_attempts"] == 1
    assert tb["context_overflow_cases"] == 1
    assert tb["http_error_attempts"] == 1
    assert tb["patch_attempts"] == 0
    assert tb["test_touch_attempts"] == 0
    assert ws["attempts"] == 1
    assert ws["context_overflow_attempt_rate"] == 1.0
    assert ws["patch_attempts"] == 0
    assert ws["test_touch_attempts"] == 0


def test_failure_diagnostics_tracks_test_touch_patch_attempt_rates() -> None:
    records = [
        {
            "run_id": "r1",
            "case_id": "c1",
            "condition": "traceback_only",
            "success": False,
            "note": "tests_still_failing",
            "patch_meta": {"patch_file_count": 2, "patch_touches_tests": True},
        },
        {
            "run_id": "r1",
            "case_id": "c1",
            "condition": "traceback_only",
            "success": False,
            "note": "tests_still_failing",
            "patch_meta": {"patch_file_count": 1, "patch_touches_tests": False},
        },
        {
            "run_id": "r1",
            "case_id": "c2",
            "condition": "with_snapshot",
            "success": False,
            "note": "tests_still_failing",
            "patch_meta": {"patch_file_count": 1, "patch_touches_tests": True},
        },
    ]
    out = _compute_failure_diagnostics(
        raw_records=records,
        condition_a="traceback_only",
        condition_b="with_snapshot",
        top_n=5,
    )
    tb = out["per_condition"]["traceback_only"]
    ws = out["per_condition"]["with_snapshot"]
    assert tb["patch_attempts"] == 2
    assert tb["test_touch_attempts"] == 1
    assert tb["test_touch_attempt_rate"] == 0.5
    assert ws["patch_attempts"] == 1
    assert ws["test_touch_attempts"] == 1
    assert ws["test_touch_attempt_rate"] == 1.0


def test_effectiveness_summary_on_primary_slice() -> None:
    rows = [
        {
            "run_id": "r1",
            "case_id": "c_req",
            "condition": "traceback_only",
            "category": "hidden_state",
            "snapshot_dependency": "required",
            "success": False,
        },
        {
            "run_id": "r1",
            "case_id": "c_req",
            "condition": "with_snapshot",
            "category": "hidden_state",
            "snapshot_dependency": "required",
            "success": True,
        },
        {
            "run_id": "r1",
            "case_id": "c_help",
            "condition": "traceback_only",
            "category": "logic_error",
            "snapshot_dependency": "helpful",
            "success": False,
        },
        {
            "run_id": "r1",
            "case_id": "c_help",
            "condition": "with_snapshot",
            "category": "logic_error",
            "snapshot_dependency": "helpful",
            "success": True,
        },
        {
            "run_id": "r1",
            "case_id": "c_none",
            "condition": "traceback_only",
            "category": "traceback_controls",
            "snapshot_dependency": "none",
            "success": True,
        },
        {
            "run_id": "r1",
            "case_id": "c_none",
            "condition": "with_snapshot",
            "category": "traceback_controls",
            "snapshot_dependency": "none",
            "success": False,
        },
    ]
    primary_rows = _filter_rows_by_snapshot_dependency(rows, {"required", "helpful"})
    result = _compute_effectiveness_summary(
        rows=primary_rows,
        condition_a="traceback_only",
        condition_b="with_snapshot",
        n_bootstrap=200,
        seed=123,
    )
    assert result["available"] is True
    assert result["n_pairs"] == 2
    assert result["a_rate"] == 0.0
    assert result["b_rate"] == 1.0
    assert result["uplift_pp"] == 100.0
    assert result["agreement"]["available"] is True
    assert -1.0 <= result["agreement"]["kappa"] <= 1.0


def test_efficiency_summary_has_sign_test_and_bootstrap() -> None:
    rows = [
        {
            "run_id": "r1",
            "case_id": "c1",
            "condition": "traceback_only",
            "token_usage_total": 100.0,
            "runtime_sec": 10.0,
            "llm_calls_total": 3.0,
        },
        {
            "run_id": "r1",
            "case_id": "c1",
            "condition": "with_snapshot",
            "token_usage_total": 90.0,
            "runtime_sec": 9.0,
            "llm_calls_total": 2.0,
        },
        {
            "run_id": "r1",
            "case_id": "c2",
            "condition": "traceback_only",
            "token_usage_total": 50.0,
            "runtime_sec": 5.0,
            "llm_calls_total": 1.0,
        },
        {
            "run_id": "r1",
            "case_id": "c2",
            "condition": "with_snapshot",
            "token_usage_total": 70.0,
            "runtime_sec": 6.0,
            "llm_calls_total": 2.0,
        },
    ]
    result = _compute_efficiency_summaries(
        rows=rows,
        condition_a="traceback_only",
        condition_b="with_snapshot",
        n_bootstrap=200,
        seed=11,
    )
    tokens = result["token_usage_total"]
    assert tokens["available"] is True
    assert tokens["n_pairs"] == 2
    assert "sign_test" in tokens
    assert "wilcoxon" in tokens
    assert "bootstrap_ci_mean_delta" in tokens


def test_paired_sign_test_handles_large_nonzero_count() -> None:
    deltas = [1.0] * 1000 + [-1.0] * 600
    out = _paired_sign_test(deltas)
    assert out["n_nonzero"] == 1600
    assert out["n_positive"] == 1000
    assert out["n_negative"] == 600
    assert 0.0 <= out["p_value"] <= 1.0


def test_main_smoke_prints_expected_sections(tmp_path, capsys) -> None:
    jsonl = tmp_path / "mini.jsonl"
    records = [
        _record(
            run_id="run-smoke",
            case_id="case-required",
            condition="traceback_only",
            snapshot_dependency="required",
            success=False,
            tokens_total=100,
            cumulative_runtime_sec=10.0,
            llm_call_count=3,
        ),
        _record(
            run_id="run-smoke",
            case_id="case-required",
            condition="with_snapshot",
            snapshot_dependency="required",
            success=True,
            tokens_total=120,
            cumulative_runtime_sec=11.0,
            llm_call_count=2,
        ),
        _record(
            run_id="run-smoke",
            case_id="case-control",
            condition="traceback_only",
            snapshot_dependency="none",
            category="traceback_controls",
            success=True,
            tokens_total=80,
            cumulative_runtime_sec=8.0,
            llm_call_count=1,
        ),
        _record(
            run_id="run-smoke",
            case_id="case-control",
            condition="with_snapshot",
            snapshot_dependency="none",
            category="traceback_controls",
            success=False,
            tokens_total=90,
            cumulative_runtime_sec=9.0,
            llm_call_count=2,
        ),
    ]
    jsonl.write_text(
        "\n".join(json.dumps(record, ensure_ascii=True) for record in records) + "\n",
        encoding="utf-8",
    )

    eval_log = tmp_path / "eval.out"
    eval_log.write_text(
        "\n".join(
            [
                "Progress: finished executed=4, skipped=0, total=4, elapsed=10m00s",
                "============================= JOB FEEDBACK =============================",
                "Job Wall-clock time: 00:10:02",
                "CPU Utilized: 00:12:00",
                "Memory Utilized: 2.00 GB",
                "Energy Consumed: 1000 Joule / 0.2777777777 Watthours",
                "Average node power draw: 100 Watt",
            ]
        )
        + "\n",
        encoding="utf-8",
    )

    rc = main(
        [
            "--jsonl",
            str(jsonl),
            "--eval-log",
            str(eval_log),
            "--bootstrap",
            "200",
            "--seed",
            "1",
            "--print-top-failures",
            "3",
        ]
    )
    assert rc == 0
    out = capsys.readouterr().out
    assert "Primary Endpoint" in out
    assert "Secondary Efficiency" in out
    assert "Failure Diagnostics" in out
    assert "Run-Level System Context" in out
    assert "Energy consumed (run-level)" in out
    assert "One-Line Conclusion" in out
