from __future__ import annotations

import json

import pytest

from evals.analyze_results import (
    _derive_case_condition_rows,
    _load_and_group_records,
    _retry_gate_counts,
    _summarize_bucket,
)
from evals.analyze_results import (
    main as analyze_main,
)


def _record(
    *,
    run_id: str,
    case_id: str,
    condition: str,
    attempt: int,
    success: bool,
    category: str,
    difficulty: str,
    snapshot_dependency: str,
    **overrides,
):
    record = {
        "schema_version": 3,
        "run_id": run_id,
        "case_id": case_id,
        "condition": condition,
        "attempt": attempt,
        "success": success,
        "category": category,
        "difficulty": difficulty,
        "snapshot_dependency": snapshot_dependency,
        "run_metadata": {
            "run_id": run_id,
            "started_at_utc": "2026-02-10T00:00:00Z",
            "deterministic": False,
            "seed": None,
            "python_version": "3.12.0",
            "platform": "test",
            "llmdebug_version": "2.1.0",
            "patcher_name": "openai",
            "patcher_config_fingerprint": "abc123",
            "run_settings_fingerprint": "runfp123",
        },
        "attempt_metadata": {
            "attempt": attempt,
            "max_attempts": 3,
            "attempt_ordinal_label": f"{attempt}/3",
            "llm_call_count": 1,
            "tokens_prompt": 0,
            "tokens_completion": 0,
            "tokens_total": 0,
            "retry_history_budget_chars": 12000,
            "retry_history_budget_applied": False,
            "prompt_sha256": "p",
            "prompt_path": None,
            "evidence_sha256": "e",
            "evidence_path": None,
        },
        "model_metadata": {
            "backend": "openai",
            "name": "openai",
            "base_url": "http://127.0.0.1:1234/v1",
            "model": "dummy",
            "temperature": 0.0,
            "max_tokens": 2048,
            "response_mode": "edits",
            "api_compat_seed_requested": None,
            "api_compat_seed_acknowledged": False,
            "api_compat_seed_response": None,
        },
    }
    record.update(overrides)
    return record


def test_analyze_grouping_and_uplift(tmp_path) -> None:
    records = [
        _record(
            run_id="run_a",
            case_id="case_one",
            condition="traceback_only",
            category="hidden_state",
            difficulty="easy",
            snapshot_dependency="required",
            attempt=1,
            success=False,
            before_duration_sec=1.0,
            after_duration_sec=0.5,
            cumulative_runtime_sec=1.6,
            patch_meta={"usage": {"total_tokens": 10}},
        ),
        _record(
            run_id="run_a",
            case_id="case_one",
            condition="traceback_only",
            category="hidden_state",
            difficulty="easy",
            snapshot_dependency="required",
            attempt=2,
            success=True,
            before_duration_sec=0.5,
            after_duration_sec=0.2,
            cumulative_runtime_sec=2.1,
            patch_meta={"usage": {"total_tokens": 20}},
        ),
        _record(
            run_id="run_a",
            case_id="case_one",
            condition="with_snapshot",
            category="hidden_state",
            difficulty="easy",
            snapshot_dependency="required",
            attempt=1,
            success=True,
            before_duration_sec=0.9,
            after_duration_sec=0.2,
            cumulative_runtime_sec=1.4,
            patch_meta={"usage": {"total_tokens": 15}},
        ),
        _record(
            run_id="run_a",
            case_id="case_one",
            condition="with_snapshot_structured",
            category="hidden_state",
            difficulty="easy",
            snapshot_dependency="required",
            attempt=1,
            success=True,
            before_duration_sec=0.8,
            after_duration_sec=0.2,
            cumulative_runtime_sec=1.3,
            patch_meta={"usage": {"total_tokens": 14}},
        ),
        _record(
            run_id="run_a",
            case_id="case_two",
            condition="traceback_only",
            category="parse_value",
            difficulty="medium",
            snapshot_dependency="helpful",
            attempt=1,
            success=False,
            before_duration_sec=1.1,
            cumulative_runtime_sec=1.3,
        ),
        _record(
            run_id="run_a",
            case_id="case_two",
            condition="with_snapshot",
            category="parse_value",
            difficulty="medium",
            snapshot_dependency="helpful",
            attempt=1,
            success=False,
            before_duration_sec=1.0,
            cumulative_runtime_sec=1.5,
            patch_meta={"usage": {"total_tokens": 5}},
        ),
        _record(
            run_id="run_b",
            case_id="case_one",
            condition="traceback_only",
            category="hidden_state",
            difficulty="easy",
            snapshot_dependency="required",
            attempt=1,
            success=False,
            before_duration_sec=1.3,
            cumulative_runtime_sec=1.8,
        ),
        _record(
            run_id="run_b",
            case_id="case_one",
            condition="with_snapshot",
            category="hidden_state",
            difficulty="easy",
            snapshot_dependency="required",
            attempt=1,
            success=True,
            before_duration_sec=1.2,
            after_duration_sec=0.1,
            cumulative_runtime_sec=1.6,
            patch_meta={"usage": {"total_tokens": 12}},
        ),
    ]
    jsonl = tmp_path / "results.jsonl"
    jsonl.write_text("\n".join(json.dumps(record) for record in records) + "\n", encoding="utf-8")

    grouped, input_errors = _load_and_group_records([str(jsonl)])
    assert len(grouped) == 7
    assert input_errors == {"files_unreadable": 0, "json_lines_invalid": 0, "records_skipped": 0}

    rows = _derive_case_condition_rows(grouped)
    assert len(rows) == 7
    row_index = {(row["run_id"], row["case_id"], row["condition"]): row for row in rows}
    assert row_index[("run_a", "case_one", "traceback_only")]["runtime_sec"] == pytest.approx(2.1)

    overall = _summarize_bucket(rows)
    assert overall["traceback_only"]["cases"] == 3
    assert overall["traceback_only"]["successes"] == 1
    assert overall["traceback_only"]["success_rate"] == pytest.approx(1 / 3)
    assert overall["traceback_only"]["median_attempts_to_success"] == pytest.approx(2.0)
    assert overall["traceback_only"]["mean_attempts_budget"] == pytest.approx(3.0)
    assert overall["traceback_only"]["solve_on_attempt_distribution"] == {"2": 1}
    assert overall["with_snapshot"]["cases"] == 3
    assert overall["with_snapshot"]["successes"] == 2
    assert overall["with_snapshot"]["success_rate"] == pytest.approx(2 / 3)
    assert overall["uplift_success_rate"] == pytest.approx(1 / 3)
    assert "mean_attempts_budget" in overall["traceback_only"]
    assert "mean_attempt_ratio_when_successful" in overall["traceback_only"]
    assert "solve_on_attempt_distribution" in overall["traceback_only"]
    assert "mean_tokens_prompt" in overall["traceback_only"]
    assert "mean_tokens_completion" in overall["traceback_only"]
    assert "mean_tokens_total" in overall["traceback_only"]
    assert "mean_tokens_per_attempt" in overall["traceback_only"]
    assert "mean_llm_calls_per_case" in overall["traceback_only"]
    assert overall["with_snapshot_structured"]["cases"] == 1
    assert overall["with_snapshot_structured"]["success_rate"] == pytest.approx(1.0)
    assert "mean_prompt_chars_stage_b" in overall
    assert "mean_retry_history_chars" in overall
    assert "mean_retry_history_turns" in overall
    assert "retry_history_budget_applied_rate" in overall
    assert "retry_history_truncation_rate" in overall
    assert "stage_a_request_success_rate" in overall
    assert "uplift_staged_vs_monolithic" in overall
    assert "retry_gate_block_rate" in overall
    assert "pairwise_uplifts" in overall
    assert isinstance(overall["pairwise_uplifts"], list)
    assert overall["uplift_structured_vs_snapshot"] == pytest.approx(1 / 3)
    assert overall["uplift_structured_vs_traceback"] == pytest.approx(2 / 3)

    by_category = {"hidden_state": [], "parse_value": []}
    for row in rows:
        by_category[row["category"]].append(row)
    hidden_state_summary = _summarize_bucket(by_category["hidden_state"])
    assert hidden_state_summary["with_snapshot"]["success_rate"] == pytest.approx(1.0)
    assert hidden_state_summary["traceback_only"]["success_rate"] == pytest.approx(0.5)
    assert hidden_state_summary["uplift_success_rate"] == pytest.approx(0.5)

    by_snapshot_dependency = {"required": [], "helpful": []}
    for row in rows:
        by_snapshot_dependency[row["snapshot_dependency"]].append(row)
    required_summary = _summarize_bucket(by_snapshot_dependency["required"])
    assert required_summary["traceback_only"]["cases"] == 2
    assert required_summary["with_snapshot"]["cases"] == 2
    assert required_summary["uplift_success_rate"] == pytest.approx(0.5)


def test_analyze_grouping_tolerates_malformed_jsonl(tmp_path, capsys) -> None:
    valid_a = _record(
        run_id="run",
        case_id="case",
        condition="traceback_only",
        category="hidden_state",
        difficulty="easy",
        snapshot_dependency="required",
        attempt=1,
        success=False,
    )
    valid_b = _record(
        run_id="run",
        case_id="case",
        condition="with_snapshot",
        category="hidden_state",
        difficulty="easy",
        snapshot_dependency="required",
        attempt=1,
        success=True,
    )
    jsonl = tmp_path / "results_mixed.jsonl"
    jsonl.write_text(
        "\n".join(
            [
                json.dumps(valid_a),
                "this-is-not-json",
                json.dumps(valid_b),
                '{"run_id":"run","condition":"with_snapshot","attempt":2}',
            ]
        )
        + "\n",
        encoding="utf-8",
    )

    grouped, input_errors = _load_and_group_records([str(jsonl)])
    assert len(grouped) == 2
    assert input_errors["json_lines_invalid"] == 1
    assert input_errors["records_skipped"] == 1

    exit_code = analyze_main([str(jsonl)])
    assert exit_code == 0
    output = capsys.readouterr().out
    assert "Input diagnostics:" in output
    assert "json_lines_invalid=1" in output
    assert "records_skipped=1" in output


def test_retry_gate_counts_use_evaluated_flag() -> None:
    attempts, blocks = _retry_gate_counts(
        [
            {"patch_meta": {"retry_gate_evaluated": True, "retry_gate_blocked": False}},
            {"patch_meta": {"retry_gate_evaluated": True, "retry_gate_blocked": True}},
            {"patch_meta": {"retry_gate_blocked": True}},
            {"patch_meta": {"retry_gate_blocked": None}},
            {"patch_meta": {}},
            {},
        ]
    )
    assert attempts == 3
    assert blocks == 2


def test_analyze_grouping_strict_input_fails_on_malformed_jsonl(tmp_path, capsys) -> None:
    valid = _record(
        run_id="run",
        case_id="case",
        condition="traceback_only",
        category="hidden_state",
        difficulty="easy",
        snapshot_dependency="required",
        attempt=1,
        success=False,
    )
    jsonl = tmp_path / "results_bad.jsonl"
    jsonl.write_text(
        json.dumps(valid) + "\nnot-json\n",
        encoding="utf-8",
    )

    exit_code = analyze_main(["--strict-input", str(jsonl)])
    assert exit_code == 2
    output = capsys.readouterr().out
    assert "Input error:" in output


def test_analyze_grouping_rejects_missing_schema_fields(tmp_path, capsys) -> None:
    jsonl = tmp_path / "results_missing_schema.jsonl"
    jsonl.write_text(
        '{"run_id":"run","case_id":"case","condition":"traceback_only","attempt":1,"success":false}\n',
        encoding="utf-8",
    )

    exit_code = analyze_main([str(jsonl)])
    assert exit_code == 2
    output = capsys.readouterr().out
    assert "missing required field" in output
