from __future__ import annotations

import argparse
import glob
import math
import re
from pathlib import Path
from typing import Any, cast

import polars as pl
from scipy import stats
from sklearn.metrics import cohen_kappa_score

from evals.analyze_results import _derive_case_condition_rows, _load_and_group_records
from evals.stats import analyze_paired_results


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description=(
            "Generate a paper-ready plain-text report for one llmdebug eval run "
            "(effectiveness + secondary efficiency + diagnostics)."
        )
    )
    parser.add_argument(
        "--jsonl",
        required=True,
        help="Path (or glob) to eval JSONL file(s).",
    )
    parser.add_argument(
        "--eval-log",
        default="",
        help="Optional path to eval stdout log containing run-level system/energy feedback.",
    )
    parser.add_argument(
        "--a",
        default="traceback_only",
        help="Condition name for baseline A side.",
    )
    parser.add_argument(
        "--b",
        default="with_snapshot",
        help="Condition name for treatment B side.",
    )
    parser.add_argument(
        "--primary-slice",
        default="required,helpful",
        help="Comma-separated snapshot_dependency values for primary endpoint.",
    )
    parser.add_argument(
        "--bootstrap",
        type=int,
        default=10000,
        help="Bootstrap resamples for confidence intervals.",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=42,
        help="Random seed for bootstrap.",
    )
    parser.add_argument(
        "--print-top-failures",
        type=int,
        default=10,
        help="Number of top failure reasons to print per condition.",
    )
    parser.add_argument(
        "--strict-input",
        action="store_true",
        help="Fail on first unreadable input file or malformed JSONL record.",
    )
    args = parser.parse_args(argv)

    expanded = _expand_paths([args.jsonl])
    if not expanded:
        print(f"No input matched --jsonl={args.jsonl!r}")
        return 1

    try:
        grouped, input_errors = _load_and_group_records(expanded, strict_input=args.strict_input)
    except ValueError as exc:
        print(f"Input error: {exc}")
        return 2

    if not grouped:
        print("No records found.")
        _print_input_errors(input_errors)
        return 1

    rows = _derive_case_condition_rows(grouped)
    if not rows:
        print("No valid case-condition rows found.")
        return 1

    raw_records = _flatten_grouped_records(grouped)
    primary_slice = _parse_primary_slice(args.primary_slice)
    run_context = _build_run_context(
        rows=rows, raw_records=raw_records, eval_log_path=args.eval_log
    )
    primary_rows = _filter_rows_by_snapshot_dependency(rows, primary_slice)
    required_rows = _filter_rows_by_snapshot_dependency(rows, {"required"})
    control_rows = _filter_rows_by_snapshot_dependency(rows, {"none"})

    overall_effect = _compute_effectiveness_summary(
        rows=rows,
        condition_a=args.a,
        condition_b=args.b,
        n_bootstrap=args.bootstrap,
        seed=args.seed,
    )
    primary_effect = _compute_effectiveness_summary(
        rows=primary_rows,
        condition_a=args.a,
        condition_b=args.b,
        n_bootstrap=args.bootstrap,
        seed=args.seed,
    )
    required_effect = _compute_effectiveness_summary(
        rows=required_rows,
        condition_a=args.a,
        condition_b=args.b,
        n_bootstrap=args.bootstrap,
        seed=args.seed,
    )
    control_effect = _compute_effectiveness_summary(
        rows=control_rows,
        condition_a=args.a,
        condition_b=args.b,
        n_bootstrap=args.bootstrap,
        seed=args.seed,
    )
    category_stats = _compute_category_stats(
        rows=primary_rows,
        condition_a=args.a,
        condition_b=args.b,
        n_bootstrap=args.bootstrap,
        seed=args.seed,
    )
    efficiency = _compute_efficiency_summaries(
        rows=primary_rows,
        condition_a=args.a,
        condition_b=args.b,
        n_bootstrap=max(200, args.bootstrap // 2),
        seed=args.seed,
    )
    failures = _compute_failure_diagnostics(
        raw_records=raw_records,
        condition_a=args.a,
        condition_b=args.b,
        top_n=max(1, args.print_top_failures),
    )

    _print_report(
        run_context=run_context,
        condition_a=args.a,
        condition_b=args.b,
        primary_slice=primary_slice,
        overall_effect=overall_effect,
        primary_effect=primary_effect,
        required_effect=required_effect,
        control_effect=control_effect,
        category_stats=category_stats,
        efficiency=efficiency,
        failures=failures,
        input_errors=input_errors,
    )
    return 0


def _expand_paths(items: list[str]) -> list[str]:
    out: list[str] = []
    for item in items:
        matches = sorted(glob.glob(item))
        out.extend(matches if matches else [item])
    return out


def _flatten_grouped_records(
    grouped: dict[tuple[str, str, str], list[dict[str, Any]]],
) -> list[dict[str, Any]]:
    out: list[dict[str, Any]] = []
    for key in sorted(grouped):
        out.extend(grouped[key])
    return out


def _parse_primary_slice(raw: str) -> set[str]:
    values = {part.strip() for part in raw.split(",") if part.strip()}
    return values or {"required", "helpful"}


def _filter_rows_by_snapshot_dependency(
    rows: list[dict[str, Any]],
    allowed: set[str],
) -> list[dict[str, Any]]:
    return [row for row in rows if str(row.get("snapshot_dependency") or "") in allowed]


def _compute_effectiveness_summary(
    *,
    rows: list[dict[str, Any]],
    condition_a: str,
    condition_b: str,
    n_bootstrap: int,
    seed: int | None,
) -> dict[str, Any]:
    result = analyze_paired_results(
        rows,
        n_bootstrap=max(0, n_bootstrap),
        seed=seed,
        condition_a=condition_a,
        condition_b=condition_b,
    )
    if "error" in result:
        return {"available": False, "error": result["error"]}

    a_rate = float(result["a_success_rate"])
    b_rate = float(result["b_success_rate"])
    uplift = b_rate - a_rate
    relative = (uplift / a_rate) if a_rate > 0 else None
    agreement = _paired_binary_agreement(
        rows=rows,
        condition_a=condition_a,
        condition_b=condition_b,
    )
    return {
        "available": True,
        "n_pairs": int(result["n_pairs"]),
        "a_rate": a_rate,
        "b_rate": b_rate,
        "uplift": uplift,
        "uplift_pp": uplift * 100.0,
        "relative_change": relative,
        "mcnemar": result.get("overall_mcnemar", {}),
        "cohens_h": result.get("overall_cohens_h", {}),
        "agreement": agreement,
        "bootstrap_ci": result.get("overall_bootstrap_ci", {}),
        "power": result.get("overall_power", {}),
        "raw": result,
    }


def _paired_binary_agreement(
    *,
    rows: list[dict[str, Any]],
    condition_a: str,
    condition_b: str,
) -> dict[str, Any]:
    normalized: list[dict[str, Any]] = []
    for row in rows:
        run_id = str(row.get("run_id") or "")
        case_id = str(row.get("case_id") or "")
        condition = str(row.get("condition") or "")
        if not run_id or not case_id or condition not in {condition_a, condition_b}:
            continue
        normalized.append(
            {
                "run_id": run_id,
                "case_id": case_id,
                "condition": condition,
                "success": int(bool(row.get("success"))),
            }
        )

    if not normalized:
        return {"available": False, "error": "No matched A/B pairs found."}

    frame = pl.DataFrame(normalized)
    paired = (
        frame.group_by(["run_id", "case_id"])
        .agg(
            pl.col("success").filter(pl.col("condition") == condition_a).first().alias("a_success"),
            pl.col("success").filter(pl.col("condition") == condition_b).first().alias("b_success"),
        )
        .drop_nulls(["a_success", "b_success"])
    )
    if paired.is_empty():
        return {"available": False, "error": "No matched A/B pairs found."}

    a_success = paired["a_success"].cast(pl.Int64).to_list()
    b_success = paired["b_success"].cast(pl.Int64).to_list()
    kappa = float(cohen_kappa_score(a_success, b_success))
    return {
        "available": True,
        "kappa": kappa,
        "magnitude": _cohens_kappa_magnitude(kappa),
    }


def _cohens_kappa_magnitude(kappa: float) -> str:
    abs_kappa = abs(kappa)
    if abs_kappa < 0.2:
        return "slight"
    if abs_kappa < 0.4:
        return "fair"
    if abs_kappa < 0.6:
        return "moderate"
    if abs_kappa < 0.8:
        return "substantial"
    return "almost_perfect"


def _compute_category_stats(
    *,
    rows: list[dict[str, Any]],
    condition_a: str,
    condition_b: str,
    n_bootstrap: int,
    seed: int | None,
) -> dict[str, Any]:
    result = analyze_paired_results(
        rows,
        n_bootstrap=max(0, n_bootstrap),
        seed=seed,
        condition_a=condition_a,
        condition_b=condition_b,
    )
    if "error" in result:
        return {"available": False, "error": result["error"], "rows": []}

    per_category = result.get("per_category", {})
    holm = result.get("holm_bonferroni", {})
    rows_out: list[dict[str, Any]] = []
    if isinstance(per_category, dict):
        for category, value in sorted(per_category.items()):
            if not isinstance(value, dict):
                continue
            a_rate = value.get("a_success_rate")
            b_rate = value.get("b_success_rate")
            uplift = (
                float(b_rate) - float(a_rate)
                if isinstance(a_rate, (int, float)) and isinstance(b_rate, (int, float))
                else None
            )
            mcn = value.get("mcnemar", {}) if isinstance(value.get("mcnemar"), dict) else {}
            hb = holm.get(category, {}) if isinstance(holm, dict) else {}
            rows_out.append(
                {
                    "category": category,
                    "n_pairs": int(value.get("n_pairs", 0)),
                    "uplift": uplift,
                    "mcnemar_p": _as_float(mcn.get("p_value")),
                    "mcnemar_significant": bool(mcn.get("significant", False)),
                    "holm_adjusted_p": _as_float(hb.get("adjusted_p")),
                    "holm_rejected": bool(hb.get("rejected", False)),
                    "cohens_h": _as_float((value.get("cohens_h") or {}).get("h")),
                }
            )
    rows_out.sort(
        key=lambda item: (
            _null_sort_key(item.get("holm_adjusted_p")),
            -abs(_as_float(item.get("uplift")) or 0.0),
        )
    )
    return {"available": True, "rows": rows_out}


def _null_sort_key(value: Any) -> tuple[int, float]:
    if isinstance(value, (int, float)):
        return (0, float(value))
    return (1, math.inf)


def _compute_efficiency_summaries(
    *,
    rows: list[dict[str, Any]],
    condition_a: str,
    condition_b: str,
    n_bootstrap: int,
    seed: int | None,
) -> dict[str, dict[str, Any]]:
    metrics = {
        "token_usage_total": "Tokens total (lower is better)",
        "runtime_sec": "Runtime sec (lower is better)",
        "llm_calls_total": "LLM calls total (lower is better)",
    }
    out: dict[str, dict[str, Any]] = {}
    for metric, label in metrics.items():
        deltas, a_values, b_values = _paired_metric_deltas(
            rows=rows,
            condition_a=condition_a,
            condition_b=condition_b,
            metric=metric,
        )
        out[metric] = _summarize_paired_deltas(
            label=label,
            deltas=deltas,
            a_values=a_values,
            b_values=b_values,
            n_bootstrap=n_bootstrap,
            seed=seed,
        )
    return out


def _paired_metric_deltas(
    *,
    rows: list[dict[str, Any]],
    condition_a: str,
    condition_b: str,
    metric: str,
) -> tuple[list[float], list[float], list[float]]:
    normalized: list[dict[str, Any]] = []
    for row in rows:
        run_id = str(row.get("run_id") or "")
        case_id = str(row.get("case_id") or "")
        condition = str(row.get("condition") or "")
        if not run_id or not case_id or condition not in {condition_a, condition_b}:
            continue
        metric_value = row.get(metric)
        if not isinstance(metric_value, (int, float)):
            continue
        normalized.append(
            {
                "run_id": run_id,
                "case_id": case_id,
                "condition": condition,
                metric: float(metric_value),
            }
        )

    if not normalized:
        return [], [], []

    frame = pl.DataFrame(normalized)
    paired = (
        frame.group_by(["run_id", "case_id"])
        .agg(
            pl.col(metric).filter(pl.col("condition") == condition_a).first().alias("a_value"),
            pl.col(metric).filter(pl.col("condition") == condition_b).first().alias("b_value"),
        )
        .drop_nulls(["a_value", "b_value"])
        .sort(["run_id", "case_id"])
    )
    if paired.is_empty():
        return [], [], []

    a_values = paired["a_value"].cast(pl.Float64).to_list()
    b_values = paired["b_value"].cast(pl.Float64).to_list()
    deltas = [b_value - a_value for a_value, b_value in zip(a_values, b_values, strict=True)]
    return deltas, a_values, b_values


def _summarize_paired_deltas(
    *,
    label: str,
    deltas: list[float],
    a_values: list[float],
    b_values: list[float],
    n_bootstrap: int,
    seed: int | None,
) -> dict[str, Any]:
    if not deltas:
        return {"available": False, "label": label, "error": "No matched numeric pairs."}

    mean_delta = _mean(deltas)
    median_delta = _median(deltas)
    mean_a = _mean(a_values)
    mean_b = _mean(b_values)
    pct_change = (
        ((mean_b - mean_a) / mean_a)
        if mean_a not in (None, 0) and mean_b is not None
        else None
    )
    sign = _paired_sign_test(deltas)
    wilcoxon = _paired_wilcoxon_test(deltas)
    ci = _bootstrap_mean_ci(
        values=deltas,
        n_bootstrap=max(0, n_bootstrap),
        confidence=0.95,
        seed=seed,
    )
    return {
        "available": True,
        "label": label,
        "n_pairs": len(deltas),
        "mean_a": mean_a,
        "mean_b": mean_b,
        "mean_delta": mean_delta,
        "median_delta": median_delta,
        "pct_change_vs_a": pct_change,
        "sign_test": sign,
        "wilcoxon": wilcoxon,
        "bootstrap_ci_mean_delta": ci,
    }


def _paired_sign_test(deltas: list[float]) -> dict[str, Any]:
    positives = sum(1 for value in deltas if value > 0)
    negatives = sum(1 for value in deltas if value < 0)
    nonzero = positives + negatives
    if nonzero == 0:
        return {
            "n_nonzero": 0,
            "n_positive": 0,
            "n_negative": 0,
            "p_value": 1.0,
        }
    p_value = _exact_binomial_two_sided(positives, nonzero)
    return {
        "n_nonzero": nonzero,
        "n_positive": positives,
        "n_negative": negatives,
        "p_value": p_value,
    }


def _paired_wilcoxon_test(deltas: list[float]) -> dict[str, Any]:
    nonzero_deltas = [value for value in deltas if value != 0]
    n_nonzero = len(nonzero_deltas)
    if n_nonzero == 0:
        return {
            "n_nonzero": 0,
            "statistic": 0.0,
            "p_value": 1.0,
            "method": "wilcoxon_signed_rank",
        }
    try:
        result = cast(
            Any,
            stats.wilcoxon(
                nonzero_deltas,
                zero_method="wilcox",
                alternative="two-sided",
                method="auto",
            ),
        )
        return {
            "n_nonzero": n_nonzero,
            "statistic": float(result.statistic),
            "p_value": float(result.pvalue),
            "method": "wilcoxon_signed_rank",
        }
    except ValueError:
        return {
            "n_nonzero": n_nonzero,
            "statistic": None,
            "p_value": None,
            "method": "wilcoxon_signed_rank",
        }


def _exact_binomial_two_sided(k: int, n: int) -> float:
    result = stats.binomtest(k, n, 0.5, alternative="two-sided")
    return float(result.pvalue)


def _bootstrap_mean_ci(
    *,
    values: list[float],
    n_bootstrap: int,
    confidence: float,
    seed: int | None,
) -> dict[str, Any]:
    if not values or n_bootstrap <= 0 or len(values) < 2:
        mean_value = _mean(values)
        return {
            "mean": mean_value,
            "ci_lower": mean_value,
            "ci_upper": mean_value,
            "confidence": confidence,
            "n_bootstrap": max(0, n_bootstrap),
        }

    bootstrap_kwargs: dict[str, Any] = {}
    if seed is not None:
        bootstrap_kwargs["random_state"] = seed

    result = cast(Any, stats.bootstrap)(
        (values,),
        statistic=lambda sample: float(sum(sample) / len(sample)),
        vectorized=False,
        n_resamples=n_bootstrap,
        confidence_level=confidence,
        method="basic",
        **bootstrap_kwargs,
    )
    mean_value = _mean(values)
    return {
        "mean": mean_value,
        "ci_lower": float(result.confidence_interval.low),
        "ci_upper": float(result.confidence_interval.high),
        "confidence": confidence,
        "n_bootstrap": n_bootstrap,
    }


def _compute_failure_diagnostics(
    *,
    raw_records: list[dict[str, Any]],
    condition_a: str,
    condition_b: str,
    top_n: int,
) -> dict[str, Any]:
    """Summarize attempt-level incidents and case-level overflow outcomes.

    Metric semantics:
    - ``http_error_attempts`` counts attempts where ``patch_meta.error == "http_error"``.
      This is an attempt-incident counter (retries included), not a terminal
      failure count.
    - ``context_overflow_attempts`` is the subset of attempts whose HTTP error
      body indicates context-size overflow.
    - ``patch_attempts`` counts attempts that proposed at least one file change.
    - ``test_touch_attempts`` counts patch attempts where proposed edits touched
      test-like paths (for example ``tests/``, ``test_*.py``, ``*_test.py``).
    - ``context_overflow_case_*`` fields are case-level aggregates over
      (run_id, case_id, condition), reporting overflow incidence and whether any
      attempt for the case eventually succeeded.
    """
    conditions = {condition_a, condition_b}
    flattened: list[dict[str, Any]] = []

    for record in raw_records:
        condition = str(record.get("condition") or "")
        if condition not in conditions:
            continue
        run_id = str(record.get("run_id") or "")
        case_id = str(record.get("case_id") or "")

        patch_meta = record.get("patch_meta")
        note = str(record.get("note") or "")
        error = ""
        status = ""
        body = ""
        patch_touches_tests = False
        patch_file_count = 0
        if isinstance(patch_meta, dict):
            error = str(patch_meta.get("error") or "")
            status_val = patch_meta.get("status")
            status = str(status_val) if status_val is not None else ""
            body = str(patch_meta.get("body") or "")
            patch_touches_tests = bool(patch_meta.get("patch_touches_tests"))
            patch_file_count_raw = patch_meta.get("patch_file_count")
            if isinstance(patch_file_count_raw, (int, float)):
                patch_file_count = max(0, int(patch_file_count_raw))
            elif isinstance(patch_meta.get("patch_file_paths"), list):
                patch_file_count = len(patch_meta.get("patch_file_paths") or [])

        overflow = "exceeds the available context size" in body
        success = bool(record.get("success"))
        reason = (
            ""
            if success
            else _failure_reason(error=error, status=status, note=note, overflow=overflow)
        )
        flattened.append(
            {
                "run_id": run_id,
                "case_id": case_id,
                "condition": condition,
                "success": success,
                "is_http_error": error == "http_error",
                "is_overflow": overflow,
                "patch_touches_tests": patch_touches_tests,
                "has_patch": patch_file_count > 0,
                "reason": reason,
            }
        )

    attempt_stats_by_condition: dict[str, dict[str, Any]] = {}
    case_stats_by_condition: dict[str, dict[str, Any]] = {}
    top_reasons_by_condition: dict[str, list[tuple[str, int]]] = {
        condition: [] for condition in conditions
    }

    if flattened:
        frame = pl.DataFrame(flattened)
        attempt_stats = frame.group_by("condition").agg(
            pl.len().alias("attempts"),
            pl.col("is_http_error").cast(pl.Int64).sum().alias("http_error_attempts"),
            pl.col("is_overflow").cast(pl.Int64).sum().alias("context_overflow_attempts"),
            pl.col("has_patch").cast(pl.Int64).sum().alias("patch_attempts"),
            pl.col("patch_touches_tests").cast(pl.Int64).sum().alias("test_touch_attempts"),
        )
        attempt_stats_by_condition = {row["condition"]: row for row in attempt_stats.to_dicts()}

        case_stats = (
            frame.group_by(["run_id", "case_id", "condition"])
            .agg(
                pl.col("is_overflow").any().alias("case_overflow"),
                pl.col("success").any().alias("case_success"),
            )
            .group_by("condition")
            .agg(
                pl.len().alias("cases"),
                pl.col("case_overflow").cast(pl.Int64).sum().alias("context_overflow_cases"),
                (pl.col("case_overflow") & pl.col("case_success"))
                .cast(pl.Int64)
                .sum()
                .alias("context_overflow_case_successes"),
                (pl.col("case_overflow") & ~pl.col("case_success"))
                .cast(pl.Int64)
                .sum()
                .alias("context_overflow_case_failures"),
            )
        )
        case_stats_by_condition = {row["condition"]: row for row in case_stats.to_dicts()}

        top_reason_rows = (
            frame.filter(~pl.col("success"))
            .group_by(["condition", "reason"])
            .agg(pl.len().alias("count"))
            .sort(["condition", "count", "reason"], descending=[False, True, False])
            .to_dicts()
        )
        for row in top_reason_rows:
            condition = str(row["condition"])
            if condition not in top_reasons_by_condition:
                continue
            reasons = top_reasons_by_condition[condition]
            if len(reasons) >= top_n:
                continue
            reasons.append((str(row["reason"]), int(row["count"])))

    per_condition: dict[str, dict[str, Any]] = {}
    for condition in sorted(conditions):
        attempt_stats = attempt_stats_by_condition.get(condition, {})
        case_stats = case_stats_by_condition.get(condition, {})

        attempts = int(attempt_stats.get("attempts", 0) or 0)
        http_error_attempts = int(attempt_stats.get("http_error_attempts", 0) or 0)
        overflow_attempts = int(attempt_stats.get("context_overflow_attempts", 0) or 0)
        patch_attempts = int(attempt_stats.get("patch_attempts", 0) or 0)
        test_touch_attempts = int(attempt_stats.get("test_touch_attempts", 0) or 0)
        cases = int(case_stats.get("cases", 0) or 0)
        overflow_cases = int(case_stats.get("context_overflow_cases", 0) or 0)
        overflow_case_successes = int(case_stats.get("context_overflow_case_successes", 0) or 0)
        overflow_case_failures = int(case_stats.get("context_overflow_case_failures", 0) or 0)

        per_condition[condition] = {
            "attempts": attempts,
            "http_error_attempts": http_error_attempts,
            "http_error_attempt_rate": _ratio(http_error_attempts, attempts),
            "context_overflow_attempts": overflow_attempts,
            "context_overflow_attempt_rate": _ratio(overflow_attempts, attempts),
            "patch_attempts": patch_attempts,
            "test_touch_attempts": test_touch_attempts,
            "test_touch_attempt_rate": _ratio(test_touch_attempts, patch_attempts),
            "cases": cases,
            "context_overflow_cases": overflow_cases,
            "context_overflow_case_rate": _ratio(overflow_cases, cases),
            "context_overflow_case_successes": overflow_case_successes,
            "context_overflow_case_failures": overflow_case_failures,
            "top_reasons": top_reasons_by_condition.get(condition, []),
        }
    return {"per_condition": per_condition}


def _failure_reason(*, error: str, status: str, note: str, overflow: bool) -> str:
    if overflow:
        return "http_error:context_exceeded"
    if error and status:
        return f"{error}:{status}"
    if error:
        return error
    if note:
        return f"note:{note}"
    return "unknown"


def _ratio(numer: int, denom: int) -> float | None:
    if denom <= 0:
        return None
    return float(numer) / float(denom)


def _build_run_context(
    *,
    rows: list[dict[str, Any]],
    raw_records: list[dict[str, Any]],
    eval_log_path: str,
) -> dict[str, Any]:
    run_ids = sorted({str(row.get("run_id") or "") for row in rows if row.get("run_id")})
    patchers = sorted(
        {
            str((record.get("run_metadata") or {}).get("patcher_name") or "")
            for record in raw_records
            if isinstance(record.get("run_metadata"), dict)
        }
    )
    model_names = sorted(
        {
            str((record.get("model_metadata") or {}).get("model") or "")
            for record in raw_records
            if isinstance(record.get("model_metadata"), dict)
        }
    )
    out: dict[str, Any] = {
        "run_ids": [value for value in run_ids if value],
        "n_case_condition_rows": len(rows),
        "patchers": [value for value in patchers if value],
        "models": [value for value in model_names if value],
    }
    if eval_log_path:
        log_path = Path(eval_log_path)
        if log_path.exists():
            out["eval_log"] = _parse_eval_log(log_path.read_text(encoding="utf-8"))
            out["eval_log_path"] = str(log_path)
    return out


def _parse_eval_log(text: str) -> dict[str, Any]:
    job_feedback = _parse_job_feedback_block(text)
    progress_elapsed = None
    match = re.search(
        r"Progress:\s*finished\s+executed=\d+,\s*skipped=\d+,\s*total=\d+,\s*elapsed=([^\n]+)",
        text,
    )
    if match:
        progress_elapsed = match.group(1).strip()
    if progress_elapsed:
        job_feedback["runner_elapsed"] = progress_elapsed
    return job_feedback


def _parse_job_feedback_block(text: str) -> dict[str, Any]:
    out: dict[str, Any] = {}
    marker = "============================= JOB FEEDBACK ============================="
    idx = text.rfind(marker)
    if idx < 0:
        return out
    block = text[idx:].splitlines()
    for line in block:
        if ":" not in line:
            continue
        key, value = line.split(":", 1)
        k = key.strip().lower()
        v = value.strip()
        if not k:
            continue
        out[k] = v

    energy_line = out.get("energy consumed")
    if isinstance(energy_line, str):
        match = re.search(r"([0-9.]+)\s+Joule\s*/\s*([0-9.]+)\s+Watthours", energy_line)
        if match:
            out["energy_joule"] = float(match.group(1))
            out["energy_watthours"] = float(match.group(2))
    avg_power = out.get("average node power draw")
    if isinstance(avg_power, str):
        match = re.search(r"([0-9.]+)\s+Watt", avg_power)
        if match:
            out["average_node_power_watt"] = float(match.group(1))
    return out


def _print_report(
    *,
    run_context: dict[str, Any],
    condition_a: str,
    condition_b: str,
    primary_slice: set[str],
    overall_effect: dict[str, Any],
    primary_effect: dict[str, Any],
    required_effect: dict[str, Any],
    control_effect: dict[str, Any],
    category_stats: dict[str, Any],
    efficiency: dict[str, dict[str, Any]],
    failures: dict[str, Any],
    input_errors: dict[str, int],
) -> None:
    print("Run Metadata")
    print(f"- Run IDs: {', '.join(run_context.get('run_ids', [])) or 'n/a'}")
    print(f"- Patcher(s): {', '.join(run_context.get('patchers', [])) or 'n/a'}")
    print(f"- Model(s): {', '.join(run_context.get('models', [])) or 'n/a'}")
    print(f"- Case-condition rows: {run_context.get('n_case_condition_rows', 0)}")
    _print_input_errors(input_errors)

    print("\nPrimary Endpoint")
    print(f"- Primary snapshot slice: {', '.join(sorted(primary_slice))}")
    _print_effect_line("Primary", condition_a, condition_b, primary_effect)

    print("\nSecondary Slices")
    _print_effect_line("All cases", condition_a, condition_b, overall_effect)
    _print_effect_line("Required only", condition_a, condition_b, required_effect)
    _print_effect_line(
        "Controls (snapshot_dependency=none)", condition_a, condition_b, control_effect
    )

    print("\nSecondary Efficiency (paired deltas, B - A)")
    for metric in ("token_usage_total", "runtime_sec", "llm_calls_total"):
        summary = efficiency.get(metric, {})
        _print_efficiency_line(summary)

    print("\nFailure Diagnostics")
    per_condition = failures.get("per_condition", {})
    for condition in sorted(per_condition):
        stats = per_condition[condition]
        print(
            f"- {condition}: context-overflow attempts="
            f"{stats.get('context_overflow_attempts', 0)}/{stats.get('attempts', 0)} "
            f"({_fmt_pct(stats.get('context_overflow_attempt_rate'))}), "
            f"context-overflow cases={stats.get('context_overflow_cases', 0)}/"
            f"{stats.get('cases', 0)} ({_fmt_pct(stats.get('context_overflow_case_rate'))}), "
            f"http_error attempts={stats.get('http_error_attempts', 0)}, "
            f"test-touch patch attempts={stats.get('test_touch_attempts', 0)}/"
            f"{stats.get('patch_attempts', 0)} "
            f"({_fmt_pct(stats.get('test_touch_attempt_rate'))})"
        )
        top = stats.get("top_reasons", [])
        if isinstance(top, list) and top:
            rendered = ", ".join(f"{name}:{count}" for name, count in top)
            print(f"  top failure reasons: {rendered}")

    print("\nCategory Highlights (primary slice)")
    if not category_stats.get("available"):
        print(f"- unavailable: {category_stats.get('error', 'unknown')}")
    else:
        rows = category_stats.get("rows", [])
        if not rows:
            print("- no matched category pairs")
        else:
            for row in rows[:12]:
                uplift = _fmt_signed_pp(row.get("uplift"))
                p_val = _fmt_float(row.get("mcnemar_p"))
                adj = _fmt_float(row.get("holm_adjusted_p"))
                sig = "yes" if row.get("holm_rejected") else "no"
                print(
                    f"- {row.get('category')}: uplift={uplift}, n={row.get('n_pairs', 0)}, "
                    f"McNemar p={p_val}, Holm adj p={adj}, Holm sig={sig}"
                )

    print("\nRun-Level System Context")
    eval_log = run_context.get("eval_log", {})
    if isinstance(eval_log, dict) and eval_log:
        _print_if_present(eval_log, "job wall-clock time", "Job wall-clock")
        _print_if_present(eval_log, "runner_elapsed", "Runner elapsed")
        _print_if_present(eval_log, "cpu utilized", "CPU utilized")
        _print_if_present(eval_log, "cpu efficiency", "CPU efficiency")
        _print_if_present(eval_log, "memory utilized", "Memory utilized")
        _print_if_present(eval_log, "memory efficiency", "Memory efficiency")
        if "energy_joule" in eval_log and "energy_watthours" in eval_log:
            print(
                "- Energy consumed (run-level): "
                f"{_fmt_float(eval_log.get('energy_joule'))} J / "
                f"{_fmt_float(eval_log.get('energy_watthours'))} Wh"
            )
        _print_if_present(eval_log, "average node power draw", "Average node power draw")
    else:
        print("- unavailable (no --eval-log or parseable job feedback block)")

    print("\nOne-Line Conclusion")
    print(_conclusion_line(condition_a=condition_a, condition_b=condition_b, effect=primary_effect))


def _print_if_present(data: dict[str, Any], key: str, label: str) -> None:
    value = data.get(key)
    if value not in (None, ""):
        print(f"- {label}: {value}")


def _conclusion_line(*, condition_a: str, condition_b: str, effect: dict[str, Any]) -> str:
    if not effect.get("available"):
        return (
            f"{condition_b} vs {condition_a}: primary slice unavailable "
            f"({effect.get('error', 'no matched pairs')})."
        )
    uplift_pp = effect.get("uplift_pp")
    if not isinstance(uplift_pp, (int, float)):
        return f"{condition_b} vs {condition_a}: primary slice did not yield numeric uplift."
    direction = "better" if uplift_pp > 0 else "worse" if uplift_pp < 0 else "equal"
    return (
        f"On snapshot-relevant cases, {condition_b} is {abs(uplift_pp):.2f} pp {direction} than "
        f"{condition_a} (McNemar p={_fmt_float((effect.get('mcnemar') or {}).get('p_value'))})."
    )


def _print_effect_line(
    label: str,
    condition_a: str,
    condition_b: str,
    effect: dict[str, Any],
) -> None:
    if not effect.get("available"):
        print(f"- {label}: unavailable ({effect.get('error', 'no matched pairs')})")
        return

    a_rate = effect.get("a_rate")
    b_rate = effect.get("b_rate")
    uplift_pp = effect.get("uplift_pp")
    direction = "better" if isinstance(uplift_pp, (int, float)) and uplift_pp > 0 else "worse"
    if isinstance(uplift_pp, (int, float)) and abs(uplift_pp) < 1e-12:
        direction = "equal"
    if not isinstance(uplift_pp, (int, float)):
        print(f"- {label}: unavailable (non-numeric uplift)")
        return
    mcn = effect.get("mcnemar", {})
    bci = effect.get("bootstrap_ci", {})
    ch = effect.get("cohens_h", {})
    agreement = effect.get("agreement", {})
    print(
        f"- {label}: {condition_b} is {abs(float(uplift_pp)):.2f} pp {direction} than {condition_a} "
        f"({_fmt_pct(a_rate)} -> {_fmt_pct(b_rate)}), n={effect.get('n_pairs', 0)}, "
        f"McNemar p={_fmt_float(mcn.get('p_value'))} "
        f"(n_01={mcn.get('n_01', 0)}, n_10={mcn.get('n_10', 0)}), "
        f"95% CI[{_fmt_pct_signed(bci.get('ci_lower'))}, {_fmt_pct_signed(bci.get('ci_upper'))}], "
        f"Cohen h={_fmt_float(ch.get('h'))}, "
        f"Cohen kappa={_fmt_float((agreement or {}).get('kappa'))}"
    )


def _print_efficiency_line(summary: dict[str, Any]) -> None:
    if not summary.get("available"):
        print(
            f"- {summary.get('label', 'metric')}: unavailable ({summary.get('error', 'unknown')})"
        )
        return
    sign = summary.get("sign_test", {})
    wilcoxon = summary.get("wilcoxon", {})
    ci = summary.get("bootstrap_ci_mean_delta", {})
    print(
        f"- {summary.get('label')}: mean delta={_fmt_float(summary.get('mean_delta'))}, "
        f"median delta={_fmt_float(summary.get('median_delta'))}, "
        f"mean A={_fmt_float(summary.get('mean_a'))}, "
        f"mean B={_fmt_float(summary.get('mean_b'))}, "
        f"pct vs A={_fmt_pct_signed(summary.get('pct_change_vs_a'))}, "
        f"sign-test p={_fmt_float(sign.get('p_value'))} "
        f"(+={sign.get('n_positive', 0)}, -={sign.get('n_negative', 0)}), "
        f"Wilcoxon p={_fmt_float(wilcoxon.get('p_value'))}, "
        f"bootstrap 95% CI mean delta="
        f"[{_fmt_float(ci.get('ci_lower'))}, {_fmt_float(ci.get('ci_upper'))}]"
    )


def _print_input_errors(input_errors: dict[str, int]) -> None:
    files_unreadable = int(input_errors.get("files_unreadable", 0))
    json_lines_invalid = int(input_errors.get("json_lines_invalid", 0))
    records_skipped = int(input_errors.get("records_skipped", 0))
    if files_unreadable == 0 and json_lines_invalid == 0 and records_skipped == 0:
        return
    print(
        "- Input diagnostics: "
        f"files_unreadable={files_unreadable}, "
        f"json_lines_invalid={json_lines_invalid}, "
        f"records_skipped={records_skipped}"
    )


def _as_float(value: Any) -> float | None:
    if isinstance(value, (int, float)):
        return float(value)
    return None


def _mean(values: list[float]) -> float | None:
    if not values:
        return None
    return sum(values) / len(values)


def _median(values: list[float]) -> float | None:
    if not values:
        return None
    ordered = sorted(values)
    n = len(ordered)
    mid = n // 2
    if n % 2 == 1:
        return ordered[mid]
    return (ordered[mid - 1] + ordered[mid]) / 2.0


def _fmt_float(value: Any) -> str:
    if not isinstance(value, (int, float)):
        return "n/a"
    return f"{float(value):.4f}"


def _fmt_pct(value: Any) -> str:
    if not isinstance(value, (int, float)):
        return "n/a"
    return f"{100.0 * float(value):.2f}%"


def _fmt_pct_signed(value: Any) -> str:
    if not isinstance(value, (int, float)):
        return "n/a"
    return f"{100.0 * float(value):+.2f}%"


def _fmt_signed_pp(value: Any) -> str:
    if not isinstance(value, (int, float)):
        return "n/a"
    return f"{100.0 * float(value):+.2f} pp"


if __name__ == "__main__":
    raise SystemExit(main())
