"""Eval suite runner — executes eval cases and produces results."""
from __future__ import annotations

import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from abi_evals.comparators import COMPARATORS
from abi_evals.models import EvalCase, EvalResult, EvalSummary


def load_eval_suite(path: Path) -> dict[str, Any]:
    """Load an eval suite from JSON file."""
    with open(path) as f:
        suite = json.load(f)
    if "suite_id" not in suite:
        raise ValueError(f"Eval suite missing 'suite_id': {path}")
    if "cases" not in suite or not suite["cases"]:
        raise ValueError(f"Eval suite missing 'cases': {path}")
    return suite


def parse_cases(suite: dict[str, Any]) -> list[EvalCase]:
    """Parse eval suite cases into EvalCase objects."""
    cases = []
    for raw in suite.get("cases", []):
        cases.append(EvalCase(
            case_id=raw["case_id"],
            description=raw.get("description", ""),
            input=raw.get("input"),
            expected=raw.get("expected"),
            comparison=raw.get("comparison", "exact"),
            tolerance=raw.get("tolerance", 1e-6),
            tags=raw.get("tags", []),
            config=raw.get("config", {}),
        ))
    return cases


def run_case(case: EvalCase, evidence_pack: dict[str, Any] | None = None) -> EvalResult:
    """Run a single eval case."""
    comparator = COMPARATORS.get(case.comparison)
    if comparator is None:
        return EvalResult(
            case_id=case.case_id,
            status="error",
            message=f"Unknown comparison mode: {case.comparison}",
        )

    try:
        # Resolve actual value from evidence pack or input
        actual = _resolve_actual(case, evidence_pack)

        if case.comparison == "numeric_tolerance":
            passed, msg = comparator(actual, case.expected, case.tolerance)
        else:
            passed, msg = comparator(actual, case.expected)

        return EvalResult(
            case_id=case.case_id,
            status="pass" if passed else "fail",
            actual=actual,
            message=msg,
            diff="" if passed else msg,
        )
    except Exception as e:
        return EvalResult(
            case_id=case.case_id,
            status="error",
            message=f"Error running case: {e}",
        )


def _resolve_actual(case: EvalCase, evidence_pack: dict[str, Any] | None) -> Any:
    """Resolve actual value for comparison.

    Priority:
    1. case.input (if it's the actual value to compare)
    2. evidence_pack lookup via case.config["evidence_key"]
    3. case.input as-is
    """
    if evidence_pack and "evidence_key" in case.config:
        key = case.config["evidence_key"]
        keys = key.split(".")
        val = evidence_pack
        for k in keys:
            if isinstance(val, dict):
                val = val.get(k)
            else:
                break
        return val
    return case.input


def run_suite(
    suite: dict[str, Any],
    evidence_pack: dict[str, Any] | None = None,
    run_id: str = "00000000",
    timestamp: str | None = None,
) -> dict[str, Any]:
    """Run a full eval suite and return eval_result compatible output."""
    cases = parse_cases(suite)
    results: list[EvalResult] = []
    summary = EvalSummary()

    # Sort cases by case_id for deterministic ordering
    cases_sorted = sorted(cases, key=lambda c: c.case_id)

    for case in cases_sorted:
        result = run_case(case, evidence_pack)
        results.append(result)
        summary.total += 1
        if result.status == "pass":
            summary.passed += 1
        elif result.status == "fail":
            summary.failed += 1
        elif result.status == "error":
            summary.errors += 1
        elif result.status == "skipped":
            summary.skipped += 1

    ts = timestamp or datetime.now(UTC).isoformat()

    result_dicts = []
    for r in results:
        tokens_used = max(1, len(json.dumps(r.actual)) // 4) if r.actual is not None else 1
        result_dicts.append({
            "case_id": r.case_id,
            "status": r.status,
            "actual": r.actual,
            "message": r.message,
            "diff": r.diff,
            "tokens_used": tokens_used,
        })

    total_tokens = sum(rd["tokens_used"] for rd in result_dicts)

    return {
        "schema_version": "eval-result/1.0",
        "suite_id": suite["suite_id"],
        "run_id": run_id,
        "evaluated_at": ts,
        "results": result_dicts,
        "summary": {
            "total": summary.total,
            "passed": summary.passed,
            "failed": summary.failed,
            "errors": summary.errors,
            "skipped": summary.skipped,
        },
        "cost_summary": {
            "total_tokens": total_tokens,
            "case_token_map": {
                rd["case_id"]: rd["tokens_used"] for rd in result_dicts
            },
        },
    }


def write_results(results: dict[str, Any], path: Path) -> None:
    """Write eval results to JSON file."""
    path.parent.mkdir(parents=True, exist_ok=True)
    with open(path, "w") as f:
        json.dump(results, f, indent=2, ensure_ascii=False, sort_keys=True)
        f.write("\n")
