from __future__ import annotations

import argparse
import dataclasses
import hashlib
import html
import json
import os
import platform
import re
import string
import sys
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Tuple


def _utc_stamp() -> str:
    return datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")


def _sha256_bytes(b: bytes) -> str:
    return hashlib.sha256(b).hexdigest()


def _sha256_file(path: Path) -> str:
    return _sha256_bytes(path.read_bytes())


def _git_info() -> Dict[str, Any]:
    """Best-effort git metadata for reproducibility."""
    try:
        import subprocess

        commit = (
            subprocess.check_output(["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL)
            .decode("utf-8", errors="replace")
            .strip()
        )
        dirty = bool(
            subprocess.check_output(["git", "status", "--porcelain"], stderr=subprocess.DEVNULL)
            .decode("utf-8", errors="replace")
            .strip()
        )
        return {"commit": commit, "dirty": dirty}
    except Exception:
        return {"commit": None, "dirty": None}


def _pkg_version(name: str) -> Optional[str]:
    try:
        import importlib.metadata as md

        return md.version(name)
    except Exception:
        return None


def _json_object_from_text(text: str) -> Optional[dict]:
    s = text.strip()
    if not s:
        return None

    # If the model returned clean JSON.
    if s.startswith("{") and s.endswith("}"):
        try:
            v = json.loads(s)
            return v if isinstance(v, dict) else None
        except Exception:
            pass

    # Otherwise try to extract the first JSON object.
    m = re.search(r"\{.*\}", s, flags=re.DOTALL)
    if not m:
        return None
    try:
        v = json.loads(m.group(0))
        return v if isinstance(v, dict) else None
    except Exception:
        return None


def _extract_text(resp: Any) -> str:
    # OpenAI-style: resp.choices[0].message.content
    if hasattr(resp, "choices") and resp.choices:
        c0 = resp.choices[0]
        if hasattr(c0, "message") and hasattr(c0.message, "content"):
            return c0.message.content or ""
        if hasattr(c0, "text"):
            return c0.text or ""
    if isinstance(resp, dict):
        try:
            return resp["choices"][0]["message"]["content"]
        except Exception:
            return str(resp)
    return str(resp)


def _extract_usage(resp: Any) -> Dict[str, int]:
    usage = getattr(resp, "usage", None)
    if usage is None and isinstance(resp, dict):
        usage = resp.get("usage")
    if usage is None:
        return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}

    def _get(k: str) -> int:
        if isinstance(usage, dict):
            return int(usage.get(k, 0) or 0)
        return int(getattr(usage, k, 0) or 0)

    pt = _get("prompt_tokens")
    ct = _get("completion_tokens")
    tt = _get("total_tokens") or (pt + ct)
    return {"prompt_tokens": pt, "completion_tokens": ct, "total_tokens": tt}


def _detect_refusal(text: str) -> bool:
    t = (text or "").strip().lower()
    if not t:
        return False
    flags = [
        "i can't help",
        "i cannot help",
        "i can't comply",
        "i cannot comply",
        "i can't assist",
        "i cannot assist",
        "i'm sorry, but i can't",
        "i'm sorry, but i cannot",
        "cannot provide",
        "policy",
        "safety",
        "refuse",
    ]
    return any(f in t for f in flags)


def _text_with_citations_per_claim(answer: str, evidence_ids: List[str]) -> str:
    """
    Convert "answer + evidence_ids[]" into text that can be evaluated claim-by-claim.

    Groundedness verifier detects citations inside each claim sentence (e.g. "Paris [E1]").
    Many models output evidence ids separately (JSON field), so we deterministically
    attach citations to each extracted sentence to make the metric meaningful.
    """
    s = (answer or "").strip()
    if not s:
        return s
    cites = " ".join(f"[{x}]" for x in (evidence_ids or []) if isinstance(x, str) and x.strip())
    if not cites:
        return s
    # Split on newlines to preserve existing structure.
    lines = [ln.strip() for ln in s.splitlines() if ln.strip()]
    out = []
    for ln in lines:
        # Avoid double-injecting citations.
        if re.search(r"(?:\[|\()\s*E\d{1,4}\s*(?:\]|\))", ln):
            out.append(ln)
        else:
            out.append(f"{ln} {cites}".strip())
    return "\n".join(out)


def _norm_answer_text(s: str) -> str:
    lowered = (s or "").lower()
    no_punct = "".join(" " if ch in string.punctuation else ch for ch in lowered)
    no_articles = re.sub(r"\b(a|an|the)\b", " ", no_punct)
    return re.sub(r"\s+", " ", no_articles).strip()


def _claim_groundedness_report(
    *, answer: str, evidence_ids: List[str], evidence: List[Evidence]
) -> Optional[Dict[str, Any]]:
    try:
        from entropic_core.experimental.groundedness import analyze_groundedness
    except Exception:
        return None

    ev = [{"evidence_id": e.id, "text": e.text} for e in evidence]
    text = _text_with_citations_per_claim((answer or ""), evidence_ids or [])
    rep = analyze_groundedness(text, ev)
    s = rep.summary
    return {
        "summary": {
            "total_claims": int(s.total_claims),
            "supported_claims": int(s.supported_claims),
            "unsupported_claims": int(s.unsupported_claims),
            "citation_invented": int(s.citation_invented),
            "citation_misattributed": int(s.citation_misattributed),
            "numeric_mismatch": int(s.numeric_mismatch),
            "contradiction": int(s.contradiction),
            "unverifiable": bool(s.unverifiable),
            "supported_rate": (float(s.supported_claims) / float(max(1, s.total_claims))),
        },
        "claims": [
            {
                "claim_text": str(a.claim.text),
                "claim_type": str(a.claim.claim_type),
                "cited_evidence_ids": list(a.claim.cited_evidence_ids),
                "supported": bool(a.supported),
                "support_score": float(a.support_score),
                "predicted_evidence_ids": list(a.evidence_ids),
                "failure_types": [str(x.value) for x in a.failure_types],
                "explain": str(a.explain),
            }
            for a in rep.assessments
        ],
    }


def _estimate_cost_usd(usage: Dict[str, int], in_per_1k: float, out_per_1k: float) -> float:
    pt = float(usage.get("prompt_tokens", 0))
    ct = float(usage.get("completion_tokens", 0))
    return (pt / 1000.0) * in_per_1k + (ct / 1000.0) * out_per_1k


@dataclass(frozen=True)
class Evidence:
    id: str
    text: str


@dataclass(frozen=True)
class EvalCase:
    case_id: str
    category: str
    context: List[Evidence]
    question: str
    expected: Dict[str, Any]
    checks: List[Dict[str, Any]]


@dataclass(frozen=True)
class SuiteMeta:
    suite: str
    version: str
    created: str
    license: str
    description: str


@dataclass
class CaseResult:
    case_id: str
    ok: bool
    reasons: List[str]
    response_text: str
    response_json: Optional[dict]
    usage: Dict[str, int]
    latency_ms: float
    grounded: bool
    contradiction_handled: bool
    mode: str  # baseline|entropic
    entropy_before: Optional[float] = None
    entropy_after: Optional[float] = None
    interventions_applied: Optional[List[str]] = None
    cost_usd_est: Optional[float] = None
    tool_calls: Optional[List[Dict[str, Any]]] = None
    steps: int = 1
    retry_count: int = 0
    is_refusal: bool = False
    tool_calls_count: int = 0
    groundedness_summary: Optional[Dict[str, Any]] = None
    groundedness_claims: Optional[List[Dict[str, Any]]] = None


def load_suite(path: Path) -> Tuple[SuiteMeta, List[EvalCase]]:
    meta: Optional[SuiteMeta] = None
    cases: List[EvalCase] = []

    for line in path.read_text(encoding="utf-8").splitlines():
        if not line.strip():
            continue
        obj = json.loads(line)
        if obj.get("type") == "meta":
            meta = SuiteMeta(
                suite=obj["suite"],
                version=obj["version"],
                created=obj.get("created", ""),
                license=obj.get("license", ""),
                description=obj.get("description", ""),
            )
            continue
        if obj.get("type") != "case":
            continue
        ctx = [Evidence(id=e["id"], text=e["text"]) for e in obj.get("context", [])]
        cases.append(
            EvalCase(
                case_id=obj["id"],
                category=obj.get("category", "unknown"),
                context=ctx,
                question=obj["question"],
                expected=obj.get("expected", {}),
                checks=obj.get("checks", []),
            )
        )

    if meta is None:
        raise ValueError(f"Suite missing meta record: {path}")
    return meta, cases


_SYSTEM_TEMPLATE = """You are a careful assistant. Use ONLY the provided EVIDENCE to answer.
If the evidence is insufficient, set answer to exactly: "I don't know".

Return ONLY a JSON object with keys:
- answer: string
- evidence_ids: array of strings (IDs you used, must exist in EVIDENCE)
- confidence: number in [0,1]
- notes: string (brief; optional)

EVIDENCE:
{evidence_block}
"""


_SYSTEM_TEMPLATE_TOOL_LOOP = """You are a careful assistant. You may call TOOLS to gather evidence.
Never guess. If you need a tool, call it.

Tool call JSON (return ONLY this JSON when calling a tool):
- action: "tool"
- tool_name: string (one of: calc, read_file)
- tool_input: string or object
- notes: string (optional)

Final answer JSON (return ONLY this JSON when answering):
- answer: string
- evidence_ids: array of strings (optional; for this suite can be empty)
- confidence: number in [0,1]
- notes: string (optional)

TOOLS:
- calc: arithmetic evaluation. Input: {{"expr": "<expression>"}} or "<expression>".
- read_file: read a sandbox file. Input: {{"path": "<filename>"}} or "<filename>".

EVIDENCE (may be empty initially; tool outputs will arrive as TOOL messages):
{evidence_block}
"""


def _build_messages(case: EvalCase, *, allow_tools: bool = False) -> List[Dict[str, str]]:
    evidence_lines = "\n".join([f"- {e.id}: {e.text}" for e in case.context])
    # Include a stable case id line. Real providers can ignore it; mock provider uses it.
    tmpl = _SYSTEM_TEMPLATE_TOOL_LOOP if allow_tools else _SYSTEM_TEMPLATE
    system = f"EVAL_CASE_ID: {case.case_id}\n" + tmpl.format(evidence_block=evidence_lines)
    return [{"role": "system", "content": system}, {"role": "user", "content": case.question}]


def _check_json_schema(obj: Optional[dict]) -> Tuple[bool, str]:
    if obj is None:
        return False, "Response is not valid JSON."
    for k in ("answer", "evidence_ids", "confidence", "notes"):
        if k not in obj:
            return False, f"Missing key: {k}"
    if not isinstance(obj["answer"], str):
        return False, "answer must be a string."
    if not isinstance(obj["evidence_ids"], list) or not all(
        isinstance(x, str) for x in obj["evidence_ids"]
    ):
        return False, "evidence_ids must be a list of strings."
    try:
        c = float(obj["confidence"])
    except Exception:
        return False, "confidence must be a number."
    if not (0.0 <= c <= 1.0):
        return False, "confidence out of range [0,1]."
    if not isinstance(obj["notes"], str):
        return False, "notes must be a string."
    return True, "ok"


def _check_confidence_range(obj: Optional[dict]) -> Tuple[bool, str]:
    if obj is None:
        return False, "No JSON."
    try:
        c = float(obj.get("confidence"))
    except Exception:
        return False, "confidence must be numeric."
    if 0.0 <= c <= 1.0:
        return True, "ok"
    return False, "confidence out of range."


def _check_evidence_ids_valid(case: EvalCase, obj: Optional[dict]) -> Tuple[bool, str]:
    if obj is None:
        return False, "No JSON."
    allowed = {e.id for e in case.context}
    ids = obj.get("evidence_ids", [])
    bad = [x for x in ids if x not in allowed]
    if bad:
        return False, f"Invalid evidence_ids: {bad[:5]}"
    return True, "ok"


def _check_requires_evidence(obj: Optional[dict], min_ids: int) -> Tuple[bool, str]:
    if obj is None:
        return False, "No JSON."
    ids = obj.get("evidence_ids", [])
    if len(ids) >= min_ids:
        return True, "ok"
    return False, f"Requires at least {min_ids} evidence id(s)."


def _check_no_fake_evidence(obj: Optional[dict]) -> Tuple[bool, str]:
    if obj is None:
        return False, "No JSON."
    ids = obj.get("evidence_ids", [])
    if len(ids) == 0:
        return True, "ok"
    return False, "Should not cite evidence when answering I don't know."


def _get_field(obj: Optional[dict], field: str) -> Optional[str]:
    if obj is None:
        return None
    v = obj.get(field)
    return v if isinstance(v, str) else None


def _check_answer_exact(obj: Optional[dict], field: str, value: str) -> Tuple[bool, str]:
    v = _get_field(obj, field)
    if v is None:
        return False, f"Missing field {field}."
    if v.strip() == value:
        return True, "ok"
    return False, f"{field} != expected exact value."


def _check_answer_any_of_normalized(
    obj: Optional[dict], field: str, values: List[str]
) -> Tuple[bool, str]:
    v = _get_field(obj, field)
    if v is None:
        return False, f"Missing field {field}."
    observed = _norm_answer_text(v)
    if not observed:
        return False, f"{field} is empty."
    expected = {_norm_answer_text(str(x)) for x in (values or []) if str(x).strip()}
    if not expected:
        return False, "No expected values were configured."
    if observed in expected:
        return True, "ok"
    return False, f"{field} not in normalized expected set."


def _check_answer_regex(obj: Optional[dict], field: str, pattern: str) -> Tuple[bool, str]:
    v = _get_field(obj, field)
    if v is None:
        return False, f"Missing field {field}."
    if re.search(pattern, v, flags=re.DOTALL):
        return True, "ok"
    return False, f"{field} does not match regex."


def _check_must_not_contain_regex(
    obj: Optional[dict], field: str, pattern: str
) -> Tuple[bool, str]:
    v = _get_field(obj, field)
    if v is None:
        return False, f"Missing field {field}."
    if re.search(pattern, v, flags=re.DOTALL):
        return False, f"{field} contains forbidden pattern."
    return True, "ok"


def _check_tool_calls_min(tool_calls: Optional[List[Dict[str, Any]]], n: int) -> Tuple[bool, str]:
    calls = tool_calls or []
    if len(calls) >= int(n):
        return True, "ok"
    return False, f"Expected at least {int(n)} tool call(s)."


def _check_tool_called(tool_calls: Optional[List[Dict[str, Any]]], name: str) -> Tuple[bool, str]:
    calls = tool_calls or []
    want = (name or "").strip().lower()
    if not want:
        return False, "Missing tool name."
    for c in calls:
        if str(c.get("tool_name", "")).strip().lower() == want:
            return True, "ok"
    return False, f"Expected tool '{want}' to be called."


def _check_claim_groundedness(
    claim_report: Optional[Dict[str, Any]],
    *,
    min_supported_rate: float,
    max_unsupported_claims: int,
    max_citation_invented: int,
    max_contradiction: int,
    allow_unverifiable: bool,
) -> Tuple[bool, str]:
    if not isinstance(claim_report, dict):
        return False, "Claim groundedness report is unavailable."
    summary = claim_report.get("summary")
    if not isinstance(summary, dict):
        return False, "Claim groundedness summary is unavailable."

    unsupported = int(summary.get("unsupported_claims", 0) or 0)
    invented = int(summary.get("citation_invented", 0) or 0)
    contradiction = int(summary.get("contradiction", 0) or 0)
    supported_rate = float(summary.get("supported_rate", 0.0) or 0.0)
    unverifiable = bool(summary.get("unverifiable", False))

    if unverifiable and not allow_unverifiable:
        return False, "Claim groundedness is unverifiable for this answer."
    if supported_rate < float(min_supported_rate):
        return False, (
            f"supported_rate={supported_rate:.3f} < min_supported_rate={float(min_supported_rate):.3f}"
        )
    if unsupported > int(max_unsupported_claims):
        return False, (
            f"unsupported_claims={unsupported} > max_unsupported_claims={int(max_unsupported_claims)}"
        )
    if invented > int(max_citation_invented):
        return False, (
            f"citation_invented={invented} > max_citation_invented={int(max_citation_invented)}"
        )
    if contradiction > int(max_contradiction):
        return False, f"contradiction={contradiction} > max_contradiction={int(max_contradiction)}"
    return True, "ok"


def _claim_report_for_case_obj(case: EvalCase, obj: Optional[dict]) -> Optional[Dict[str, Any]]:
    if not isinstance(obj, dict):
        return None
    ans = str(obj.get("answer") or "")
    ev_ids = obj.get("evidence_ids", [])
    if not isinstance(ev_ids, list):
        ev_ids = []
    ev_ids2 = [str(x) for x in ev_ids if isinstance(x, str)]
    return _claim_groundedness_report(answer=ans, evidence_ids=ev_ids2, evidence=case.context)


def grade_case(
    case: EvalCase,
    response_text: str,
    *,
    tool_calls: Optional[List[Dict[str, Any]]] = None,
    claim_groundedness: Optional[Dict[str, Any]] = None,
) -> Tuple[bool, List[str], Optional[dict], bool, bool]:
    obj = _json_object_from_text(response_text)
    reasons: List[str] = []

    grounded = True
    contradiction_handled = True

    for chk in case.checks:
        t = chk["type"]
        if t == "json_schema":
            ok, r = _check_json_schema(obj)
        elif t == "confidence_range":
            ok, r = _check_confidence_range(obj)
        elif t == "evidence_ids_valid":
            ok, r = _check_evidence_ids_valid(case, obj)
        elif t == "requires_evidence":
            ok, r = _check_requires_evidence(obj, int(chk.get("min_ids", 1)))
        elif t == "no_fake_evidence":
            ok, r = _check_no_fake_evidence(obj)
        elif t == "answer_exact":
            ok, r = _check_answer_exact(obj, chk.get("field", "answer"), chk["value"])
        elif t == "answer_any_of_normalized":
            vals = chk.get("values", [])
            if isinstance(vals, list):
                values = [str(x) for x in vals]
            else:
                values = []
            ok, r = _check_answer_any_of_normalized(obj, chk.get("field", "answer"), values)
        elif t == "answer_regex":
            ok, r = _check_answer_regex(obj, chk.get("field", "answer"), chk["pattern"])
        elif t == "must_not_contain_regex":
            ok, r = _check_must_not_contain_regex(obj, chk.get("field", "answer"), chk["pattern"])
        elif t == "tool_calls_min":
            ok, r = _check_tool_calls_min(tool_calls, int(chk.get("min", 1)))
        elif t == "tool_called":
            ok, r = _check_tool_called(tool_calls, str(chk.get("name", "")))
        elif t == "claim_groundedness":
            ok, r = _check_claim_groundedness(
                claim_groundedness,
                min_supported_rate=float(chk.get("min_supported_rate", 1.0)),
                max_unsupported_claims=int(chk.get("max_unsupported_claims", 0)),
                max_citation_invented=int(chk.get("max_citation_invented", 0)),
                max_contradiction=int(chk.get("max_contradiction", 0)),
                allow_unverifiable=bool(chk.get("allow_unverifiable", False)),
            )
        else:
            ok, r = False, f"Unknown check: {t}"

        if not ok:
            reasons.append(r)

        # Derived metrics
        if (
            t
            in {"evidence_ids_valid", "requires_evidence", "no_fake_evidence", "claim_groundedness"}
            and not ok
        ):
            grounded = False
        if (
            case.category == "contradiction"
            and t in {"answer_regex", "must_not_contain_regex"}
            and not ok
        ):
            contradiction_handled = False
        if (
            case.category == "tool_output_contradiction"
            and t in {"answer_regex", "must_not_contain_regex"}
            and not ok
        ):
            contradiction_handled = False

    return (len(reasons) == 0), reasons, obj, grounded, contradiction_handled


def _openrouter_client():
    try:
        from openai import OpenAI  # type: ignore
    except Exception as e:
        raise SystemExit(
            "Missing dependency: openai. Install with: pip install \"entropic-core[openrouter,evals]\""
        ) from e

    api_key = os.getenv("OPENROUTER_API_KEY")
    if not api_key:
        raise SystemExit("Missing env var OPENROUTER_API_KEY")

    base_url = os.getenv("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1")

    default_headers: Dict[str, str] = {}
    http_referer = os.getenv("OPENROUTER_HTTP_REFERER")
    app_title = os.getenv("OPENROUTER_APP_TITLE")
    if http_referer:
        default_headers["HTTP-Referer"] = http_referer
    if app_title:
        default_headers["X-Title"] = app_title

    return OpenAI(api_key=api_key, base_url=base_url, default_headers=default_headers)


class _MockMessage:
    def __init__(self, content: str):
        self.content = content


class _MockChoice:
    def __init__(self, content: str):
        self.message = _MockMessage(content)
        self.finish_reason = "stop"


class _MockUsage:
    def __init__(self, prompt_tokens: int, completion_tokens: int):
        self.prompt_tokens = prompt_tokens
        self.completion_tokens = completion_tokens
        self.total_tokens = prompt_tokens + completion_tokens


class _MockResponse:
    def __init__(
        self, model: str, content: str, prompt_tokens: int = 120, completion_tokens: int = 60
    ):
        self.model = model
        self.choices = [_MockChoice(content)]
        self.usage = _MockUsage(prompt_tokens, completion_tokens)


class _MockChatCompletions:
    def __init__(self, *, cases_by_id: Dict[str, EvalCase]):
        self._cases_by_id = cases_by_id

    def create(
        self, *, model: str, messages: List[Dict[str, str]], temperature: float = 0.2, **kwargs: Any
    ) -> _MockResponse:
        sys_text = ""
        for m in messages:
            if m.get("role") == "system":
                sys_text += str(m.get("content") or "") + "\n"

        # Detect Entropic injections (enabled in entropic mode).
        has_grounding = (
            "[ENTROPIC_CORE:GROUNDING" in sys_text or "[FACTUAL GROUNDING REQUIRED]" in sys_text
        )
        has_stability = (
            "[ENTROPIC_CORE:STABILITY" in sys_text
            or "[SYSTEM STABILITY PROTOCOL ACTIVE]" in sys_text
        )

        m = re.search(r"^EVAL_CASE_ID:\s*(\S+)\s*$", sys_text, flags=re.MULTILINE)
        case_id = m.group(1) if m else ""
        case = self._cases_by_id.get(case_id)

        if case is None:
            obj = {
                "answer": "I don't know",
                "evidence_ids": [],
                "confidence": 0.1,
                "notes": "mock: missing case",
            }
            return _MockResponse(model, json.dumps(obj))

        # Tool loop mode: if Entropic injection is present, follow a simple tool protocol.
        if case.category == "tool_loop":
            tool_msgs = [m for m in messages if m.get("role") == "tool"]
            if not tool_msgs:
                if has_grounding:
                    if "release_date.txt" in (case.question or ""):
                        tool_call = {
                            "action": "tool",
                            "tool_name": "read_file",
                            "tool_input": {"path": "release_date.txt"},
                        }
                    else:
                        tool_call = {
                            "action": "tool",
                            "tool_name": "calc",
                            "tool_input": {"expr": "(17*5)-1"},
                        }
                    return _MockResponse(model, json.dumps(tool_call))

                # Baseline: refuse to guess without evidence.
                obj = {
                    "answer": "I don't know",
                    "evidence_ids": [],
                    "confidence": 0.2,
                    "notes": "mock",
                }
                return _MockResponse(model, json.dumps(obj))

            # After tool output exists, return a final answer using it.
            last_tool = str(tool_msgs[-1].get("content") or "").strip()
            lines = [ln.strip() for ln in last_tool.splitlines() if ln.strip()]
            out_line = lines[-1] if lines else ""
            final_obj = {"answer": out_line, "evidence_ids": [], "confidence": 0.9, "notes": "mock"}
            return _MockResponse(model, json.dumps(final_obj))

        def _requires_evidence(c: EvalCase) -> bool:
            return any(chk.get("type") == "requires_evidence" for chk in c.checks)

        def _forbidden_pattern_exists(c: EvalCase) -> bool:
            return any(chk.get("type") == "must_not_contain_regex" for chk in c.checks)

        expected_exact = case.expected.get("answer_exact")
        expected_regex = case.expected.get("answer_regex")
        if isinstance(expected_exact, str):
            target_answer = expected_exact
        elif isinstance(expected_regex, str):
            if "contradic" in expected_regex or "inconsisten" in expected_regex:
                target_answer = "Cannot determine due to contradiction in the evidence."
            else:
                target_answer = "wraps LLM calls; observe -> diagnose -> intervene -> evidence"
        else:
            target_answer = case.context[0].text if case.context else "I don't know"

        # Baseline behavior: omit evidence ids even when required.
        if not has_grounding and _requires_evidence(case):
            ev_ids: List[str] = []
        else:
            ev_ids = [case.context[0].id] if case.context and _requires_evidence(case) else []

        # Baseline behavior on contradiction cases: leak a forbidden claim.
        if case.category in {
            "contradiction",
            "tool_output_contradiction",
        } and _forbidden_pattern_exists(case):
            if not has_stability:
                target_answer = "There is a contradiction. Alpha = 3."

        # "I don't know" cases should not cite evidence.
        if target_answer == "I don't know":
            ev_ids = []

        obj = {
            "answer": target_answer,
            "evidence_ids": ev_ids,
            "confidence": 0.9 if has_grounding else 0.6,
            "notes": "mock",
        }
        return _MockResponse(model, json.dumps(obj))


class _MockOpenAIClient:
    def __init__(self, *, cases_by_id: Dict[str, EvalCase]):
        self.chat = type("chat", (), {})()
        self.chat.completions = _MockChatCompletions(cases_by_id=cases_by_id)


def _run_mode(
    mode: str,
    model: str,
    cases: List[EvalCase],
    *,
    provider: str,
    entropic_enabled: bool,
    in_per_1k: Optional[float],
    out_per_1k: Optional[float],
    runtime_events_path: Optional[Path] = None,
    run_id: str = "",
) -> List[CaseResult]:
    prov = (provider or "").lower().strip()
    if prov == "mock":
        client = _MockOpenAIClient(cases_by_id={c.case_id: c for c in cases})
    else:
        client = _openrouter_client()
    create_fn = client.chat.completions.create

    middleware = None
    outcome_emit = None
    if entropic_enabled:
        import entropic_core
        from entropic_core.core.llm_middleware import LLMMiddleware, MiddlewareConfig
        from entropic_core.core.runtime_events import (
            JsonlEventSink,
            RuntimeEvent,
            safe_emit,
        )

        # "Benchmark mode": always inject grounding/stability, disable retries to keep latency comparable.
        cfg = MiddlewareConfig(
            high_entropy_threshold=0.0,
            hallucination_threshold=0.0,
            enable_retry_logic=False,
            enable_cost_optimization=False,
            enable_context_pruning=False,
            enable_evidence_audit=True,
        )
        if runtime_events_path is not None:
            sink = JsonlEventSink(Path(runtime_events_path))
            cfg.event_sink = sink
            cfg.run_id = str(run_id or "")

            def _emit_outcome(*, case_id: str, ok: bool, reasons: List[str], category: str) -> None:
                safe_emit(
                    sink,
                    RuntimeEvent(
                        type="outcome",
                        run_id=str(run_id or ""),
                        request_id=str(case_id or ""),
                        provider=str(prov or ""),
                        model=str(model or ""),
                        payload={
                            "case_id": str(case_id or ""),
                            "category": str(category or ""),
                            "ok": bool(ok),
                            "reasons": list(reasons or []),
                            "mode": str(mode or ""),
                        },
                    ),
                )

            outcome_emit = _emit_outcome
        middleware = LLMMiddleware(config=cfg)
        # Stable wrapper (patches in-place).
        entropic_core.wrap(
            client, provider="openai" if prov == "mock" else "openrouter", middleware=middleware
        )
        create_fn = client.chat.completions.create

    out: List[CaseResult] = []

    for c in cases:
        groundedness_report: Optional[Dict[str, Any]] = None
        if c.category == "tool_loop":
            from .tools import ToolError, default_registry

            here = Path(__file__).resolve().parent
            reg = default_registry(sandbox_dir=here / "tool_sandbox")

            messages = _build_messages(c, allow_tools=True)
            tool_calls: List[Dict[str, Any]] = []
            steps = 0

            usage_total = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
            latency_total_ms = 0.0
            cost_total = 0.0
            cost_total_ok = True

            final_text = ""
            final_obj: Optional[dict] = None
            final_ok = False
            final_reasons: List[str] = ["No final answer."]
            grounded = True
            contra = True

            for _step in range(1, 6):
                steps += 1
                t0 = time.perf_counter()
                resp = create_fn(model=model, messages=messages, temperature=0.2)
                latency_ms = (time.perf_counter() - t0) * 1000.0
                latency_total_ms += latency_ms

                text = _extract_text(resp)
                usage = _extract_usage(resp)
                for k in ("prompt_tokens", "completion_tokens", "total_tokens"):
                    usage_total[k] += int(usage.get(k, 0) or 0)

                if in_per_1k is not None and out_per_1k is not None:
                    cost_total += _estimate_cost_usd(usage, in_per_1k, out_per_1k)
                else:
                    cost_total_ok = False

                obj = _json_object_from_text(text)
                if obj is None:
                    final_text = text
                    final_obj = None
                    final_ok, final_reasons, final_obj, grounded, contra = grade_case(
                        c,
                        text,
                        tool_calls=tool_calls,
                        claim_groundedness=groundedness_report,
                    )
                    break

                action = str(obj.get("action") or "").strip().lower()
                if action == "tool":
                    tool_name = str(obj.get("tool_name") or "").strip()
                    tool_input = obj.get("tool_input")
                    tool_calls.append({"tool_name": tool_name, "tool_input": tool_input})
                    try:
                        out_text = reg.execute(tool_name, tool_input)
                    except ToolError as e:
                        messages.append({"role": "assistant", "content": text})
                        messages.append(
                            {"role": "tool", "content": f"TOOL_ERROR: {type(e).__name__}: {e}"}
                        )
                        final_text = text
                        final_ok = False
                        final_reasons = [f"Tool error: {e}"]
                        final_obj = obj
                        break

                    messages.append({"role": "assistant", "content": text})
                    messages.append({"role": "tool", "content": out_text})
                    continue

                # Otherwise treat as final.
                final_text = text
                groundedness_report = _claim_report_for_case_obj(c, obj)
                final_ok, final_reasons, final_obj, grounded, contra = grade_case(
                    c,
                    text,
                    tool_calls=tool_calls,
                    claim_groundedness=groundedness_report,
                )
                break

            text = final_text
            usage = usage_total
            latency_ms = latency_total_ms
            ok = final_ok
            reasons = final_reasons
            obj = final_obj
            cost = cost_total if cost_total_ok else None
            retry_count = 0
            is_refusal = _detect_refusal(text)
            tool_calls_count = len(tool_calls)
        else:
            messages = _build_messages(c)

            t0 = time.perf_counter()
            resp = create_fn(model=model, messages=messages, temperature=0.2)
            latency_ms = (time.perf_counter() - t0) * 1000.0

            text = _extract_text(resp)
            usage = _extract_usage(resp)
            obj = _json_object_from_text(text)
            groundedness_report = _claim_report_for_case_obj(c, obj)
            ok, reasons, obj, grounded, contra = grade_case(
                c,
                text,
                claim_groundedness=groundedness_report,
            )

            cost = None
            if in_per_1k is not None and out_per_1k is not None:
                cost = _estimate_cost_usd(usage, in_per_1k, out_per_1k)
            retry_count = 0
            is_refusal = _detect_refusal(text)
            tool_calls_count = 0

        if callable(outcome_emit):
            try:
                outcome_emit(case_id=c.case_id, ok=ok, reasons=reasons, category=c.category)
            except Exception:
                pass

        entropy_before = None
        entropy_after = None
        interventions = None
        if middleware is not None and middleware.call_history:
            last = middleware.call_history[-1]
            entropy_before = last.entropy_before
            entropy_after = last.entropy_after
            interventions = list(last.interventions_applied or [])
            retry_count = int(getattr(last, "retry_count", 0) or 0)
            is_refusal = bool(getattr(last, "is_refusal", False))
            tool_calls_count = int(getattr(last, "tool_calls_count", 0) or 0)

        groundedness_summary = None
        groundedness_claims = None
        if isinstance(groundedness_report, dict):
            if isinstance(groundedness_report.get("summary"), dict):
                groundedness_summary = groundedness_report.get("summary")
            claims = groundedness_report.get("claims")
            if isinstance(claims, list):
                groundedness_claims = claims

        out.append(
            CaseResult(
                case_id=c.case_id,
                ok=ok,
                reasons=reasons,
                response_text=text,
                response_json=obj,
                usage=usage,
                latency_ms=latency_ms,
                grounded=grounded,
                contradiction_handled=contra,
                mode=mode,
                entropy_before=entropy_before,
                entropy_after=entropy_after,
                interventions_applied=interventions,
                cost_usd_est=cost,
                tool_calls=tool_calls if c.category == "tool_loop" else None,
                steps=steps if c.category == "tool_loop" else 1,
                retry_count=int(retry_count or 0),
                is_refusal=bool(is_refusal),
                tool_calls_count=int(tool_calls_count or 0),
                groundedness_summary=groundedness_summary,
                groundedness_claims=groundedness_claims,
            )
        )

    return out


def _summarize(results: List[CaseResult]) -> Dict[str, Any]:
    total = len(results)
    passed = sum(1 for r in results if r.ok)
    grounded = sum(1 for r in results if r.grounded)
    contra_ok = sum(1 for r in results if r.contradiction_handled)
    refusals = sum(1 for r in results if r.is_refusal)
    retries = sum(int(r.retry_count or 0) for r in results)
    tool_calls = sum(int(r.tool_calls_count or 0) for r in results)
    # Tool-loop cases are marked by category in the dataset; we store tool_calls list for them.
    tool_loop = [r for r in results if (r.tool_calls is not None)]
    tool_loop_total = len(tool_loop)
    tool_loop_ok = sum(1 for r in tool_loop if r.ok)
    tool_loop_used = sum(1 for r in tool_loop if len(r.tool_calls or []) > 0)
    claim_total = 0
    claim_supported = 0
    for r in results:
        s = r.groundedness_summary
        if isinstance(s, dict):
            claim_total += int(s.get("total_claims", 0) or 0)
            claim_supported += int(s.get("supported_claims", 0) or 0)
    tokens = {
        "prompt_tokens": sum(r.usage.get("prompt_tokens", 0) for r in results),
        "completion_tokens": sum(r.usage.get("completion_tokens", 0) for r in results),
        "total_tokens": sum(r.usage.get("total_tokens", 0) for r in results),
    }
    lat = [r.latency_ms for r in results]
    cost = [r.cost_usd_est for r in results if r.cost_usd_est is not None]
    return {
        "total": total,
        "passed": passed,
        "pass_rate": passed / max(1, total),
        "verifiable_error_rate": 1.0 - (passed / max(1, total)),
        "grounded_rate": grounded / max(1, total),
        "contradiction_handling_rate": contra_ok / max(1, total),
        "refusal_rate": refusals / max(1, total),
        "tool_loop_total": int(tool_loop_total),
        "tool_use_rate": (float(tool_loop_used) / float(max(1, tool_loop_total)))
        if tool_loop_total
        else None,
        "tool_correct_rate": (float(tool_loop_ok) / float(max(1, tool_loop_total)))
        if tool_loop_total
        else None,
        "retries_total": retries,
        "retries_avg": float(retries) / float(max(1, total)),
        "tool_calls_total": tool_calls,
        "tool_calls_avg": float(tool_calls) / float(max(1, total)),
        "groundedness_claims_total": claim_total,
        "groundedness_claims_supported": claim_supported,
        "groundedness_claim_supported_rate": float(claim_supported) / float(max(1, claim_total)),
        "tokens": tokens,
        "latency_ms_avg": sum(lat) / max(1, len(lat)),
        "latency_ms_p95": sorted(lat)[int(0.95 * (len(lat) - 1))] if lat else 0.0,
        "cost_usd_est": sum(cost) if cost else None,
    }


def _write_jsonl(path: Path, rows: Iterable[dict]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")


def _svg_compare(
    a: float,
    b: float,
    width: int = 340,
    height: int = 100,
    label_a: str = "baseline",
    label_b: str = "entropic",
) -> str:
    pad = 14
    bar_w = (width - pad * 2) // 2 - 10
    max_h = height - 42
    a_h = int(max_h * max(0.0, min(1.0, a)))
    b_h = int(max_h * max(0.0, min(1.0, b)))
    base_y = height - 26
    ax = pad
    bx = pad + bar_w + 20
    return f"""
<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}">
  <line x1="{pad}" y1="{base_y}" x2="{width - pad}" y2="{base_y}" stroke="rgba(255,255,255,0.28)" stroke-width="1"/>
  <rect x="{ax}" y="{base_y - a_h}" width="{bar_w}" height="{a_h}" fill="#c44" opacity="0.85"/>
  <text x="{ax + bar_w / 2}" y="{base_y + 16}" text-anchor="middle" font-family="ui-monospace, Menlo, Consolas, monospace" font-size="12">{html.escape(label_a)}</text>
  <text x="{ax + bar_w / 2}" y="{base_y - a_h - 6}" text-anchor="middle" font-family="ui-monospace, Menlo, Consolas, monospace" font-size="12">{a * 100:.0f}%</text>
  <rect x="{bx}" y="{base_y - b_h}" width="{bar_w}" height="{b_h}" fill="#2a8" opacity="0.90"/>
  <text x="{bx + bar_w / 2}" y="{base_y + 16}" text-anchor="middle" font-family="ui-monospace, Menlo, Consolas, monospace" font-size="12">{html.escape(label_b)}</text>
  <text x="{bx + bar_w / 2}" y="{base_y - b_h - 6}" text-anchor="middle" font-family="ui-monospace, Menlo, Consolas, monospace" font-size="12">{b * 100:.0f}%</text>
</svg>
""".strip()


def _write_report(outdir: Path, payload: Dict[str, Any]) -> None:
    base = payload["summary"]["baseline"]
    ent = payload["summary"]["entropic"]

    svg_pass = _svg_compare(
        base["pass_rate"], ent["pass_rate"], label_a="baseline", label_b="entropic"
    )
    svg_ground = _svg_compare(
        base["grounded_rate"], ent["grounded_rate"], label_a="baseline", label_b="entropic"
    )
    svg_contra = _svg_compare(
        base["contradiction_handling_rate"],
        ent["contradiction_handling_rate"],
        label_a="baseline",
        label_b="entropic",
    )

    def esc(s: str) -> str:
        return html.escape(s, quote=False)

    rows = []
    # Render cases side-by-side, matching by case_id
    bmap = {r["case_id"]: r for r in payload["results"]["baseline"]}
    emap = {r["case_id"]: r for r in payload["results"]["entropic"]}
    for cid in payload["case_order"]:
        br = bmap[cid]
        er = emap[cid]
        rows.append(
            f"""
<section class="case">
  <div class="casehead">
    <div class="cid">{esc(cid)}</div>
    <div class="badges">
      <span class="badge {'ok' if br['ok'] else 'bad'}">A {'PASS' if br['ok'] else 'FAIL'}</span>
      <span class="badge {'ok' if er['ok'] else 'bad'}">B {'PASS' if er['ok'] else 'FAIL'}</span>
    </div>
  </div>
  <div class="grid">
    <div class="pane">
      <div class="panehead">Baseline</div>
      <div class="meta">tokens: {br['usage']['total_tokens']} | latency: {br['latency_ms']:.0f}ms | steps: {int(br.get('steps', 1) or 1)} | tools: {len(br.get('tool_calls') or [])}</div>
      <pre>{esc(br['response_text'])}</pre>
      <div class="why">{esc(' | '.join(br.get('reasons', [])) or 'ok')}</div>
    </div>
    <div class="pane">
      <div class="panehead">Entropic</div>
      <div class="meta">tokens: {er['usage']['total_tokens']} | latency: {er['latency_ms']:.0f}ms | steps: {int(er.get('steps', 1) or 1)} | tools: {len(er.get('tool_calls') or [])}</div>
      <pre>{esc(er['response_text'])}</pre>
      <div class="why">{esc(' | '.join(er.get('reasons', [])) or 'ok')}</div>
      <details>
        <summary>entropy + interventions</summary>
        <pre>{esc(json.dumps({k: er.get(k) for k in ('entropy_before', 'entropy_after', 'interventions_applied')}, indent=2))}</pre>
      </details>
    </div>
  </div>
</section>
""".strip()
        )

    html_doc = f"""<!doctype html>
<html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <title>Entropic Evals Report</title>
    <style>
      :root {{
        --bg: #0b0e12;
        --panel: #111826;
        --text: #e8eef8;
        --muted: #a8b3c7;
        --ok: #1fbf8f;
        --bad: #ff5a5f;
        --line: rgba(232,238,248,0.12);
      }}
      html, body {{
        background: radial-gradient(1200px 700px at 20% 10%, rgba(42,136,170,0.20), transparent 60%),
                    radial-gradient(900px 500px at 90% 20%, rgba(196,68,68,0.18), transparent 55%),
                    var(--bg);
        color: var(--text); margin: 0;
        font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif;
      }}
      .wrap {{ max-width: 1150px; margin: 0 auto; padding: 26px 18px 60px; }}
      header {{ display: grid; gap: 8px; }}
      h1 {{ margin: 0; font-size: 26px; }}
      .sub {{ color: var(--muted); font-size: 13px; }}
      .cards {{ display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 12px; margin-top: 14px; }}
      .card {{ background: rgba(255,255,255,0.03); border: 1px solid var(--line); border-radius: 14px; padding: 12px; }}
      .card h2 {{ margin: 0 0 8px; font-size: 13px; color: var(--muted); font-weight: 700; text-transform: uppercase; letter-spacing: 0.35px; }}
      .kpis {{ display: grid; grid-template-columns: 1fr 1fr; gap: 8px; }}
      .kpi {{ background: rgba(0,0,0,0.22); border: 1px solid var(--line); border-radius: 12px; padding: 10px; }}
      .kpi .label {{ color: var(--muted); font-size: 12px; }}
      .kpi .val {{ margin-top: 6px; font-family: ui-monospace, Menlo, Consolas, monospace; font-size: 16px; }}
      .cases {{ margin-top: 14px; display: grid; gap: 12px; }}
      .case {{ background: rgba(255,255,255,0.02); border: 1px solid var(--line); border-radius: 14px; padding: 12px; }}
      .casehead {{ display: flex; justify-content: space-between; align-items: center; gap: 10px; margin-bottom: 10px; }}
      .cid {{ font-family: ui-monospace, Menlo, Consolas, monospace; font-size: 13px; color: #dbe7ff; }}
      .badges {{ display: flex; gap: 8px; }}
      .badge {{ font-family: ui-monospace, Menlo, Consolas, monospace; font-size: 12px; padding: 2px 8px; border-radius: 999px; border: 1px solid var(--line); }}
      .badge.ok {{ background: rgba(31,191,143,0.18); color: #baf6e2; }}
      .badge.bad {{ background: rgba(255,90,95,0.18); color: #ffd0d1; }}
      .grid {{ display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }}
      .pane {{ background: rgba(0,0,0,0.22); border: 1px solid var(--line); border-radius: 12px; padding: 10px; overflow: hidden; }}
      .panehead {{ font-weight: 750; }}
      .meta {{ margin-top: 6px; color: var(--muted); font-size: 12px; }}
      .why {{ margin-top: 8px; color: var(--muted); font-size: 12px; }}
      pre {{ margin: 10px 0 0; white-space: pre-wrap; word-wrap: break-word; font-size: 12px; line-height: 1.35; font-family: ui-monospace, Menlo, Consolas, monospace; color: #dce7fb; }}
      details {{ margin-top: 10px; }}
      summary {{ cursor: pointer; color: var(--muted); font-size: 12px; }}
      @media (max-width: 980px) {{
        .cards {{ grid-template-columns: 1fr; }}
        .grid {{ grid-template-columns: 1fr; }}
      }}
    </style>
  </head>
  <body>
    <div class="wrap">
      <header>
        <h1>Entropic Evals</h1>
        <div class="sub">
          suite: <span style="font-family: ui-monospace, Menlo, Consolas, monospace;">{esc(payload['suite']['suite'])}@{esc(payload['suite']['version'])}</span>
          | provider: <span style="font-family: ui-monospace, Menlo, Consolas, monospace;">{esc(payload.get('provider', ''))}</span>
          | model: <span style="font-family: ui-monospace, Menlo, Consolas, monospace;">{esc(payload['model'])}</span>
          | run: <span style="font-family: ui-monospace, Menlo, Consolas, monospace;">{esc(payload['run_id'])}</span>
        </div>
      </header>

      <div class="cards">
        <div class="card">
          <h2>Pass Rate</h2>
          {svg_pass}
          <div class="kpis">
            <div class="kpi"><div class="label">A passed</div><div class="val">{base['passed']}/{base['total']}</div></div>
            <div class="kpi"><div class="label">B passed</div><div class="val">{ent['passed']}/{ent['total']}</div></div>
            <div class="kpi"><div class="label">A verifiable error</div><div class="val">{base['verifiable_error_rate'] * 100:.0f}%</div></div>
            <div class="kpi"><div class="label">B verifiable error</div><div class="val">{ent['verifiable_error_rate'] * 100:.0f}%</div></div>
          </div>
        </div>
        <div class="card">
          <h2>Groundedness</h2>
          {svg_ground}
          <div class="kpis">
            <div class="kpi"><div class="label">A grounded</div><div class="val">{base['grounded_rate'] * 100:.0f}%</div></div>
            <div class="kpi"><div class="label">B grounded</div><div class="val">{ent['grounded_rate'] * 100:.0f}%</div></div>
            <div class="kpi"><div class="label">A claim support</div><div class="val">{base.get('groundedness_claim_supported_rate', 0.0) * 100:.0f}%</div></div>
            <div class="kpi"><div class="label">B claim support</div><div class="val">{ent.get('groundedness_claim_supported_rate', 0.0) * 100:.0f}%</div></div>
          </div>
        </div>
        <div class="card">
          <h2>Contradictions</h2>
          {svg_contra}
          <div class="kpis">
            <div class="kpi"><div class="label">A tokens</div><div class="val">{base['tokens']['total_tokens']}</div></div>
            <div class="kpi"><div class="label">B tokens</div><div class="val">{ent['tokens']['total_tokens']}</div></div>
            <div class="kpi"><div class="label">A refusals</div><div class="val">{base.get('refusal_rate', 0.0) * 100:.0f}%</div></div>
            <div class="kpi"><div class="label">B refusals</div><div class="val">{ent.get('refusal_rate', 0.0) * 100:.0f}%</div></div>
          </div>
        </div>
      </div>

      <div class="cases">
        {''.join(rows)}
      </div>
    </div>
  </body>
</html>
"""
    (outdir / "report.html").write_text(html_doc, encoding="utf-8")


def main(argv: Optional[List[str]] = None) -> int:
    here = Path(__file__).resolve().parent
    ds_dir = here / "datasets"

    p = argparse.ArgumentParser(
        prog="entropic-evals",
        description="Reproducible A/B evals for Entropic Core (experimental).",
    )
    p.add_argument(
        "--suite",
        default=os.getenv("ENTROPIC_EVAL_SUITE", "core_v1"),
        help="Suite name (dataset file without .jsonl) under experimental/evals/datasets/",
    )
    p.add_argument(
        "--provider",
        default=os.getenv("ENTROPIC_EVAL_PROVIDER", "openrouter"),
        choices=["openrouter", "mock"],
    )
    p.add_argument("--model", default=os.getenv("OPENROUTER_MODEL"))
    p.add_argument("--out", default=os.getenv("ENTROPIC_EVAL_OUT", "entropic_evals_runs"))
    p.add_argument("--max-cases", type=int, default=0, help="0 means all cases.")
    p.add_argument("--no-ab", action="store_true", help="Run only baseline (no Entropic).")
    p.add_argument(
        "--list-suites",
        action="store_true",
        help="List available suites and exit.",
    )
    p.add_argument(
        "--runtime-events",
        action="store_true",
        help="Write runtime events.jsonl and runtime_report.html for entropic mode.",
    )
    p.add_argument(
        "--external-sample-size",
        type=int,
        default=int(os.getenv("ENTROPIC_EXTERNAL_SAMPLE_SIZE", "250") or "250"),
        help="Sample size for auto-built external benchmark suites (if selected).",
    )
    p.add_argument(
        "--external-seed",
        type=int,
        default=int(os.getenv("ENTROPIC_EXTERNAL_SEED", "1337") or "1337"),
        help="Random seed for external benchmark sampling.",
    )
    p.add_argument(
        "--refresh-external",
        action="store_true",
        help="Regenerate external suite dataset even if local cache file exists.",
    )
    p.add_argument(
        "--price-in", type=float, default=None, help="USD per 1K prompt tokens (optional)."
    )
    p.add_argument(
        "--price-out", type=float, default=None, help="USD per 1K completion tokens (optional)."
    )
    args = p.parse_args(argv)

    if args.list_suites:
        here = Path(__file__).resolve().parent
        ds_dir = here / "datasets"
        suites = {p.stem for p in ds_dir.glob("*.jsonl")}
        try:
            from .external import EXTERNAL_SUITES

            suites.update(EXTERNAL_SUITES)
        except Exception:
            pass
        suites_sorted = sorted(suites)
        for s in suites_sorted:
            print(s)
        return 0

    if not args.model:
        if str(args.provider).lower().strip() == "mock":
            args.model = "mock-v1"
        else:
            raise SystemExit("Missing model. Set OPENROUTER_MODEL or pass --model.")

    suite_path = ds_dir / f"{args.suite}.jsonl"
    try:
        from .external import EXTERNAL_SUITES, build_squad_v1_external
    except Exception:
        EXTERNAL_SUITES = tuple()
        build_squad_v1_external = None

    if args.suite in EXTERNAL_SUITES:
        if args.refresh_external and suite_path.exists():
            suite_path.unlink()
        if not suite_path.exists():
            if args.suite == "squad_v1_external" and callable(build_squad_v1_external):
                build_squad_v1_external(
                    out_path=suite_path,
                    sample_size=max(0, int(args.external_sample_size)),
                    seed=int(args.external_seed),
                )
            else:
                raise SystemExit(f"Unsupported external suite: {args.suite}")

    if not suite_path.exists():
        raise SystemExit(f"Suite file not found: {suite_path}")

    meta, cases = load_suite(suite_path)
    if args.max_cases and args.max_cases > 0:
        cases = cases[: args.max_cases]

    run_id = _utc_stamp()
    outdir = (
        Path(args.out)
        / meta.suite
        / str(args.provider)
        / f"{run_id}_{re.sub(r'[^a-zA-Z0-9._-]+', '_', args.model)}"
    )
    outdir.mkdir(parents=True, exist_ok=True)

    env_info = {
        "python": sys.version.split()[0],
        "platform": platform.platform(),
        "entropic_core_version": None,
        "git": _git_info(),
        "provider_sdk_versions": {"openai": _pkg_version("openai")},
    }
    try:
        import entropic_core

        env_info["entropic_core_version"] = entropic_core.__version__
    except Exception:
        pass

    payload: Dict[str, Any] = {
        "run_id": run_id,
        "provider": args.provider,
        "model": args.model,
        "suite": dataclasses.asdict(meta),
        "suite_sha256": _sha256_file(suite_path),
        "created_utc": datetime.now(timezone.utc).isoformat(),
        "env": env_info,
        "pricing": {"price_in_usd_per_1k": args.price_in, "price_out_usd_per_1k": args.price_out},
        "runtime_events": {"enabled": bool(args.runtime_events), "path": "events.jsonl"},
        "results": {},
        "summary": {},
        "case_order": [c.case_id for c in cases],
    }

    # A: baseline
    baseline = _run_mode(
        "baseline",
        args.model,
        cases,
        provider=args.provider,
        entropic_enabled=False,
        in_per_1k=args.price_in,
        out_per_1k=args.price_out,
    )
    payload["results"]["baseline"] = [dataclasses.asdict(r) for r in baseline]
    payload["summary"]["baseline"] = _summarize(baseline)

    # B: entropic
    if not args.no_ab:
        events_path = outdir / "events.jsonl"
        entropic = _run_mode(
            "entropic",
            args.model,
            cases,
            provider=args.provider,
            entropic_enabled=True,
            in_per_1k=args.price_in,
            out_per_1k=args.price_out,
            runtime_events_path=(events_path if args.runtime_events else None),
            run_id=run_id,
        )
        payload["results"]["entropic"] = [dataclasses.asdict(r) for r in entropic]
        payload["summary"]["entropic"] = _summarize(entropic)
    else:
        payload["results"]["entropic"] = []
        payload["summary"]["entropic"] = {
            "total": 0,
            "passed": 0,
            "pass_rate": 0.0,
            "verifiable_error_rate": 1.0,
            "grounded_rate": 0.0,
            "contradiction_handling_rate": 0.0,
            "refusal_rate": 0.0,
            "tool_loop_total": 0,
            "tool_use_rate": None,
            "tool_correct_rate": None,
            "retries_total": 0,
            "retries_avg": 0.0,
            "tool_calls_total": 0,
            "tool_calls_avg": 0.0,
            "groundedness_claims_total": 0,
            "groundedness_claims_supported": 0,
            "groundedness_claim_supported_rate": 0.0,
            "tokens": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
            "latency_ms_avg": 0.0,
            "latency_ms_p95": 0.0,
            "cost_usd_est": None,
        }

    # Write artifacts (versioned outputs).
    (outdir / "run.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
    _write_jsonl(outdir / "baseline.jsonl", payload["results"]["baseline"])
    if payload["results"]["entropic"]:
        _write_jsonl(outdir / "entropic.jsonl", payload["results"]["entropic"])

    # Minimal CSV for spreadsheet diffing.
    def _to_csv_rows(mode: str, rows: List[dict]) -> List[str]:
        header = "mode,case_id,ok,grounded,contradiction_handled,total_tokens,latency_ms"
        lines = [header]
        for r in rows:
            lines.append(
                ",".join(
                    [
                        mode,
                        r["case_id"],
                        str(bool(r["ok"])).lower(),
                        str(bool(r["grounded"])).lower(),
                        str(bool(r["contradiction_handled"])).lower(),
                        str(int(r["usage"].get("total_tokens", 0))),
                        f"{float(r['latency_ms']):.1f}",
                    ]
                )
            )
        return lines

    csv_lines = []
    csv_lines += _to_csv_rows("baseline", payload["results"]["baseline"])
    if payload["results"]["entropic"]:
        csv_lines += _to_csv_rows("entropic", payload["results"]["entropic"])[1:]
    (outdir / "summary.csv").write_text("\n".join(csv_lines) + "\n", encoding="utf-8")

    _write_report(outdir, payload)

    # Optional runtime report from runtime events (entropic mode).
    if args.runtime_events and not args.no_ab:
        try:
            from entropic_core.experimental.reporting.runtime_report import generate_runtime_report

            generate_runtime_report(
                events_path=outdir / "events.jsonl",
                outdir=outdir,
                run_meta={
                    "run_id": run_id,
                    "provider": args.provider,
                    "model": args.model,
                    "suite": meta.suite,
                    "mode": "entropic",
                },
            )
        except Exception:
            pass

    print(str(outdir / "report.html"))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
