from __future__ import annotations

import json
from pathlib import Path

from entropic_core.experimental.evals.cli import EvalCase, Evidence, grade_case, load_suite


def test_suite_loads_and_meta_present() -> None:
    # Load packaged dataset (source tree).
    ds = Path("src/entropic_core/experimental/evals/datasets/core_v1.jsonl")
    meta, cases = load_suite(ds)
    assert meta.suite == "core_v1"
    assert meta.version
    assert len(cases) >= 5


def test_grade_case_json_schema_and_evidence_checks() -> None:
    ds = Path("src/entropic_core/experimental/evals/datasets/core_v1.jsonl")
    _, cases = load_suite(ds)
    c = next(x for x in cases if x.case_id == "core_v1_002")

    # Correct JSON + evidence.
    resp = '{"answer":"protect(), unprotect(), create_middleware(), wrap(client)","evidence_ids":["E1"],"confidence":0.9,"notes":""}'
    ok, reasons, obj, grounded, contra = grade_case(c, resp)
    assert ok is True
    assert reasons == []
    assert obj is not None
    assert grounded is True
    assert contra is True


def test_grade_case_answer_any_of_normalized() -> None:
    case = EvalCase(
        case_id="norm_001",
        category="factual_qa_external",
        context=[Evidence(id="E1", text="The Denver Broncos are based in Denver.")],
        question="Which team is based in Denver?",
        expected={"answers_any_of": ["Denver Broncos", "The Denver Broncos"]},
        checks=[
            {"type": "json_schema"},
            {"type": "requires_evidence", "min_ids": 1},
            {
                "type": "answer_any_of_normalized",
                "field": "answer",
                "values": ["Denver Broncos", "The Denver Broncos"],
            },
        ],
    )
    resp = json.dumps(
        {
            "answer": "the denver broncos",
            "evidence_ids": ["E1"],
            "confidence": 0.9,
            "notes": "",
        }
    )
    ok, reasons, _, grounded, _ = grade_case(case, resp)
    assert ok is True
    assert reasons == []
    assert grounded is True


def test_grade_case_claim_groundedness_check_fails_unsupported() -> None:
    case = EvalCase(
        case_id="claim_001",
        category="grounded_extract",
        context=[Evidence(id="E1", text="Alpha is 3.")],
        question="What is Alpha?",
        expected={},
        checks=[
            {"type": "json_schema"},
            {"type": "requires_evidence", "min_ids": 1},
            {
                "type": "claim_groundedness",
                "min_supported_rate": 0.5,
                "max_unsupported_claims": 0,
                "max_citation_invented": 0,
                "allow_unverifiable": False,
            },
        ],
    )
    resp = json.dumps(
        {
            "answer": "Alpha is 7.",
            "evidence_ids": ["E1"],
            "confidence": 0.8,
            "notes": "",
        }
    )
    claim_report = {
        "summary": {
            "total_claims": 1,
            "supported_claims": 0,
            "unsupported_claims": 1,
            "citation_invented": 0,
            "citation_misattributed": 0,
            "numeric_mismatch": 1,
            "contradiction": 0,
            "unverifiable": False,
            "supported_rate": 0.0,
        },
        "claims": [],
    }
    ok, reasons, _, grounded, _ = grade_case(case, resp, claim_groundedness=claim_report)
    assert ok is False
    assert any("supported_rate" in r or "unsupported_claims" in r for r in reasons)
    assert grounded is False
