"""Prompt eval suite runner using MockProvider for offline evaluation.

Runs prompt-eval-suite/1.0 files against a deterministic mock provider.
No network calls. Fully offline. Hash-based deterministic output generation.

Usage:
    from abi_evals.prompt_eval import run_prompt_eval_suite, PromptMockProvider
    results = run_prompt_eval_suite(Path("suite.json"))
"""
from __future__ import annotations

import hashlib
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any


@dataclass
class MockProviderResult:
    """Result from a mock provider invocation."""

    output: str
    tokens_used: int
    prompt_id: str


class PromptMockProvider:
    """Deterministic mock provider for prompt evaluation.

    Uses hash-based deterministic output generation.
    No network calls. Fully offline.
    """

    def __init__(self) -> None:
        self._responses: dict[str, str] = {}

    def set_response(self, input_hash: str, response: str) -> None:
        """Register a canned response for a specific input hash."""
        self._responses[input_hash] = response

    def invoke(self, prompt_id: str, inputs: dict[str, Any]) -> MockProviderResult:
        """Generate deterministic output based on input hash.

        The input hash is derived from the canonical JSON representation
        of (prompt_id, inputs). If a canned response was registered via
        set_response(), that response is returned; otherwise a structured
        fallback is generated deterministically.
        """
        canonical = json.dumps(
            {"prompt_id": prompt_id, "inputs": inputs},
            sort_keys=True,
            ensure_ascii=False,
        )
        input_hash = hashlib.sha256(canonical.encode()).hexdigest()[:16]

        if input_hash in self._responses:
            output = self._responses[input_hash]
        else:
            # Deterministic fallback: echo structured response
            output = json.dumps(
                {
                    "explanation": f"[mock] Explanation for input hash {input_hash}",
                    "findings": [],
                    "implementation": f"[mock] Implementation for {prompt_id}",
                    "verdict": "approve",
                },
                sort_keys=True,
                ensure_ascii=False,
            )

        tokens = (len(canonical) + len(output)) // 4  # deterministic token estimate
        return MockProviderResult(
            output=output,
            tokens_used=tokens,
            prompt_id=prompt_id,
        )


def _compute_input_hash(prompt_id: str, inputs: dict[str, Any]) -> str:
    """Compute the deterministic input hash for a (prompt_id, inputs) pair."""
    canonical = json.dumps(
        {"prompt_id": prompt_id, "inputs": inputs},
        sort_keys=True,
        ensure_ascii=False,
    )
    return hashlib.sha256(canonical.encode()).hexdigest()[:16]


def _check_required_keys(
    output_data: dict[str, Any],
    expected: dict[str, Any],
) -> tuple[str, str, str]:
    """Check required_keys constraint. Returns (status, message, diff)."""
    if "required_keys" not in expected:
        return "pass", "", ""
    missing = [k for k in expected["required_keys"] if k not in output_data]
    if missing:
        return (
            "fail",
            f"Missing keys: {missing}",
            f"expected keys {expected['required_keys']}, got {list(output_data.keys())}",
        )
    return "pass", "", ""


def _check_contains_patterns(
    output_str: str,
    expected: dict[str, Any],
) -> tuple[str, str, str]:
    """Check contains_patterns constraint. Returns (status, message, diff)."""
    if "contains_patterns" not in expected:
        return "pass", "", ""
    missing_patterns = [
        p for p in expected["contains_patterns"] if p.lower() not in output_str.lower()
    ]
    if missing_patterns:
        return (
            "fail",
            f"Missing patterns: {missing_patterns}",
            f"patterns {missing_patterns} not found in output",
        )
    return "pass", "", ""


def _check_budget(
    case: dict[str, Any],
    tokens_used: int,
) -> tuple[str, str, str]:
    """Check budget_expectation constraint. Returns (status, message, diff)."""
    if "budget_expectation" not in case:
        return "pass", "", ""
    budget = case["budget_expectation"]
    if "max_tokens" in budget and tokens_used > budget["max_tokens"]:
        return (
            "fail",
            f"Budget exceeded: {tokens_used} > {budget['max_tokens']}",
            f"tokens_used={tokens_used}, max_tokens={budget['max_tokens']}",
        )
    return "pass", "", ""


def _run_single_case(
    case: dict[str, Any],
    prompt_id: str,
    provider: PromptMockProvider,
) -> dict[str, Any]:
    """Run a single prompt eval case and return a result dict."""
    case_id = case["case_id"]
    try:
        result = provider.invoke(prompt_id, case["input"])

        # Parse output as JSON if possible
        if result.output.startswith("{"):
            output_data = json.loads(result.output)
        else:
            output_data = {"raw": result.output}

        status = "pass"
        message = ""
        diff = ""
        expected = case.get("expected", {})

        # Check required_keys
        s, m, d = _check_required_keys(output_data, expected)
        if s == "fail":
            status, message, diff = s, m, d

        # Check contains_patterns (only if still passing)
        if status == "pass":
            s, m, d = _check_contains_patterns(result.output, expected)
            if s == "fail":
                status, message, diff = s, m, d

        # Check budget (only if still passing)
        if status == "pass":
            s, m, d = _check_budget(case, result.tokens_used)
            if s == "fail":
                status, message, diff = s, m, d

        if status == "pass":
            message = "All checks passed"

        return {
            "case_id": case_id,
            "status": status,
            "actual": output_data,
            "message": message,
            "diff": diff,
        }
    except Exception as exc:
        return {
            "case_id": case_id,
            "status": "error",
            "actual": None,
            "message": str(exc),
            "diff": "",
        }


def load_prompt_eval_suite(path: Path) -> dict[str, Any]:
    """Load and validate a prompt eval suite from JSON file."""
    suite = json.loads(path.read_text(encoding="utf-8"))
    if "suite_id" not in suite:
        raise ValueError(f"Prompt eval suite missing 'suite_id': {path}")
    if "prompt_id" not in suite:
        raise ValueError(f"Prompt eval suite missing 'prompt_id': {path}")
    if "cases" not in suite or not suite["cases"]:
        raise ValueError(f"Prompt eval suite missing 'cases': {path}")
    return suite


def run_prompt_eval_suite(
    suite_path: Path,
    provider: PromptMockProvider | None = None,
) -> dict[str, Any]:
    """Run a prompt eval suite and return eval-result/1.0 compatible results.

    Args:
        suite_path: Path to a prompt-eval-suite/1.0 JSON file.
        provider: Optional mock provider. If None, a default is created.

    Returns:
        An eval-result/1.0 compatible dict with results and summary.
    """
    suite = load_prompt_eval_suite(suite_path)
    if provider is None:
        provider = PromptMockProvider()

    prompt_id = suite["prompt_id"]
    results: list[dict[str, Any]] = []
    passed = 0
    failed = 0
    errors = 0

    for case in sorted(suite["cases"], key=lambda c: c["case_id"]):
        case_result = _run_single_case(case, prompt_id, provider)
        results.append(case_result)

        if case_result["status"] == "pass":
            passed += 1
        elif case_result["status"] == "fail":
            failed += 1
        else:
            errors += 1

    # Deterministic run_id from suite content
    run_id = hashlib.sha256(
        json.dumps(suite, sort_keys=True, ensure_ascii=False).encode()
    ).hexdigest()[:8]

    return {
        "schema_version": "eval-result/1.0",
        "suite_id": suite["suite_id"],
        "run_id": run_id,
        "results": results,
        "summary": {
            "total": len(results),
            "passed": passed,
            "failed": failed,
            "errors": errors,
            "skipped": 0,
        },
    }
