"""Eval runner — orchestrate eval execution against an MCP server."""

from __future__ import annotations

import asyncio
import time
from datetime import datetime
from pathlib import Path
from typing import Any, Protocol

from hatchdx.eval.assertions import evaluate_assertion, parse_tool_response_data
from hatchdx.eval.golden import compare_golden, has_golden_file, save_golden
from hatchdx.eval.models import (
    AssertionOutcome,
    EvalCase,
    EvalError,
    EvalResult,
    EvalSuite,
)
from hatchdx.eval.storage import EvalStorage
from hatchdx.harness.simulator import SimulatorError, StdioSimulator


class ToolCaller(Protocol):
    """Minimal interface for anything that can call MCP tools."""

    async def call_tool(self, name: str, arguments: dict[str, Any] | None = None) -> Any: ...


async def run_eval_suites(
    command: list[str],
    suites: list[EvalSuite],
    *,
    timeout: float = 5.0,
    evals_dir: Path | None = None,
    update_golden: bool = False,
    storage: EvalStorage | None = None,
    sandbox_engine: Any | None = None,
) -> tuple[str | None, dict[str, list[EvalResult]]]:
    """Run all eval suites and return results organized by suite name.

    Returns ``(run_id, results_by_suite)`` where *run_id* is the storage
    run ID (or None if storage is disabled), and *results_by_suite* maps
    suite names to their eval results.

    When *sandbox_engine* is provided, tool calls are executed inside the
    sandbox container instead of via the stdio simulator.
    """
    run_id: str | None = None
    if storage is not None:
        run_id = storage.start_run()

    results_by_suite: dict[str, list[EvalResult]] = {}

    if sandbox_engine is not None:
        # Use sandbox engine for tool calls
        await sandbox_engine.start()
        try:
            for suite in suites:
                suite_results = await _run_suite(
                    _SandboxToolCaller(sandbox_engine),
                    suite,
                    evals_dir=evals_dir,
                    update_golden=update_golden,
                    storage=storage,
                    run_id=run_id,
                )
                results_by_suite[suite.name] = suite_results
        finally:
            sandbox_engine.cleanup()
    else:
        # Use stdio simulator
        async with StdioSimulator(command, timeout=timeout) as sim:
            for suite in suites:
                suite_results = await _run_suite(
                    sim,
                    suite,
                    evals_dir=evals_dir,
                    update_golden=update_golden,
                    storage=storage,
                    run_id=run_id,
                )
                results_by_suite[suite.name] = suite_results

    if storage is not None and run_id is not None:
        storage.finish_run(run_id)

    return run_id, results_by_suite


async def _run_suite(
    caller: Any,
    suite: EvalSuite,
    *,
    evals_dir: Path | None,
    update_golden: bool,
    storage: EvalStorage | None,
    run_id: str | None,
) -> list[EvalResult]:
    """Run all eval cases in a single suite."""
    suite_results: list[EvalResult] = []

    for eval_case in suite.evals:
        result = await _run_single_eval(
            caller,
            eval_case,
            suite_name=suite.name,
            evals_dir=evals_dir,
            update_golden=update_golden,
        )
        suite_results.append(result)

        if storage is not None and run_id is not None:
            storage.store_result(run_id, suite.name, result)

    return suite_results


class _SandboxToolCaller:
    """Adapts SandboxEngine to the call_tool interface expected by _run_single_eval."""

    def __init__(self, engine: Any) -> None:
        self._engine = engine

    async def call_tool(self, name: str, arguments: dict[str, Any] | None = None) -> Any:
        from hatchdx.harness.simulator import ToolCallResult

        result = await self._engine.run_tool(name, arguments or {})
        return ToolCallResult(
            content=result.content,
            is_error=result.is_error,
        )


async def _run_single_eval(
    caller: Any,
    eval_case: EvalCase,
    *,
    suite_name: str,
    evals_dir: Path | None = None,
    update_golden: bool = False,
) -> EvalResult:
    """Call the tool and evaluate all assertions for a single eval case."""
    start = time.perf_counter()
    try:
        tool_result = await caller.call_tool(eval_case.tool, eval_case.input)
    except SimulatorError as exc:
        elapsed_ms = (time.perf_counter() - start) * 1000
        return EvalResult(
            eval_case=eval_case,
            passed=False,
            assertion_results=[],
            latency_ms=elapsed_ms,
            timestamp=datetime.now(),
            error=f"Tool call failed: {exc}",
        )
    except asyncio.TimeoutError:
        elapsed_ms = (time.perf_counter() - start) * 1000
        return EvalResult(
            eval_case=eval_case,
            passed=False,
            assertion_results=[],
            latency_ms=elapsed_ms,
            timestamp=datetime.now(),
            error="Tool call timed out",
        )
    except Exception as exc:
        elapsed_ms = (time.perf_counter() - start) * 1000
        return EvalResult(
            eval_case=eval_case,
            passed=False,
            assertion_results=[],
            latency_ms=elapsed_ms,
            timestamp=datetime.now(),
            error=f"Unexpected error ({type(exc).__name__}): {exc}",
        )

    elapsed_ms = (time.perf_counter() - start) * 1000

    # Parse tool response content into structured data
    data = parse_tool_response_data(tool_result.content)

    # Evaluate each assertion
    assertion_results: list[AssertionOutcome] = []
    all_passed = True

    for assertion in eval_case.assertions:
        if assertion.type == "golden_file":
            passed, reason = _evaluate_golden_assertion(
                assertion,
                data,
                suite_name=suite_name,
                eval_name=eval_case.name,
                evals_dir=evals_dir,
                update_golden=update_golden,
            )
        else:
            passed, reason = evaluate_assertion(assertion, data)

        assertion_results.append(AssertionOutcome(
            assertion=assertion,
            passed=passed,
            reason=reason,
        ))
        if not passed:
            all_passed = False

    return EvalResult(
        eval_case=eval_case,
        passed=all_passed,
        assertion_results=assertion_results,
        latency_ms=elapsed_ms,
        timestamp=datetime.now(),
    )


def _evaluate_golden_assertion(
    assertion: Any,
    data: Any,
    *,
    suite_name: str,
    eval_name: str,
    evals_dir: Path | None,
    update_golden: bool,
) -> tuple[bool, str]:
    """Evaluate a golden_file assertion."""
    if evals_dir is None:
        return False, "Cannot evaluate golden_file assertion: evals directory not configured"

    if update_golden or not has_golden_file(evals_dir, suite_name, eval_name):
        path = save_golden(evals_dir, suite_name, eval_name, data)
        return True, f"Golden file saved: {path}"

    matched, diff_text = compare_golden(evals_dir, suite_name, eval_name, data)
    if matched:
        return True, "Response matches golden file"

    return False, f"Response differs from golden file:\n{diff_text}"
