"""hdx eval — run quality evaluation suites against an MCP server."""

from __future__ import annotations

import asyncio
from pathlib import Path

import click

from hatchdx.eval.loader import load_suites
from hatchdx.eval.models import EvalError
from hatchdx.eval.reporter import generate_html_report, report_eval_results
from hatchdx.eval.runner import run_eval_suites
from hatchdx.eval.storage import EvalStorage
from hatchdx.harness.simulator import SimulatorError
from hatchdx.utils import console
from hatchdx.utils.command import resolve_command
from hatchdx.utils.config import ConfigError, load_config


@click.command("eval")
@click.option(
    "--evals-dir",
    "-d",
    default=None,
    help="Path to evals directory (default: evals/)",
)
@click.option(
    "--suite",
    "-s",
    "suite_filter",
    default=None,
    help="Run only suites whose name matches this value",
)
@click.option(
    "--timeout",
    "-t",
    default=5000,
    type=int,
    help="Timeout per tool call in milliseconds",
)
@click.option(
    "--update-golden",
    is_flag=True,
    help="Update golden files with current responses",
)
@click.option(
    "--html-report",
    default=None,
    type=click.Path(),
    help="Generate an HTML report at this path",
)
@click.option(
    "--compare",
    is_flag=True,
    help="Show comparison against previous run (regression detection)",
)
@click.option(
    "--sandbox",
    is_flag=True,
    help="Run evals inside a sandboxed container",
)
@click.option(
    "--policy",
    "-p",
    "sandbox_policy",
    default=None,
    help="Sandbox policy preset (strict/standard/permissive) or path to YAML file. Implies --sandbox.",
)
@click.option(
    "--skip-validation",
    is_flag=True,
    help="Skip the automatic protocol validation pre-check",
)
def eval_cmd(
    evals_dir,
    suite_filter,
    timeout,
    update_golden,
    html_report,
    compare,
    sandbox,
    sandbox_policy,
    skip_validation,
):
    """Run quality evaluation suites against your MCP server."""
    # 1. Load project config
    try:
        config = load_config()
    except ConfigError as exc:
        console.error(str(exc))
        raise SystemExit(1)

    console.info(f"Evaluating {config.server_name}")
    click.echo()

    # 2. Discover eval suites
    evals_path = Path(evals_dir) if evals_dir else Path("evals")

    try:
        suites = load_suites(evals_path)
    except EvalError as exc:
        console.error(str(exc))
        raise SystemExit(1)

    if not suites:
        console.warning(f"No eval suites found in {evals_path}/")
        click.echo()
        click.echo("  Create an eval suite file:")
        click.echo(f"    {evals_path}/quality.yaml")
        click.echo()
        click.echo("  Example:")
        click.echo('    suite: "my tool quality"')
        click.echo('    description: "Quality checks for my tools"')
        click.echo("    evals:")
        click.echo('      - name: "basic check"')
        click.echo('        tool: "my_tool"')
        click.echo("        input: {}")
        click.echo("        assertions:")
        click.echo("          - type: schema")
        click.echo("            schema: { type: object }")
        raise SystemExit(1)

    # 3. Apply suite filter
    if suite_filter:
        suites = [s for s in suites if suite_filter.lower() in s.name.lower()]
        if not suites:
            console.warning(f"No suites match filter: {suite_filter}")
            raise SystemExit(1)

    total_evals = sum(len(s.evals) for s in suites)
    console.step(
        f"Discovered {len(suites)} suite{'s' if len(suites) != 1 else ''} "
        f"with {total_evals} eval{'s' if total_evals != 1 else ''}"
    )

    # 4. Resolve server command
    try:
        command = resolve_command(config.server_command)
    except ValueError as exc:
        console.error(f"Invalid server command: {exc}")
        raise SystemExit(1)

    if not command:
        console.error("Server command is empty. Check your hdx config.")
        raise SystemExit(1)

    timeout_sec = timeout / 1000

    # 5. Run validation pre-check (unless skipped)
    if not skip_validation and not sandbox:
        _run_validation_precheck(command, config.server_name, timeout_sec)

    # 6. Set up storage
    storage = EvalStorage()

    # 7. Resolve sandbox engine if requested
    # --policy implies --sandbox
    use_sandbox = sandbox or sandbox_policy is not None
    sandbox_engine = None

    if use_sandbox:
        sandbox_engine = _create_sandbox_engine(sandbox_policy)
        console.step("Running evals in sandbox...")
    else:
        console.step("Starting server...")

    # 8. Run evals
    try:
        run_id, results_by_suite = asyncio.run(
            run_eval_suites(
                command,
                suites,
                timeout=timeout_sec,
                evals_dir=evals_path,
                update_golden=update_golden,
                storage=storage,
                sandbox_engine=sandbox_engine,
            )
        )
    except SimulatorError as exc:
        click.echo()
        console.error(f"Failed to start server: {exc}")
        click.echo()
        click.echo("  Make sure your server is installed:")
        click.echo("    uv sync")
        storage.close()
        raise SystemExit(1)
    except Exception as exc:
        click.echo()
        console.error(f"Eval execution failed: {exc}")
        storage.close()
        raise SystemExit(1)

    if use_sandbox:
        console.success("Sandbox evals complete")
    else:
        console.success("Server started")

    # 9. Detect regressions
    regressions = None
    if compare and run_id is not None:
        regressions_list = []
        for suite in suites:
            regressions_list.extend(
                storage.detect_regressions(suite.name, run_id)
            )
        regressions = regressions_list if regressions_list else None

    storage.close()

    # 10. Report results
    exit_code = report_eval_results(
        results_by_suite,
        suites,
        regressions=regressions,
    )

    # 11. Generate HTML report if requested
    if html_report:
        report_path = generate_html_report(
            results_by_suite,
            suites,
            Path(html_report),
            regressions=regressions,
        )
        click.echo()
        console.success(f"HTML report saved to {report_path}")

    raise SystemExit(exit_code)


def _run_validation_precheck(
    command: list[str], server_name: str, timeout: float
) -> None:
    """Run protocol validation as a pre-check before evals.

    Errors cause a hard failure. Warnings are displayed but don't block.
    """
    from hatchdx.validator.models import ValidatorError
    from hatchdx.validator.reporter import report_validation
    from hatchdx.validator.runner import run_validation

    console.step("Running protocol validation pre-check...")

    try:
        report = asyncio.run(
            run_validation(
                command,
                server_name=server_name,
                timeout=timeout,
            )
        )
    except (SimulatorError, ValidatorError) as exc:
        console.warning(f"Validation pre-check failed: {exc}")
        console.warning("Continuing with evals anyway...")
        click.echo()
        return

    if report.has_errors:
        click.echo()
        report_validation(report)
        click.echo()
        console.error(
            "Protocol validation found errors. Fix them before running evals."
        )
        console.step("Use --skip-validation to bypass this check.")
        raise SystemExit(1)

    if report.warning_count > 0:
        click.echo()
        report_validation(report)
        click.echo()
        console.warning("Validation warnings found (non-blocking). Continuing with evals...")
        click.echo()
    else:
        console.success("Protocol validation passed")


def _create_sandbox_engine(policy: str | None) -> object:
    """Create a SandboxEngine for sandboxed eval execution."""
    from hatchdx.sandbox.engine import SandboxEngine
    from hatchdx.sandbox.runtime import SandboxError

    preset = None
    policy_path = None

    if policy is not None:
        if policy in ("strict", "standard", "permissive"):
            preset = policy
        else:
            policy_path = policy

    try:
        return SandboxEngine(
            project_path=".",
            preset=preset,
            policy_path=policy_path,
        )
    except SandboxError as exc:
        console.error(f"Failed to create sandbox: {exc}")
        raise SystemExit(1)
