"""CLI for evalkit — run, serve, report."""

from __future__ import annotations

import json
import os
from typing import Any

import click
from dotenv import load_dotenv

from evalkit.engine import EvaluationEngine
from evalkit.judges.base import BaseJudge
from evalkit.models import EvaluationReport, Suite, TestCase
from evalkit.suites import BUILTIN_SUITES


@click.group()
@click.version_option(package_name="evalkit-llm")
def app() -> None:
    """evalkit — Simple LLM/RAG evaluation for teams."""
    load_dotenv()


def _make_judge(provider: str, model: str | None, api_key: str | None) -> BaseJudge:
    # Judge imports are lazy — these are optional dependencies
    if provider == "openai":
        from evalkit.judges.openai import OpenAIJudge

        return OpenAIJudge(model=model or "gpt-4o-mini", api_key=api_key)
    if provider == "azure":
        from evalkit.judges.openai import AzureOpenAIJudge

        azure_key = api_key or os.environ.get("AZURE_OPENAI_API_KEY")
        return AzureOpenAIJudge(model=model or "gpt-4o-mini", api_key=azure_key)
    if provider == "anthropic":
        from evalkit.judges.anthropic import AnthropicJudge

        return AnthropicJudge(model=model or "claude-haiku-4-5-20251001", api_key=api_key)
    if provider == "ollama":
        from evalkit.judges.ollama import OllamaJudge

        return OllamaJudge(model=model or "gpt-oss")
    raise ValueError(f"Unknown judge provider: {provider!r}")


@app.command()
@click.option(
    "--input",
    "-i",
    "input_file",
    type=click.Path(exists=True),
    help="CSV or JSON file with test cases",
)
@click.option(
    "--endpoint",
    "-e",
    help="RAG API endpoint URL to query for answers",
)
@click.option(
    "--questions",
    "-q",
    type=click.Path(exists=True),
    help="Text file with questions (one per line, used with --endpoint)",
)
@click.option(
    "--suite",
    "-s",
    default=Suite.RAG_QA.value,
    show_default=True,
    type=click.Choice([s.value for s in Suite]),
    help="Evaluation suite to use",
)
@click.option(
    "--judge",
    "-j",
    default="openai",
    show_default=True,
    type=click.Choice(["openai", "azure", "anthropic", "ollama"]),
    help="LLM judge provider",
)
@click.option(
    "--model",
    "-m",
    default=None,
    help="Judge model name (default depends on provider)",
)
@click.option(
    "--api-key",
    "-k",
    "api_key",
    default=None,
    envvar=["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "AZURE_OPENAI_API_KEY"],
    help="API key (or set OPENAI_API_KEY / ANTHROPIC_API_KEY env var)",
)
@click.option(
    "--output",
    "-o",
    default=None,
    type=click.Path(),
    help="Output file for report (JSON). Prints to stdout if not set.",
)
@click.option(
    "--concurrency",
    "-c",
    default=4,
    show_default=True,
    help="Number of concurrent evaluations",
)
def run(
    input_file: str | None,
    endpoint: str | None,
    questions: str | None,
    suite: str,
    judge: str,
    model: str | None,
    api_key: str | None,
    output: str | None,
    concurrency: int,
) -> None:
    """Run an evaluation against a RAG system."""
    if input_file is None and endpoint is None:
        raise click.UsageError("Provide either --input or --endpoint.")
    if input_file is not None and endpoint is not None:
        raise click.UsageError("Provide either --input or --endpoint, not both.")

    test_cases: list[TestCase]
    if input_file is not None:
        test_cases = _load_test_cases_from_file(input_file)
    else:
        if not questions:
            raise click.UsageError("--questions is required when using --endpoint.")
        assert endpoint is not None
        test_cases = _load_test_cases_from_endpoint(endpoint, questions)

    suite_obj = BUILTIN_SUITES[Suite(suite)]()
    j = _make_judge(judge, model, api_key) if suite_obj.requires_judge else None
    engine = EvaluationEngine(judge=j, concurrency=concurrency)
    report = engine.run(test_cases, suite=suite_obj)

    report_json = report.model_dump_json(indent=2)
    if output:
        with open(output, "w", encoding="utf-8") as fh:
            fh.write(report_json)
    else:
        click.echo(report_json)

    click.echo(
        f"Score: {report.summary.average_score:.2f} | " f"Passed: {report.summary.passed}/{report.summary.total_cases}",
        err=True,
    )


def _load_test_cases_from_file(path: str) -> list[TestCase]:
    if path.lower().endswith(".json"):
        with open(path, encoding="utf-8") as fh:
            data: list[dict[str, Any]] = json.load(fh)
        return [TestCase(**row) for row in data]

    import csv

    test_cases: list[TestCase] = []
    with open(path, newline="", encoding="utf-8") as fh:
        reader = csv.DictReader(fh)
        for row in reader:
            context_raw = row.get("context", "[]")
            try:
                context: list[str] = json.loads(context_raw)
            except (json.JSONDecodeError, TypeError):
                context = [c.strip() for c in context_raw.split("|") if c.strip()]

            test_cases.append(
                TestCase(
                    question=row["question"],
                    context=context,
                    answer=row["answer"],
                    expected_answer=row.get("expected_answer") or None,
                )
            )
    return test_cases


def _load_test_cases_from_endpoint(endpoint: str, questions_file: str) -> list[TestCase]:
    import urllib.error
    import urllib.request

    with open(questions_file, encoding="utf-8") as fh:
        questions = [line.strip() for line in fh if line.strip()]

    test_cases: list[TestCase] = []
    for q in questions:
        payload = json.dumps({"question": q}).encode("utf-8")
        req = urllib.request.Request(
            endpoint,
            data=payload,
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        try:
            with urllib.request.urlopen(req, timeout=30) as resp:
                body: dict[str, Any] = json.loads(resp.read().decode("utf-8"))
        except urllib.error.URLError as exc:
            raise click.ClickException(f"Failed to call endpoint {endpoint!r}: {exc}") from exc
        except json.JSONDecodeError as exc:
            raise click.ClickException(f"Invalid JSON response from endpoint {endpoint!r}: {exc}") from exc

        context = body.get("context", [])
        if isinstance(context, str):
            context = [context]

        test_cases.append(
            TestCase(
                question=q,
                context=context,
                answer=body.get("answer", ""),
                expected_answer=body.get("expected_answer"),
            )
        )
    return test_cases


@app.command()
@click.option("--host", default="127.0.0.1", show_default=True)
@click.option("--port", "-p", default=8000, show_default=True)
@click.option(
    "--db",
    "database_url",
    default="sqlite:///./evalkit.db",
    show_default=True,
    envvar="DATABASE_URL",
    help="SQLAlchemy database URL",
)
@click.option(
    "--judge",
    "-j",
    "judge_provider",
    default=None,
    envvar="EVALKIT_JUDGE_PROVIDER",
    type=click.Choice(["openai", "azure", "anthropic", "ollama"]),
    help="Default judge provider (env: EVALKIT_JUDGE_PROVIDER, default: openai)",
)
@click.option(
    "--model",
    "-m",
    "judge_model",
    default=None,
    envvar="EVALKIT_JUDGE_MODEL",
    help="Default judge model (env: EVALKIT_JUDGE_MODEL)",
)
@click.option(
    "--suite",
    "-s",
    "suite",
    default=None,
    envvar="EVALKIT_SUITE",
    type=click.Choice([s.value for s in Suite]),
    help="Default evaluation suite (env: EVALKIT_SUITE, default: rag_qa)",
)
@click.option("--reload", is_flag=True, default=False, help="Auto-reload on code changes (dev mode)")
def serve(
    host: str,
    port: int,
    database_url: str,
    judge_provider: str | None,
    judge_model: str | None,
    suite: str | None,
    reload: bool,
) -> None:
    """Start the evalkit web dashboard."""
    try:
        import uvicorn  # noqa: F401

        from evalkit.server.app import create_app  # noqa: F401
    except ImportError:
        raise click.ClickException("Server dependencies not installed. Run: pip install evalkit-llm[server]")

    os.environ.setdefault("DATABASE_URL", database_url)
    if judge_provider:
        os.environ["EVALKIT_JUDGE_PROVIDER"] = judge_provider
    if judge_model:
        os.environ["EVALKIT_JUDGE_MODEL"] = judge_model
    if suite:
        os.environ["EVALKIT_SUITE"] = suite
    eff_provider = os.environ.get("EVALKIT_JUDGE_PROVIDER", "openai")
    eff_model = os.environ.get("EVALKIT_JUDGE_MODEL", "(provider default)")
    eff_suite = os.environ.get("EVALKIT_SUITE", Suite.RAG_QA.value)
    click.echo(f"Starting evalkit server at http://{host}:{port}")
    click.echo(f"  Judge:  {eff_provider} / {eff_model}")
    click.echo(f"  Suite:  {eff_suite}")
    click.echo(f"  DB:     {database_url}")

    uvicorn.run("evalkit.server.app:app", host=host, port=port, reload=reload)


@app.command()
@click.argument("report_file", type=click.Path(exists=True))
@click.option(
    "--format",
    "-f",
    "fmt",
    default="text",
    type=click.Choice(["text", "json"]),
    show_default=True,
)
def report(report_file: str, fmt: str) -> None:
    """Display a saved evaluation report."""
    with open(report_file, encoding="utf-8") as fh:
        raw = fh.read()

    eval_report = EvaluationReport.model_validate_json(raw)

    if fmt == "json":
        click.echo(eval_report.model_dump_json(indent=2))
        return

    eval_report.print_report()
