"""Evaluation orchestrator — measure a skill against tasks without optimization.

Called by: cli.py
Calls: executor.execute_trials(), export.export_eval_bundle(), task_utils, task_contract

Runs the same trial execution path as optimize but without the
proposer/improvement loop. Use for baselining a skill before optimization
or verifying after.

Usage:
    clawbench eval \
        --skill skills/gws-gmail/SKILL.md \
        --tasks-dir tasks
"""

from __future__ import annotations

import math
import random
import threading
import time
from collections import defaultdict
from dataclasses import dataclass, field
from pathlib import Path

from clawbench._env import PASS_THRESHOLD
from clawbench._paths import _PROJECT_ROOT
from clawbench.backend import BenchflowBackend
from clawbench.executor import execute_trials
from clawbench.export import export_eval_bundle, generate_run_id
from clawbench.task_contract import validate_tasks
from clawbench.task_utils import discover_tasks


@dataclass
class EvalResult:
    """Per-task evaluation result."""
    task_name: str
    score: float
    passed: bool
    duration_sec: float
    rewards: dict = field(default_factory=dict)
    status: str = "completed"
    error: str | None = None


@dataclass
class EvalSummary:
    """Aggregate evaluation summary."""
    total: int
    passed: int
    avg_reward: float
    pass_rate: float


@dataclass
class EvalBatchResult:
    """Full result of an eval run."""
    skill: str
    tasks_dir: str
    agent: str
    model: str
    results: list[EvalResult]
    summary: EvalSummary
    task_summary: list[dict] = field(default_factory=list)
    elapsed_sec: float = 0.0

    def to_dict(self) -> dict:
        d = {
            "skill": self.skill,
            "tasks_dir": self.tasks_dir,
            "agent": self.agent,
            "model": self.model,
            "elapsed_sec": round(self.elapsed_sec, 1),
            "results": [
                {
                    "task_name": r.task_name,
                    "score": r.score,
                    "passed": r.passed,
                    "duration_sec": r.duration_sec,
                    "status": r.status,
                    **({"rewards": r.rewards} if r.rewards else {}),
                    **({"error": r.error} if r.error else {}),
                }
                for r in self.results
            ],
            "summary": {
                "total": self.summary.total,
                "passed": self.summary.passed,
                "avg_reward": self.summary.avg_reward,
                "pass_rate": self.summary.pass_rate,
            },
        }
        if self.task_summary:
            d["task_summary"] = self.task_summary
        return d



def run_eval(
    skill_path: Path | None,
    tasks_dir: Path,
    tasks: list[dict] | None = None,
    task_filter: str | None = None,
    task_tags: list[str] | None = None,
    agent_name: str = "claude-code",
    agent_import_path: str | None = None,
    model_name: str = "anthropic/claude-sonnet-4-6",
    environment_type: str = "docker",
    n_workers: int = 1,
    n_repeats: int = 1,
    verbose: bool = False,
    skip_verified: bool = False,
    update_cache: bool = False,
    skills_dir: Path | None = None,
    output_dir: Path | None = None,
) -> EvalBatchResult:
    """Evaluate a skill against tasks.

    Args:
        skill_path: Path to SKILL.md file (optional — omit for baseline).
        tasks_dir: Tasks directory.
        tasks: Pre-filtered task list (from resolve_tasks). When provided,
            task_filter and task_tags are ignored.
        task_filter: Optional single task name to evaluate (legacy, prefer tasks).
        task_tags: Optional tag list for AND-filtering tasks (legacy, prefer tasks).
        agent_name: Agent name for trials.
        agent_import_path: Import path for unregistered agents.
        model_name: Model for agent execution.
        environment_type: "docker" or "daytona".
        n_repeats: Run each task N times (for variance estimation).
        verbose: Print per-task progress.
        skills_dir: Optional skills directory for agent-native discovery.
        output_dir: When set, write results.json + trial artifacts into this directory.

    Returns:
        EvalBatchResult with per-task scores and aggregate summary.

    Raises:
        InvalidEvalOutput: On eval reward contract violations.
        ValidationError: On task directory contract violations.
        SystemExit: On no tasks found.
    """
    skill_path = Path(skill_path) if skill_path else None
    tasks_dir = Path(tasks_dir)

    # 1. Resolve tasks
    if tasks is not None:
        task_paths = tasks
    else:
        task_paths = discover_tasks(tasks_dir, task_tags=task_tags)
        if not task_paths:
            raise SystemExit(
                f"No tasks found in {tasks_dir}."
                f" Check that the path contains task subdirectories (each with a task.toml)."
                f" Run 'clawbench check --tasks-dir <path>' to validate."
            )

        if task_filter:
            matched = [t for t in task_paths if t["task_name"] == task_filter]
            if not matched:
                from clawbench.task_utils import _did_you_mean
                available = sorted(t["task_name"] for t in task_paths)
                suggestion = _did_you_mean(task_filter, available)
                raise SystemExit(
                    f"Task '{task_filter}' not found in {tasks_dir}.{suggestion}\n"
                    f"Available ({len(available)}): {', '.join(available)}"
                )
            task_paths = matched

    # 2. Filter verified tasks
    cache: dict = {}
    if skip_verified or update_cache:
        from clawbench.score_cache import load_cache, is_verified
        cache = load_cache(tasks_dir)

    if skip_verified and cache and not task_filter:
        skipped = []
        filtered = []
        for t in task_paths:
            task_dir = Path(t["task_path"])
            if is_verified(t["task_name"], task_dir, cache):
                skipped.append(t["task_name"])
            else:
                filtered.append(t)
        if skipped:
            if verbose:
                for name in skipped:
                    print(f"  Skipping: {name}  (verified, oracle=1.0)", flush=True)
            task_paths = filtered

    # 3. Validation gate
    validate_tasks(task_paths, check_callable=False)

    # 4. Generate run ID for trial tagging
    run_id = generate_run_id(skill_path, agent_name, model_name, environment_type)

    # 5. Create backend
    backend_impl = BenchflowBackend(
        agent_name=agent_name,
        model_name=model_name,
        environment_type=environment_type,
        agent_import_path=agent_import_path,
        keep_trials=True,
    )
    backend_impl.setup()

    # 6. Load skill (empty string when --skill omitted → baseline run)
    skill_content = skill_path.read_text() if skill_path else ""
    candidate = {"skills": skill_content}

    # 7. Expand tasks × repeats and shuffle for balanced concurrency
    if n_repeats > 1:
        work_items = [
            {**task, "_repeat": r}
            for task in task_paths
            for r in range(n_repeats)
        ]
        random.shuffle(work_items)
    else:
        work_items = task_paths

    # 8. Execute trials via executor
    eval_t0 = time.time()
    raw_results = execute_trials(
        backend=backend_impl,
        candidate=candidate,
        work_items=work_items,
        environment_type=environment_type,
        n_workers=n_workers,
        n_repeats=n_repeats,
        verbose=verbose,
        skills_dir=skills_dir,
        run_id=run_id,
    )

    # 9. Convert raw result dicts → EvalResult + update cache
    cache_lock = threading.Lock() if n_workers > 1 and update_cache else None
    results: list[EvalResult] = []
    for r in raw_results:
        result = EvalResult(
            task_name=r["task_name"],
            score=r["score"],
            passed=r["passed"],
            duration_sec=r["duration_sec"],
            rewards=r["rewards"],
            status=r["status"],
            error=r["error"],
        )
        results.append(result)

        if update_cache and result.status == "completed":
            from clawbench.score_cache import record_result, save_cache
            # Find the original task dict to get task_path
            task_dict = next(
                (t for t in task_paths if t["task_name"] == result.task_name), None
            )
            if task_dict:
                if cache_lock:
                    with cache_lock:
                        record_result(result.task_name, Path(task_dict["task_path"]), result.score, cache)
                        save_cache(tasks_dir, cache)
                else:
                    record_result(result.task_name, Path(task_dict["task_path"]), result.score, cache)
                    save_cache(tasks_dir, cache)

    backend_impl.cleanup()

    # 10. Aggregate summary (exclude NaN sandbox_error trials)
    valid_results = [r for r in results if not math.isnan(r.score)]
    total = len(valid_results)
    passed = sum(1 for r in valid_results if r.score > PASS_THRESHOLD)
    avg_reward = sum(r.score for r in valid_results) / total if total else 0.0
    pass_rate = passed / total if total else 0.0

    summary = EvalSummary(
        total=total,
        passed=passed,
        avg_reward=round(avg_reward, 3),
        pass_rate=round(pass_rate, 3),
    )

    # 11. Per-task aggregation (useful when n_repeats > 1)
    task_summary: list[dict] = []
    if n_repeats > 1:
        scores_by_task: dict[str, list[float]] = defaultdict(list)
        errors_by_task: dict[str, int] = defaultdict(int)
        for r in results:
            if not math.isnan(r.score):
                scores_by_task[r.task_name].append(r.score)
            else:
                errors_by_task[r.task_name] += 1

        for name in sorted(scores_by_task):
            scores = scores_by_task[name]
            n = len(scores)
            mean = sum(scores) / n
            variance = sum((s - mean) ** 2 for s in scores) / n if n > 1 else 0.0
            std = math.sqrt(variance)
            task_pass_rate = sum(1 for s in scores if s > PASS_THRESHOLD) / n
            entry: dict = {
                "task_name": name,
                "n_runs": n,
                "mean_score": round(mean, 3),
                "std_score": round(std, 3),
                "min_score": round(min(scores), 3),
                "max_score": round(max(scores), 3),
                "pass_rate": round(task_pass_rate, 3),
                "scores": [round(s, 3) for s in scores],
            }
            n_err = errors_by_task.get(name, 0)
            if n_err:
                entry["n_errors"] = n_err
            task_summary.append(entry)

    elapsed_sec = time.time() - eval_t0

    eval_result = EvalBatchResult(
        skill=str(skill_path) if skill_path else "(baseline)",
        tasks_dir=str(tasks_dir),
        agent=agent_name,
        model=model_name,
        results=results,
        summary=summary,
        task_summary=task_summary,
        elapsed_sec=elapsed_sec,
    )

    # 12. Export bundle
    if output_dir:
        trials_dir_resolved = _PROJECT_ROOT / ".local" / "trials"
        export_eval_bundle(Path(output_dir), eval_result.to_dict(), trials_dir_resolved, run_id)

    return eval_result
