"""Tests for eval command and eval module.

Covers:
    - CLI surface (--help, options)
    - Batch mode (all tasks)
    - Single-task mode (--task filter)
    - Validation gate (rejects invalid task dirs)
    - Eval-output guardrails (InvalidEvalOutput on non-numeric rewards)
    - --output-dir writes results + trial artifacts
    - No tasks found exits with error
    - Infrastructure error handling
    - to_dict() output format
"""

from __future__ import annotations

import json
import math
import os
import sys
import textwrap
from pathlib import Path
from unittest import mock

import pytest
from click.testing import CliRunner
from daytona.common.errors import DaytonaError

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

_TASKS_DIR = None


def _make_task_dir(tmp_path: Path, name: str, score: float = 1.0) -> Path:
    """Create a minimal valid Harbor task directory."""
    task_dir = tmp_path / "tasks" / name
    task_dir.mkdir(parents=True, exist_ok=True)

    (task_dir / "task.toml").write_text(f'name = "{name}"\n')
    (task_dir / "instruction.md").write_text(f"# {name}\nDo the task.\n")

    tests_dir = task_dir / "tests"
    tests_dir.mkdir(exist_ok=True)
    (tests_dir / "evaluate.py").write_text(textwrap.dedent(f"""\
        def evaluate(submission):
            return {{"rewards": {{"score": {score}}}}}
    """))

    return task_dir


def _make_mock_backend(results_map: dict[str, tuple[float, dict]]):
    """Create a mock Backend that returns predefined TrialResults."""
    from clawbench.backend import TrialResult

    class MockBackend:
        def setup(self): pass
        def cleanup(self): pass

        def run_trial(self, candidate, task, *, skills_dir=None, run_id=None):
            task_name = task["task_name"]
            if task_name in results_map:
                score, side_info = results_map[task_name]
                return TrialResult(score=score, side_info=side_info)
            return TrialResult(score=0.0, side_info={"Feedback": {"Status": "no_verifier_result"}})

        async def run_trial_async(self, candidate, task, *, skills_dir=None, run_id=None):
            return self.run_trial(candidate, task, skills_dir=skills_dir, run_id=run_id)

    return MockBackend()


def _make_raising_backend(exc):
    """Create a mock Backend whose run_trial raises the given exception."""
    class RaisingBackend:
        def setup(self): pass
        def cleanup(self): pass

        def run_trial(self, candidate, task, *, skills_dir=None, run_id=None):
            raise exc

        async def run_trial_async(self, candidate, task, *, skills_dir=None, run_id=None):
            raise exc

    return RaisingBackend()


# ---------------------------------------------------------------------------
# CLI surface tests
# ---------------------------------------------------------------------------

class TestHarborEvalCLI:
    """Test CLI option surface for eval."""

    def test_help_shows_expected_options(self):
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["eval", "--help"])
        assert result.exit_code == 0
        assert "--skill" in result.output
        assert "--tasks-dir" in result.output
        assert "--task" in result.output
        assert "--output-dir" in result.output
        assert "--agent" in result.output
        assert "--model" in result.output
        assert "-e" in result.output
        assert "--verbose" in result.output

    def test_help_does_not_show_engine(self):
        """eval has no --engine option (eval only, no optimization)."""
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["eval", "--help"])
        assert "--engine" not in result.output

    def test_skill_optional_for_baseline(self):
        """eval accepts no --skill (baseline mode) — Click doesn't reject it."""
        from clawbench.cli import cli
        runner = CliRunner()
        # Invoke without --skill. Should not complain about missing --skill.
        # It will fail because default tasks/ dir doesn't exist in test env,
        # but the error should NOT be about --skill being required.
        result = runner.invoke(cli, ["eval"])
        assert "skill" not in result.output.lower() or "tasks" in result.output.lower()

    def test_tasks_dir_defaults_to_tasks(self):
        """--tasks-dir defaults to tasks/ (not required)."""
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["eval", "--help"])
        assert "tasks/" in result.output


# ---------------------------------------------------------------------------
# Core evaluation tests (mocked fitness fn)
# ---------------------------------------------------------------------------

class TestHarborEvalBatch:
    """Test batch mode — evaluates all discovered tasks."""

    def test_batch_evaluates_all_tasks(self, tmp_path):
        _make_task_dir(tmp_path, "task-a", score=1.0)
        _make_task_dir(tmp_path, "task-b", score=0.5)
        tasks_dir = tmp_path / "tasks"

        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test Skill\n")

        results_map = {
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0}}}),
            "task-b": (0.5, {"Feedback": {"Status": "completed", "Rewards": {"score": 0.5}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(
                skill_path=skill_path,
                tasks_dir=tasks_dir,
            )

        assert result.summary.total == 2
        assert result.summary.passed == 1  # task-a passes (1.0 > PASS_THRESHOLD), task-b fails (0.5)
        assert result.summary.avg_reward == 0.75
        names = [r.task_name for r in result.results]
        assert "task-a" in names
        assert "task-b" in names

    def test_batch_no_tasks_exits(self, tmp_path):
        tasks_dir = tmp_path / "empty_tasks"
        tasks_dir.mkdir()
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        from clawbench.eval import run_eval
        with pytest.raises(SystemExit, match="No tasks found"):
            run_eval(skill_path=skill_path, tasks_dir=tasks_dir)


class TestHarborEvalSingleTask:
    """Test single-task mode (--task filter)."""

    def test_single_task_filters(self, tmp_path):
        _make_task_dir(tmp_path, "task-a", score=1.0)
        _make_task_dir(tmp_path, "task-b", score=0.0)
        tasks_dir = tmp_path / "tasks"

        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test Skill\n")

        results_map = {
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(
                skill_path=skill_path,
                tasks_dir=tasks_dir,
                task_filter="task-a",
            )

        assert result.summary.total == 1
        assert result.results[0].task_name == "task-a"
        assert result.results[0].score == 1.0

    def test_single_task_not_found(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        from clawbench.eval import run_eval
        with pytest.raises(SystemExit, match="not found"):
            run_eval(
                skill_path=skill_path,
                tasks_dir=tasks_dir,
                task_filter="nonexistent-task",
            )


# ---------------------------------------------------------------------------
# Validation gate tests
# ---------------------------------------------------------------------------

class TestHarborEvalValidation:
    """Test that validation gate rejects invalid task directories."""

    def test_missing_task_toml_not_discovered(self, tmp_path):
        """A dir without task.toml is not discovered — results in no tasks found."""
        task_dir = tmp_path / "tasks" / "bad-task"
        task_dir.mkdir(parents=True)
        (task_dir / "instruction.md").write_text("# Bad\n")
        tests_dir = task_dir / "tests"
        tests_dir.mkdir()
        (tests_dir / "evaluate.py").write_text("def evaluate(s): return {'rewards': {'score': 0}}\n")

        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        from clawbench.eval import run_eval
        with pytest.raises(SystemExit, match="No tasks found"):
            run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

    def test_missing_evaluate_py_rejected(self, tmp_path):
        task_dir = tmp_path / "tasks" / "bad-task"
        task_dir.mkdir(parents=True)
        (task_dir / "task.toml").write_text('name = "bad-task"\n')
        (task_dir / "instruction.md").write_text("# Bad\n")
        # No tests/evaluate.py

        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        from clawbench.eval import run_eval
        from clawbench.task_contract import ValidationError
        with pytest.raises(ValidationError, match="missing tests/evaluate.py"):
            run_eval(skill_path=skill_path, tasks_dir=tasks_dir)


# ---------------------------------------------------------------------------
# Eval-output guardrails
# ---------------------------------------------------------------------------

class TestHarborEvalGuardrails:
    """Test eval-output guardrails (InvalidEvalOutput)."""

    def test_non_numeric_rewards_fail_fast(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (0.0, {"Feedback": {"Status": "completed", "Rewards": {"score": "not_a_number"}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            from clawbench.scoring import InvalidEvalOutput
            with pytest.raises(InvalidEvalOutput, match="must be numeric"):
                run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

    def test_non_dict_rewards_fail_fast(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (0.0, {"Feedback": {"Status": "completed", "Rewards": "not_a_dict"}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            from clawbench.scoring import InvalidEvalOutput
            with pytest.raises(InvalidEvalOutput, match="must be a dict"):
                run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

    def test_missing_score_key_fail_fast(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (0.5, {"Feedback": {"Status": "completed", "Rewards": {"precision": 0.8}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            from clawbench.scoring import InvalidEvalOutput
            with pytest.raises(InvalidEvalOutput, match="neither 'score' nor 'reward'"):
                run_eval(skill_path=skill_path, tasks_dir=tasks_dir)


# ---------------------------------------------------------------------------
# Infrastructure error handling
# ---------------------------------------------------------------------------

class TestHarborEvalErrors:
    """Test infrastructure error handling."""

    def test_trial_error_zero_score(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_raising_backend(RuntimeError("Docker crashed")),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        assert result.summary.total == 1
        assert result.results[0].score == 0.0
        assert result.results[0].status == "trial_error"
        assert "Docker crashed" in result.results[0].error

    def test_sandbox_error_nan_score(self, tmp_path):
        """DaytonaError gives NaN score and is excluded from aggregation."""
        _make_task_dir(tmp_path, "task-a")
        _make_task_dir(tmp_path, "task-b")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        # task-a succeeds, task-b hits a sandbox error
        good_backend = _make_mock_backend({
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0}}}),
        })
        # Override run_trial to raise DaytonaError for task-b
        _orig_run = good_backend.run_trial
        def _patched_run(candidate, task, *, skills_dir=None, run_id=None):
            if task["task_name"] == "task-b":
                raise DaytonaError("Failed to create sandbox: BUILD_FAILED")
            return _orig_run(candidate, task, skills_dir=skills_dir, run_id=run_id)
        good_backend.run_trial = _patched_run

        async def _patched_async(candidate, task, *, skills_dir=None, run_id=None):
            return _patched_run(candidate, task, skills_dir=skills_dir, run_id=run_id)
        good_backend.run_trial_async = _patched_async

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=good_backend,
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        # task-b should have NaN score
        sandbox_results = [r for r in result.results if r.status == "sandbox_error"]
        assert len(sandbox_results) == 1
        assert math.isnan(sandbox_results[0].score)

        # Summary should only count the valid trial (task-a)
        assert result.summary.total == 1
        assert result.summary.avg_reward == 1.0

    def test_no_verifier_result_zero_score(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (0.0, {"Feedback": {"Status": "no_verifier_result"}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        assert result.results[0].score == 0.0
        assert result.results[0].passed is False


# ---------------------------------------------------------------------------
# Output JSON tests
# ---------------------------------------------------------------------------

class TestHarborEvalOutput:
    """Test --output JSON writing and to_dict() format."""

    def test_to_dict_format(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        d = result.to_dict()
        assert "skill" in d
        assert "tasks_dir" in d
        assert "agent" in d
        assert "model" in d
        assert "results" in d
        assert "summary" in d
        assert d["summary"]["total"] == 1
        assert d["summary"]["passed"] == 1
        assert d["results"][0]["task_name"] == "task-a"
        assert d["results"][0]["score"] == 1.0
        assert d["results"][0]["passed"] is True

    def test_output_json_valid(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0, "precision": 0.9}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        output_path = tmp_path / "eval-results.json"
        output_path.write_text(json.dumps(result.to_dict(), indent=2))

        loaded = json.loads(output_path.read_text())
        assert loaded["summary"]["avg_reward"] == 1.0
        assert loaded["results"][0]["score"] == 1.0

    def test_error_result_includes_error_in_dict(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_raising_backend(RuntimeError("container timeout")),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        d = result.to_dict()
        assert d["results"][0]["error"] == "container timeout"
        assert d["results"][0]["score"] == 0.0

    def test_to_dict_includes_status(self, tmp_path):
        """to_dict() serializes the status field for each result."""
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        d = result.to_dict()
        assert d["results"][0]["status"] == "completed"

    def test_to_dict_status_sandbox_error(self, tmp_path):
        """to_dict() serializes sandbox_error status for DaytonaError trials."""
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_raising_backend(DaytonaError("sandbox failed")),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        d = result.to_dict()
        assert d["results"][0]["status"] == "sandbox_error"

    def test_sandbox_error_n_errors_in_task_summary(self, tmp_path):
        """With n_repeats, sandbox errors produce n_errors in task_summary."""
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        # Alternate: first 2 calls succeed, third raises DaytonaError
        call_count = {"n": 0}
        good_backend = _make_mock_backend({
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0}}}),
        })
        _orig_run = good_backend.run_trial

        def _patched_run(candidate, task, *, skills_dir=None, run_id=None):
            call_count["n"] += 1
            if call_count["n"] == 3:
                raise DaytonaError("transient sandbox failure")
            return _orig_run(candidate, task, skills_dir=skills_dir, run_id=run_id)

        good_backend.run_trial = _patched_run

        async def _patched_async(candidate, task, *, skills_dir=None, run_id=None):
            return _patched_run(candidate, task, skills_dir=skills_dir, run_id=run_id)
        good_backend.run_trial_async = _patched_async

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=good_backend,
        ):
            from clawbench.eval import run_eval
            result = run_eval(
                skill_path=skill_path, tasks_dir=tasks_dir, n_repeats=3
            )

        # 3 runs total: 2 valid, 1 sandbox error
        assert len(result.results) == 3
        valid = [r for r in result.results if not math.isnan(r.score)]
        errors = [r for r in result.results if math.isnan(r.score)]
        assert len(valid) == 2
        assert len(errors) == 1

        # Summary excludes NaN
        assert result.summary.total == 2

        # task_summary should have n_errors
        assert len(result.task_summary) == 1
        ts = result.task_summary[0]
        assert ts["task_name"] == "task-a"
        assert ts["n_runs"] == 2
        assert ts["n_errors"] == 1


# ---------------------------------------------------------------------------
# Pass rate / score edge cases
# ---------------------------------------------------------------------------

class TestHarborEvalScoring:
    """Test pass rate and score edge cases."""

    def test_zero_score_is_not_passed(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (0.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 0.0}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        assert result.results[0].passed is False
        assert result.summary.passed == 0
        assert result.summary.pass_rate == 0.0

    def test_mixed_results_summary(self, tmp_path):
        _make_task_dir(tmp_path, "task-a")
        _make_task_dir(tmp_path, "task-b")
        _make_task_dir(tmp_path, "task-c")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (1.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 1.0}}}),
            "task-b": (0.0, {"Feedback": {"Status": "completed", "Rewards": {"score": 0.0}}}),
            "task-c": (0.5, {"Feedback": {"Status": "completed", "Rewards": {"score": 0.5}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        assert result.summary.total == 3
        assert result.summary.passed == 1  # only 1.0 > PASS_THRESHOLD (0.8)
        assert result.summary.avg_reward == 0.5
        assert result.summary.pass_rate == round(1/3, 3)

    def test_reward_fallback_to_reward_key(self, tmp_path):
        """Score extraction falls back from rewards['score'] to rewards['reward']."""
        _make_task_dir(tmp_path, "task-a")
        tasks_dir = tmp_path / "tasks"
        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")

        results_map = {
            "task-a": (0.7, {"Feedback": {"Status": "completed", "Rewards": {"reward": 0.7}}}),
        }

        with mock.patch(
            "clawbench.eval.BenchflowBackend",
            return_value=_make_mock_backend(results_map),
        ):
            from clawbench.eval import run_eval
            result = run_eval(skill_path=skill_path, tasks_dir=tasks_dir)

        assert result.results[0].score == 0.7


# ---------------------------------------------------------------------------
# Task tag filtering tests
# ---------------------------------------------------------------------------

class TestTaskTagFiltering:
    """Test discover_tasks() OR-of-AND tag filtering."""

    def _make_tagged_task(self, tmp_path: Path, name: str, tags: list[str]) -> Path:
        task_dir = tmp_path / "tasks" / name
        task_dir.mkdir(parents=True, exist_ok=True)
        tag_str = ", ".join(f'"{t}"' for t in tags)
        (task_dir / "task.toml").write_text(
            f'name = "{name}"\n\n[metadata]\ntags = [{tag_str}]\n'
        )
        (task_dir / "instruction.md").write_text(f"# {name}\n")
        tests_dir = task_dir / "tests"
        tests_dir.mkdir(exist_ok=True)
        (tests_dir / "evaluate.py").write_text("def evaluate(s): return {'rewards': {'score': 1.0}}\n")
        return task_dir

    def test_no_tags_returns_all(self, tmp_path):
        from clawbench.task_utils import discover_tasks
        self._make_tagged_task(tmp_path, "a", ["gmail", "fast"])
        self._make_tagged_task(tmp_path, "b", ["slack"])
        tasks = discover_tasks(tmp_path / "tasks", task_tags=None)
        assert len(tasks) == 2

    def test_single_tag_filters(self, tmp_path):
        from clawbench.task_utils import discover_tasks
        self._make_tagged_task(tmp_path, "a", ["gmail", "fast"])
        self._make_tagged_task(tmp_path, "b", ["slack"])
        tasks = discover_tasks(tmp_path / "tasks", task_tags=["gmail"])
        assert [t["task_name"] for t in tasks] == ["a"]

    def test_or_across_flags(self, tmp_path):
        """Multiple --task-tag flags → OR: matches tasks with either tag."""
        from clawbench.task_utils import discover_tasks
        self._make_tagged_task(tmp_path, "a", ["gmail", "fast"])
        self._make_tagged_task(tmp_path, "b", ["slack"])
        self._make_tagged_task(tmp_path, "c", ["gcal"])
        # --task-tag gmail --task-tag slack → gmail OR slack
        tasks = discover_tasks(tmp_path / "tasks", task_tags=["gmail", "slack"])
        assert [t["task_name"] for t in tasks] == ["a", "b"]

    def test_and_within_comma(self, tmp_path):
        """Comma-separated tags within one flag → AND: task must have all."""
        from clawbench.task_utils import discover_tasks
        self._make_tagged_task(tmp_path, "a", ["gmail", "fast"])
        self._make_tagged_task(tmp_path, "b", ["gmail"])
        # --task-tag "gmail,fast" → gmail AND fast
        tasks = discover_tasks(tmp_path / "tasks", task_tags=["gmail,fast"])
        assert [t["task_name"] for t in tasks] == ["a"]

    def test_mixed_or_of_and(self, tmp_path):
        """Mixed: --task-tag "gmail,fast" --task-tag gcal → (gmail AND fast) OR gcal."""
        from clawbench.task_utils import discover_tasks
        self._make_tagged_task(tmp_path, "a", ["gmail", "fast"])
        self._make_tagged_task(tmp_path, "b", ["gmail"])
        self._make_tagged_task(tmp_path, "c", ["gcal"])
        tasks = discover_tasks(tmp_path / "tasks", task_tags=["gmail,fast", "gcal"])
        assert [t["task_name"] for t in tasks] == ["a", "c"]

    def test_no_matches(self, tmp_path):
        from clawbench.task_utils import discover_tasks
        self._make_tagged_task(tmp_path, "a", ["gmail"])
        tasks = discover_tasks(tmp_path / "tasks", task_tags=["nonexistent"])
        assert tasks == []

    def test_untagged_task_excluded(self, tmp_path):
        from clawbench.task_utils import discover_tasks
        self._make_tagged_task(tmp_path, "a", [])
        tasks = discover_tasks(tmp_path / "tasks", task_tags=["gmail"])
        assert tasks == []


# ---------------------------------------------------------------------------
# Skills directory injection tests
# ---------------------------------------------------------------------------

# ---------------------------------------------------------------------------
# Async fitness fn trajectory store test
# ---------------------------------------------------------------------------

class TestAsyncFitnessTrajectoryStore:
    """Verify that create_fitness_fn exposes an async_fn with trajectory support."""

    def test_async_fn_attribute_exists(self, tmp_path):
        """The sync fitness fn should have an .async_fn attribute for Daytona concurrency."""
        from clawbench.optimize import _create_fitness_fn as create_fitness_fn

        skill_path = tmp_path / "SKILL.md"
        skill_path.write_text("# Test\n")
        tasks_dir = tmp_path / "tasks"
        tasks_dir.mkdir()

        fitness_fn = create_fitness_fn(
            agent_name="oracle",
            model_name="test/model",
            environment_type="docker",
            trials_dir=tmp_path / "trials",
        )
        assert hasattr(fitness_fn, "async_fn"), \
            "fitness fn must expose .async_fn for async (Daytona) callers"
        assert callable(fitness_fn.async_fn)

    def test_async_fn_accepts_trajectory_store_via_closure(self, tmp_path):
        """The async fn should use the trajectory_store from the closure."""
        import inspect
        from clawbench.optimize import _create_fitness_fn as create_fitness_fn

        trajectory_store: list[dict] = []
        fitness_fn = create_fitness_fn(
            agent_name="oracle",
            model_name="test/model",
            environment_type="docker",
            trials_dir=tmp_path / "trials",
            trajectory_store=trajectory_store,
        )
        # Verify the async fn is a coroutine function
        assert inspect.iscoroutinefunction(fitness_fn.async_fn), \
            "async_fn should be a coroutine function"


# ---------------------------------------------------------------------------
# CLI plumbing tests for new flags
# ---------------------------------------------------------------------------

class TestNewCLIFlags:
    """Test that --skills-dir and --task-tag are wired through."""

    def test_help_shows_skills_dir(self):
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["eval", "--help"])
        assert "--skills-dir" in result.output

    def test_help_shows_task_tag(self):
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["eval", "--help"])
        assert "--task-tag" in result.output

    def test_optimize_help_shows_skills_dir(self):
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["optimize", "--help"])
        assert "--skills-dir" in result.output

    def test_optimize_help_shows_task_tag(self):
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["optimize", "--help"])
        assert "--task-tag" in result.output

    def test_help_shows_output_dir(self):
        from clawbench.cli import cli
        runner = CliRunner()
        result = runner.invoke(cli, ["eval", "--help"])
        assert "--output-dir" in result.output


