from __future__ import annotations

import asyncio
import contextlib
import difflib
import json
import logging
import re
import time as _time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Awaitable, Callable
import venv

from epochler.agents import coder
from epochler.contract.engine import mark_phase_completed, sync_phase_states
from epochler.contract.models import (
    ActivityType,
    AgentActivity,
    ExperimentJournal,
    ExperimentResult,
    ExperimentStatus,
    FrozenContract,
    HardwareProfile,
    PhaseArtifact,
    PhaseExecutionStatus,
    PhaseName,
)
from epochler.config import get_settings
from epochler.runner.artifacts import materialize_artifact, write_phase_payload
from epochler.runner.data_acquisition import check_dataset_exists
from epochler.runner.dependency import install_missing, is_stdlib
from epochler.runner.errors import classify_error, select_model
from epochler.runner.hardware import detect_hardware
from epochler.runner.runtime_helper import get_runtime_helper_code
from epochler.runner.sandbox import RunResult, run_script
from epochler.runner.watchdog import AgentWatchdog
from epochler.storage import store

logger = logging.getLogger(__name__)

BroadcastFn = Callable[[dict[str, Any]], Awaitable[None]]


def _code_diff(old: str, new: str) -> str:
    old_lines = old.splitlines(keepends=True)
    new_lines = new.splitlines(keepends=True)
    diff = difflib.unified_diff(old_lines, new_lines, fromfile="before", tofile="after", n=3)
    return "".join(diff)


def _utcnow() -> datetime:
    return datetime.now(timezone.utc)


_KNOWN_ISSUES: list[tuple[re.Pattern, str]] = [
    (
        re.compile(r"No module named '_lzma'", re.IGNORECASE),
        "HINT: The host Python was built without lzma support. "
        "Avoid imports that trigger `import lzma` in their module tree (e.g. "
        "`torchvision.datasets.utils`). Use alternative approaches that don't "
        "depend on lzma — for example, import only what you need from "
        "torchvision (e.g. `from torchvision import transforms`) or use a "
        "different data loading strategy.\n",
    ),
    (
        re.compile(r"No module named '_bz2'", re.IGNORECASE),
        "HINT: The host Python was built without bz2 support. "
        "Avoid code paths that trigger `import bz2`. Use alternative "
        "compression or data loading methods.\n",
    ),
    (
        re.compile(r"No module named '_sqlite3'", re.IGNORECASE),
        "HINT: The host Python was built without sqlite3 support. "
        "Use file-based storage or alternative databases instead.\n",
    ),
    (
        re.compile(r"No matching distribution found for", re.IGNORECASE),
        "HINT: A package name could not be found on PyPI. Double-check that "
        "you are not trying to pip-install a standard library module (e.g. "
        "struct, ssl, urllib, gzip, http, socket are all stdlib). Only "
        "third-party packages need to be imported for auto-installation.\n",
    ),
]


def _diagnose_known_issue(stderr: str) -> str:
    for pattern, hint in _KNOWN_ISSUES:
        if pattern.search(stderr):
            return hint
    return ""


class ExperimentRunner:
    """Runs the autonomous experiment loop with explicit ordered phases."""

    def __init__(
        self,
        contract: FrozenContract,
        broadcast: BroadcastFn,
        check_paused: Callable[[], Awaitable[bool]],
        workspace_root: str = "",
    ) -> None:
        self.contract = sync_phase_states(contract)
        self._broadcast = broadcast
        self._check_paused = check_paused
        self._hardware: HardwareProfile | None = None
        self._cancelled = False
        self._watchdog: AgentWatchdog | None = None
        self._workspace_root = workspace_root

    def cancel(self) -> None:
        self._cancelled = True

    async def _emit(
        self,
        activity_type: ActivityType,
        summary: str,
        detail: str = "",
        experiment_id: str | None = None,
        phase_name: str | None = None,
    ) -> None:
        activity = AgentActivity(
            activity_type=activity_type,
            summary=summary,
            detail=detail,
            experiment_id=experiment_id,
            phase_name=phase_name,
        )
        store.append_activity(self.contract.id, activity)
        await self._broadcast({
            "type": "activity",
            "data": activity.model_dump(mode="json"),
        })

    async def _emit_chat_progress(self, text: str, phase_name: str | None = None) -> None:
        await self._broadcast({
            "type": "chat_progress",
            "contract_id": self.contract.id,
            "text": text,
            "phase_name": phase_name,
        })

    _TRANSITIVE_DEPS: dict[str, list[str]] = {
        "sklearn": ["pandas", "scipy"],
        "lightgbm": ["numpy", "scipy"],
        "xgboost": ["numpy", "scipy"],
        "torch": ["numpy"],
        "tensorflow": ["numpy"],
        "torchvision": ["torch"],
    }

    def _extract_imports(self, code: str) -> list[str]:
        packages: set[str] = set()
        for line in code.splitlines():
            stripped = line.strip()
            if stripped.startswith("import "):
                parts = stripped.split()[1].split(",")
                for package in parts:
                    packages.add(package.strip().split(".")[0])
            elif stripped.startswith("from "):
                packages.add(stripped.split()[1].split(".")[0])
        direct = {p for p in packages if p and not is_stdlib(p)}
        expanded = set(direct)
        for pkg in direct:
            for dep in self._TRANSITIVE_DEPS.get(pkg, []):
                expanded.add(dep)
        return list(expanded)

    async def _ensure_isolated_python(self, run_root: Path) -> str:
        venv_dir = run_root / ".venv"
        python_path = venv_dir / "bin" / "python"
        if not python_path.exists():
            def _create_venv() -> None:
                builder = venv.EnvBuilder(with_pip=True, clear=False, upgrade=False)
                builder.create(venv_dir)
            await asyncio.to_thread(_create_venv)
        return str(python_path)

    async def _install_dependencies_for_code(
        self,
        code: str,
        phase_name: str,
        *,
        python_executable: str,
        lock_path: Path,
    ) -> bool:
        """Install dependencies for generated code. Returns True on success, False if any failed."""
        needed = self._extract_imports(code)
        if not needed:
            lock_path.parent.mkdir(parents=True, exist_ok=True)
            lock_path.write_text("", encoding="utf-8")
            return True
        await self._emit(
            ActivityType.THINKING,
            f"Installing dependencies for {phase_name}...",
            detail=", ".join(needed),
            phase_name=phase_name,
        )
        try:
            results = await asyncio.to_thread(
                install_missing,
                needed,
                python_executable=python_executable,
                lock_path=lock_path,
            )
            failed = {pkg: result for pkg, result in results.items() if result.startswith("error")}
            if failed:
                await self._emit(
                    ActivityType.ERROR,
                    f"Dependency install failed for {phase_name}: {failed}",
                    phase_name=phase_name,
                )
                return False
            return True
        except Exception as exc:
            logger.exception("Dependency install failed")
            await self._emit(ActivityType.ERROR, f"Dependency install error: {exc}", phase_name=phase_name)
            return False

    def _workspace(self) -> Path:
        return store.workspace_path(self.contract.id, self._workspace_root or None)

    def _phase_root(self, phase_name: str) -> Path:
        path = self._workspace() / "runs" / phase_name
        path.mkdir(parents=True, exist_ok=True)
        return path

    def _run_root(self, phase_name: str, experiment_id: str | None) -> Path:
        run_id = experiment_id or "latest"
        path = self._phase_root(phase_name) / run_id
        path.mkdir(parents=True, exist_ok=True)
        return path

    def _write_runtime_context(self, workspace: Path) -> tuple[Path, Path]:
        contract_path = workspace / "contract.json"
        hardware_path = workspace / "hardware.json"
        runtime_path = workspace / "epochler_runtime.py"
        contract_path.write_text(self.contract.model_dump_json(indent=2), encoding="utf-8")
        hardware_path.write_text(self._hardware.model_dump_json(indent=2), encoding="utf-8")
        runtime_path.write_text(get_runtime_helper_code(), encoding="utf-8")
        return contract_path, hardware_path

    def _default_phase_code(self, phase_name: str) -> str:
        return (
            f"import sys\n"
            f"print('ERROR: No script generated for phase {phase_name}. "
            f"The coder agent must generate this script.', file=sys.stderr)\n"
            f"sys.exit(1)\n"
        )

    def _get_phase_code(self, phase_name: str) -> str:
        existing = store.load_phase_script(self.contract.id, phase_name)
        if existing:
            return existing
        return self._default_phase_code(phase_name)

    def _parse_json_payload(self, result: RunResult) -> dict[str, Any]:
        for line in reversed(result.stdout.strip().splitlines()):
            line = line.strip()
            if line.startswith("{"):
                try:
                    return json.loads(line)
                except json.JSONDecodeError:
                    continue
        if result.exit_code != 0:
            return {"_error": True, "_exit_code": result.exit_code, "_stderr": result.stderr[-500:]}
        return {}

    def _validate_phase_payload(self, phase_name: str, payload: dict[str, Any], result: RunResult) -> dict[str, Any]:
        if not isinstance(payload, dict):
            logger.warning("%s phase did not return a JSON object, treating as error", phase_name)
            return {"_error": True, "_exit_code": result.exit_code, "_stderr": "No JSON payload in stdout"}
        if result.exit_code != 0:
            return payload
        if phase_name in {PhaseName.TRAINING.value, PhaseName.EVALUATION.value}:
            metrics = payload.get("metrics")
            if not isinstance(metrics, dict):
                logger.warning("%s phase exited 0 but has no 'metrics' dict — synthesizing empty metrics", phase_name)
                payload.setdefault("metrics", {})
        if "artifacts" in payload and not isinstance(payload.get("artifacts"), list):
            logger.warning("%s phase 'artifacts' is not a list — dropping it", phase_name)
            payload.pop("artifacts", None)
        return payload

    def _normalize_artifacts(
        self,
        phase_name: str,
        payload: dict[str, Any],
        output_dir: Path,
        input_artifacts: list[PhaseArtifact] | None = None,
    ) -> list[PhaseArtifact]:
        artifact_payload = payload.get("artifacts") or payload.get("artifact_refs") or []
        artifacts: list[PhaseArtifact] = []
        for artifact in artifact_payload:
            if isinstance(artifact, str):
                artifact = {"name": Path(artifact).stem, "path": artifact, "artifact_type": phase_name}
            if not isinstance(artifact, dict):
                logger.warning("Skipping non-dict artifact in %s: %r", phase_name, artifact)
                continue
            raw_path = artifact.get("path", str(output_dir))
            if not Path(raw_path).is_absolute():
                raw_path = str(output_dir / raw_path)
            metadata = artifact.get("metadata", {})
            artifacts.append(
                PhaseArtifact(
                    name=artifact.get("name", phase_name),
                    path=raw_path,
                    artifact_type=artifact.get("artifact_type", phase_name),
                    fingerprint=artifact.get("fingerprint", ""),
                    metadata=metadata if isinstance(metadata, dict) else {},
                )
            )
        if not artifacts:
            artifacts = [
            PhaseArtifact(
                name=f"{phase_name}_artifact",
                path=str(output_dir),
                artifact_type=phase_name,
                metadata={
                    "payload": payload,
                    "input_artifacts": [artifact.model_dump(mode="json") for artifact in (input_artifacts or [])],
                },
            )
        ]
        return [
            materialize_artifact(self.contract.id, phase_name, artifact, output_dir)
            for artifact in artifacts
        ]

    def _runtime_env(
        self,
        phase_name: str,
        contract_path: Path,
        hardware_path: Path,
        output_dir: Path,
        input_artifacts_path: Path | None = None,
    ) -> dict[str, str]:
        cpu_count = max(int(self._hardware.cpu_count), 1)
        env = {
            "EPOCHLER_CONTRACT_PATH": str(contract_path),
            "EPOCHLER_HARDWARE_PATH": str(hardware_path),
            "EPOCHLER_OUTPUT_DIR": str(output_dir),
            "OMP_NUM_THREADS": str(cpu_count),
            "MKL_NUM_THREADS": str(cpu_count),
            "NUMEXPR_NUM_THREADS": str(cpu_count),
            "PYTHONUNBUFFERED": "1",
            "PYTHONNOUSERSITE": "1",
            "PYTORCH_ENABLE_MPS_FALLBACK": "1",
        }
        if input_artifacts_path:
            env["EPOCHLER_INPUT_ARTIFACTS_PATH"] = str(input_artifacts_path)
        return env

    async def _stream_output(
        self,
        phase_name: str,
        experiment_id: str | None,
        stream: str,
        line: str,
    ) -> None:
        await self._broadcast({
            "type": "training_output",
            "contract_id": self.contract.id,
            "experiment_id": experiment_id,
            "stream": stream,
            "line": f"[{phase_name}] {line}",
        })

    async def _run_phase_script(
        self,
        phase_name: str,
        code: str,
        *,
        experiment_id: str | None = None,
        input_artifacts: list[PhaseArtifact] | None = None,
    ) -> tuple[RunResult, dict[str, Any], list[PhaseArtifact]]:
        from epochler.runner.sandbox import validate_script_safety

        watchdog = self._watchdog
        workspace = self._workspace()

        try:
            validate_script_safety(code, workspace)
        except ValueError as e:
            logger.error("Script safety check failed: %s", e)
            return (
                RunResult(exit_code=-1, stderr=f"BLOCKED: {e}"),
                {},
                [],
            )

        contract_path, hardware_path = self._write_runtime_context(workspace)
        phase_root = self._phase_root(phase_name)
        run_root = self._run_root(phase_name, experiment_id)

        async with watchdog.suspend(f"venv_setup:{phase_name}") if watchdog else contextlib.nullcontext():
            python_executable = await self._ensure_isolated_python(phase_root)

        input_path: Path | None = None
        if input_artifacts is not None:
            input_path = run_root / "input_artifacts.json"
            input_path.write_text(
                json.dumps([artifact.model_dump(mode="json") for artifact in input_artifacts], indent=2),
                encoding="utf-8",
            )

        script_path = store.save_phase_script(self.contract.id, phase_name, code)

        async with watchdog.suspend(f"dep_install:{phase_name}") if watchdog else contextlib.nullcontext():
            deps_ok = await self._install_dependencies_for_code(
                code,
                phase_name,
                python_executable=python_executable,
                lock_path=phase_root / "requirements.lock",
            )

        output_dir = run_root / "outputs"
        output_dir.mkdir(parents=True, exist_ok=True)

        if not deps_ok:
            fail_msg = f"Aborting {phase_name}: dependency installation failed"
            result = RunResult(exit_code=1, stderr=fail_msg)
            payload = {"_error": True, "_exit_code": 1, "_stderr": fail_msg}
            write_phase_payload(output_dir, payload)
            artifacts = self._normalize_artifacts(phase_name, payload, output_dir, input_artifacts)
            return result, payload, artifacts

        env = self._runtime_env(phase_name, contract_path, hardware_path, output_dir, input_path)
        env["EPOCHLER_RUN_ROOT"] = str(run_root)

        async with watchdog.suspend(f"script_exec:{phase_name}") if watchdog else contextlib.nullcontext():
            result = await run_script(
                script_path,
                workspace,
                timeout_seconds=(self.contract.budget_minutes or 10) * 60,
                env_override=env,
                python_executable=python_executable,
                memory_limit_mb=max(int(self._hardware.ram_total_gb * 1024 * 0.8), 512),
                on_stdout=lambda line: self._stream_output(phase_name, experiment_id, "stdout", line),
                on_stderr=lambda line: self._stream_output(phase_name, experiment_id, "stderr", line),
            )
        payload = self._validate_phase_payload(phase_name, self._parse_json_payload(result), result)
        write_phase_payload(output_dir, payload)
        artifacts = self._normalize_artifacts(phase_name, payload, output_dir, input_artifacts)
        return result, payload, artifacts

    async def _run_non_iterative_phase(
        self,
        phase_name: str,
        journal: ExperimentJournal,
        *,
        input_artifacts: list[PhaseArtifact] | None = None,
        ask_coder: bool = False,
        max_retries: int = 2,
    ) -> tuple[list[PhaseArtifact], bool]:
        """Run a non-iterative phase with automatic retry on failure.

        Returns (artifacts, success). On failure after retries, success is False
        and the caller should halt the pipeline.
        """
        phase = self.contract.phases[phase_name]
        if phase.status == PhaseExecutionStatus.COMPLETED and not phase.invalidated and phase.artifact_refs:
            return phase.artifact_refs, True

        current_code = self._get_phase_code(phase_name)
        last_stderr = ""
        last_exit_code = 0
        settings = get_settings()

        for attempt in range(1, max_retries + 2):  # +2: 1 initial + max_retries retries
            description = ""
            if ask_coder or (attempt > 1 and last_stderr):
                error_ctx: str | None = None
                model_override: str | None = None
                if attempt > 1 and last_stderr:
                    category = classify_error(last_stderr, last_exit_code)
                    model_override = select_model(category, attempt - 1, settings)
                    hint = _diagnose_known_issue(last_stderr)
                    error_ctx = (
                        f"ATTEMPT {attempt - 1}/{max_retries + 1} FAILED.\n"
                        f"Error category: {category.value}\n"
                        f"{hint}"
                        f"Error output:\n{last_stderr[-2000:]}"
                    )
                watchdog = getattr(self, "_watchdog", None)
                if watchdog:
                    watchdog.heartbeat(f"coder:{phase_name}:attempt_{attempt}")
                try:
                    iteration = await asyncio.wait_for(
                        coder.generate_phase_script(
                            self.contract,
                            phase_name=phase_name,
                            current_code=current_code,
                            journal=journal,
                            hardware=self._hardware,
                            input_artifacts=[a.model_dump(mode="json") for a in (input_artifacts or [])],
                            error_context=error_ctx,
                            model_override=model_override,
                            watchdog=watchdog,
                        ),
                        timeout=180,
                    )
                    current_code = iteration.get("code", current_code)
                    description = iteration.get("description", "")
                except asyncio.TimeoutError:
                    logger.warning("Coder timed out for %s phase (attempt %d)", phase_name, attempt)
                    await self._emit(ActivityType.ERROR, f"Coder timed out for {phase_name}", phase_name=phase_name)
                except Exception as exc:
                    logger.exception("Failed to generate %s phase script (attempt %d)", phase_name, attempt)
                    await self._emit(ActivityType.ERROR, f"Coder failed for {phase_name}: {exc}", phase_name=phase_name)

            retry_label = f" (retry {attempt - 1}/{max_retries})" if attempt > 1 else ""
            await self._emit(
                ActivityType.EDITING,
                f"Preparing {phase_name} phase{retry_label}",
                description,
                phase_name=phase_name,
            )
            result, payload, artifacts = await self._run_phase_script(
                phase_name,
                current_code,
                input_artifacts=input_artifacts,
            )

            succeeded = result.exit_code == 0
            phase.status = PhaseExecutionStatus.COMPLETED if succeeded else PhaseExecutionStatus.FAILED
            phase.invalidated = False
            phase.invalidation_reason = ""
            phase.last_run_at = _utcnow()
            mark_phase_completed(self.contract, phase_name, artifacts=artifacts, script_path=str(store.save_phase_script(self.contract.id, phase_name, current_code)))
            self.contract.phases[phase_name].status = phase.status
            self.contract.phases[phase_name].last_run_at = phase.last_run_at
            store.save_phase_manifest(self.contract.id, self.contract.phases[phase_name])
            store.save_phase_artifacts(self.contract.id, phase_name, artifacts)
            journal.phase_history.append({
                "phase": phase_name,
                "status": phase.status.value,
                "timestamp": phase.last_run_at.isoformat(),
                "payload": payload,
                "attempt": attempt,
            })

            if succeeded:
                await self._emit(
                    ActivityType.LOGGING,
                    f"{phase_name} phase completed",
                    json.dumps(payload)[:1000],
                    phase_name=phase_name,
                )
                return artifacts, True

            last_stderr = result.stderr
            last_exit_code = result.exit_code
            if attempt <= max_retries:
                await self._emit(
                    ActivityType.ERROR,
                    f"{phase_name} phase failed (attempt {attempt}/{max_retries + 1}), retrying with coder fix...",
                    result.stderr[-1000:],
                    phase_name=phase_name,
                )
            else:
                await self._emit(
                    ActivityType.ERROR,
                    f"{phase_name} phase failed after {max_retries + 1} attempts. Halting pipeline.",
                    result.stderr[-1000:],
                    phase_name=phase_name,
                )

        return artifacts, False

    async def _complete_discovery_phase(self, journal: ExperimentJournal) -> tuple[list[PhaseArtifact], bool]:
        """Complete the discovery phase, acquiring dataset if needed.

        If the dataset file doesn't exist yet, the coder agent generates an
        acquisition script (with proper imports/packages), and the normal
        ``_run_phase_script`` flow handles venv creation, dependency
        installation, and execution — no hardcoded dataset registry.

        Returns (artifacts, success).
        """
        phase_name = PhaseName.DISCOVERY.value
        phase = self.contract.phases[phase_name]

        if phase.status == PhaseExecutionStatus.COMPLETED and not phase.invalidated and phase.artifact_refs:
            return phase.artifact_refs, True

        artifacts: list[PhaseArtifact] = []

        if self.contract.dataset:
            dataset = self.contract.dataset
            workspace = self._workspace()
            project_root = Path(self._workspace_root) if self._workspace_root else None
            exists, dataset_path = check_dataset_exists(
                dataset.path, workspace, project_root=project_root,
            )

            if not exists:
                await self._emit(
                    ActivityType.THINKING,
                    f"Dataset not found at {dataset.path}, asking coder to generate acquisition script...",
                    phase_name=phase_name,
                )
                acq_ok = await self._acquire_dataset_via_coder(
                    journal, dataset_path, phase_name,
                )
                if not acq_ok:
                    phase.status = PhaseExecutionStatus.FAILED
                    phase.last_run_at = _utcnow()
                    self.contract.phases[phase_name] = phase
                    store.save_phase_manifest(self.contract.id, phase)
                    journal.phase_history.append({
                        "phase": phase_name,
                        "status": phase.status.value,
                        "timestamp": phase.last_run_at.isoformat(),
                        "error": "Coder-generated acquisition script failed",
                    })
                    return artifacts, False

                if not dataset_path.exists():
                    msg = f"Acquisition script ran but {dataset.path} still not found on disk."
                    await self._emit(ActivityType.ERROR, msg, phase_name=phase_name)
                    phase.status = PhaseExecutionStatus.FAILED
                    phase.last_run_at = _utcnow()
                    self.contract.phases[phase_name] = phase
                    store.save_phase_manifest(self.contract.id, phase)
                    journal.phase_history.append({
                        "phase": phase_name,
                        "status": phase.status.value,
                        "timestamp": phase.last_run_at.isoformat(),
                        "error": msg,
                    })
                    return artifacts, False

                self.contract.dataset.path = str(dataset_path)
                await self._emit(
                    ActivityType.LOGGING,
                    f"Dataset acquired at {dataset_path}",
                    phase_name=phase_name,
                )

            artifacts.append(
                PhaseArtifact(
                    name="dataset_reference",
                    path=self.contract.dataset.path,
                    artifact_type="dataset_reference",
                    metadata={
                        "format": self.contract.dataset.format,
                        "task_type": self.contract.dataset.task_type.value if self.contract.dataset.task_type else "",
                    },
                )
            )

        mark_phase_completed(self.contract, phase_name, artifacts=artifacts)
        phase.status = PhaseExecutionStatus.COMPLETED
        phase.last_run_at = _utcnow()
        self.contract.phases[phase_name] = phase
        store.save_phase_manifest(self.contract.id, phase)
        store.save_phase_artifacts(self.contract.id, phase_name, artifacts)
        journal.phase_history.append({
            "phase": phase_name,
            "status": phase.status.value,
            "timestamp": phase.last_run_at.isoformat(),
        })
        await self._emit(ActivityType.THINKING, "Discovery phase complete", phase_name=phase_name)
        return artifacts, True

    async def _acquire_dataset_via_coder(
        self,
        journal: ExperimentJournal,
        target_path: Path,
        phase_name: str,
        max_retries: int = 2,
    ) -> bool:
        """Use the coder agent to generate and run a data acquisition script.

        The coder sees the full contract (dataset spec, source info) and
        produces a Python script that downloads/generates the dataset.
        Dependencies are auto-installed from imports by the normal runner flow.
        """
        settings = get_settings()
        watchdog = self._watchdog

        current_code = (
            f"# Data acquisition script\n"
            f"# Target output path: {target_path}\n"
            f"# Dataset spec: {json.dumps(self.contract.dataset.model_dump(mode='json'), indent=2) if self.contract.dataset else 'None'}\n"
            f"import pathlib\n"
            f"output = pathlib.Path({str(target_path)!r})\n"
            f"output.parent.mkdir(parents=True, exist_ok=True)\n"
            f"# TODO: download or generate the dataset and save to output\n"
        )

        last_stderr = ""
        last_exit_code = 0

        for attempt in range(1, max_retries + 2):
            error_ctx: str | None = None
            if attempt > 1 and last_stderr:
                error_ctx = (
                    f"ATTEMPT {attempt - 1}/{max_retries + 1} FAILED.\n"
                    f"Error output:\n{last_stderr[-2000:]}"
                )

            if watchdog:
                watchdog.heartbeat(f"coder:acquisition:attempt_{attempt}")

            try:
                iteration = await asyncio.wait_for(
                    coder.generate_phase_script(
                        self.contract,
                        phase_name=phase_name,
                        current_code=current_code,
                        journal=journal,
                        hardware=self._hardware,
                        error_context=error_ctx,
                        watchdog=watchdog,
                    ),
                    timeout=180,
                )
                current_code = iteration.get("code", current_code)
            except asyncio.TimeoutError:
                logger.warning("Coder timed out for acquisition (attempt %d)", attempt)
                await self._emit(ActivityType.ERROR, "Coder timed out for data acquisition", phase_name=phase_name)
                continue
            except Exception as exc:
                logger.exception("Failed to generate acquisition script (attempt %d)", attempt)
                await self._emit(ActivityType.ERROR, f"Coder failed for acquisition: {exc}", phase_name=phase_name)
                continue

            retry_label = f" (retry {attempt - 1}/{max_retries})" if attempt > 1 else ""
            await self._emit(
                ActivityType.EDITING,
                f"Running data acquisition script{retry_label}",
                phase_name=phase_name,
            )

            result, payload, artifacts = await self._run_phase_script(
                phase_name,
                current_code,
            )

            if result.exit_code == 0 and target_path.exists():
                await self._emit(
                    ActivityType.LOGGING,
                    f"Data acquisition succeeded (attempt {attempt})",
                    phase_name=phase_name,
                )
                return True

            last_stderr = result.stderr
            last_exit_code = result.exit_code

            if result.exit_code != 0:
                await self._emit(
                    ActivityType.ERROR,
                    f"Acquisition script failed (attempt {attempt}/{max_retries + 1})",
                    result.stderr[-1000:],
                    phase_name=phase_name,
                )
            elif not target_path.exists():
                last_stderr = f"Script exited 0 but target file {target_path} was not created."
                await self._emit(
                    ActivityType.ERROR,
                    last_stderr,
                    phase_name=phase_name,
                )

        await self._emit(
            ActivityType.ERROR,
            f"Data acquisition failed after {max_retries + 1} attempts.",
            phase_name=phase_name,
        )
        return False

    async def run(self) -> ExperimentJournal:
        self._hardware = detect_hardware()
        await self._emit(ActivityType.THINKING, "Setting up experiment workspace...", self._hardware.model_dump_json())

        self._watchdog = AgentWatchdog(stall_timeout=150)
        runner_task = asyncio.current_task()
        supervisor_task = asyncio.create_task(self._watchdog.supervise(runner_task))

        try:
            return await self._run_inner()
        finally:
            supervisor_task.cancel()
            try:
                await supervisor_task
            except asyncio.CancelledError:
                pass

    async def _run_inner(self) -> ExperimentJournal:
        watchdog = self._watchdog
        watchdog.heartbeat("discovery")
        logger.info("Runner: _run_inner START contract=%s", self.contract.id)

        workspace = self._workspace()
        workspace.mkdir(parents=True, exist_ok=True)
        journal = store.load_journal(self.contract.id) or ExperimentJournal(contract_id=self.contract.id)

        try:
            return await self.__run_inner_body(journal, watchdog, workspace)
        except asyncio.CancelledError:
            logger.warning("Runner: _run_inner CANCELLED, saving journal")
            store.save_journal(journal)
            raise
        except Exception:
            logger.exception("Runner: _run_inner unexpected error, saving journal")
            store.save_journal(journal)
            raise

    def _phase_already_done(self, phase_name: str) -> bool:
        phase = self.contract.phases[phase_name]
        return (
            phase.status == PhaseExecutionStatus.COMPLETED
            and not phase.invalidated
            and bool(phase.artifact_refs)
        )

    async def __run_inner_body(
        self,
        journal: ExperimentJournal,
        watchdog: AgentWatchdog,
        workspace: Path,
    ) -> ExperimentJournal:
        # ── Universal phase-scan: find resume point ──
        _PHASE_LABELS = {
            PhaseName.DISCOVERY.value: "Discovery",
            PhaseName.PREPROCESSING.value: "Preprocessing",
            PhaseName.FEATURIZATION.value: "Featurization",
            PhaseName.TRAINING.value: "Training",
            PhaseName.EVALUATION.value: "Evaluation",
            PhaseName.ANALYSIS.value: "Analysis",
        }
        resume_point: str | None = None
        for phase_name in self.contract.phase_order:
            if self._phase_already_done(phase_name):
                label = _PHASE_LABELS.get(phase_name, phase_name)
                await self._emit_chat_progress(f"Resuming — {label} already complete.", phase_name)
                logger.info("Runner: skipping completed phase %s", phase_name)
            else:
                resume_point = phase_name
                break

        if resume_point is None:
            logger.info("Runner: all phases already complete, nothing to do")
            await self._emit_chat_progress("All phases already complete.", PhaseName.ANALYSIS.value)
            store.save_journal(journal)
            return journal

        resume_label = _PHASE_LABELS.get(resume_point, resume_point)
        logger.info("Runner: resuming from phase %s", resume_point)

        # ── Recover artifacts from completed upstream phases ──
        def _recover_artifacts(phase_name: str) -> list[PhaseArtifact]:
            phase = self.contract.phases.get(phase_name)
            return list(phase.artifact_refs) if phase and phase.artifact_refs else []

        # ── Phase: Discovery ──
        if resume_point == PhaseName.DISCOVERY.value:
            await self._emit_chat_progress("Starting discovery — inspecting dataset...", PhaseName.DISCOVERY.value)
        if not self._phase_already_done(PhaseName.DISCOVERY.value):
            _, discovery_ok = await self._complete_discovery_phase(journal)
            if not discovery_ok:
                await self._emit(
                    ActivityType.ERROR,
                    "Pipeline halted: dataset could not be acquired during discovery.",
                    phase_name=PhaseName.DISCOVERY.value,
                )
                await self._emit_chat_progress("Discovery failed — dataset could not be acquired.", PhaseName.DISCOVERY.value)
                store.save_journal(journal)
                return journal

        # ── Phase: Preprocessing ──
        watchdog.heartbeat("preprocessing")
        if self._phase_already_done(PhaseName.PREPROCESSING.value):
            preprocessing_artifacts = _recover_artifacts(PhaseName.PREPROCESSING.value)
        else:
            if resume_point == PhaseName.PREPROCESSING.value:
                await self._emit_chat_progress("Starting preprocessing...", PhaseName.PREPROCESSING.value)
            preprocessing_artifacts, prep_ok = await self._run_non_iterative_phase(
                PhaseName.PREPROCESSING.value,
                journal,
                ask_coder=True,
            )
            if not prep_ok:
                await self._emit(
                    ActivityType.ERROR,
                    "Pipeline halted: preprocessing phase could not be completed.",
                    phase_name=PhaseName.PREPROCESSING.value,
                )
                await self._emit_chat_progress("Preprocessing failed — pipeline halted.", PhaseName.PREPROCESSING.value)
                store.save_journal(journal)
                return journal

        # ── Phase: Featurization ──
        watchdog.heartbeat("featurization")
        if self._phase_already_done(PhaseName.FEATURIZATION.value):
            featurization_artifacts = _recover_artifacts(PhaseName.FEATURIZATION.value)
        else:
            if resume_point == PhaseName.FEATURIZATION.value:
                await self._emit_chat_progress("Starting featurization...", PhaseName.FEATURIZATION.value)
            featurization_artifacts, feat_ok = await self._run_non_iterative_phase(
                PhaseName.FEATURIZATION.value,
                journal,
                input_artifacts=preprocessing_artifacts,
                ask_coder=True,
            )
            if not feat_ok:
                await self._emit(
                    ActivityType.ERROR,
                    "Pipeline halted: featurization phase could not be completed.",
                    phase_name=PhaseName.FEATURIZATION.value,
                )
                await self._emit_chat_progress("Featurization failed — pipeline halted.", PhaseName.FEATURIZATION.value)
                store.save_journal(journal)
                return journal

        # ── Phase: Training + Evaluation (iterative) ──
        training_done = self._phase_already_done(PhaseName.TRAINING.value)
        eval_done = self._phase_already_done(PhaseName.EVALUATION.value)

        if training_done and eval_done:
            training_artifacts = _recover_artifacts(PhaseName.TRAINING.value)
        else:
            prior_experiments = len(journal.experiments)
            if prior_experiments > 0:
                await self._emit_chat_progress(
                    f"Resuming training from experiment #{prior_experiments + 1}...",
                    PhaseName.TRAINING.value,
                )
            else:
                await self._emit_chat_progress("Features ready. Starting training loop...", PhaseName.TRAINING.value)

            current_code = self._get_phase_code(PhaseName.TRAINING.value)
            run_number = len(journal.experiments)
            max_runs = self.contract.max_experiments or 10
            logger.info("Runner: max_experiments=%d (configured=%s)", max_runs, self.contract.max_experiments)
            MAX_CONSECUTIVE_CRASHES = 5
            consecutive_crashes = 0
            error_escalation: dict[str, int] = {}
            settings = get_settings()
            training_artifacts: list[PhaseArtifact] = []

            while run_number < max_runs and not self._cancelled:
                if await self._check_paused():
                    await self._emit(ActivityType.WAITING, "Paused by user")
                    while await self._check_paused() and not self._cancelled:
                        watchdog.heartbeat("paused")
                        await asyncio.sleep(2)
                    if self._cancelled:
                        break
                    await self._emit(ActivityType.THINKING, "Resumed")

                run_number += 1
                experiment_id = f"run_{run_number:04d}"
                logger.info("Runner: === EXPERIMENT #%d START ===", run_number)

                await self._broadcast({
                    "type": "experiment_started",
                    "contract_id": self.contract.id,
                    "experiment": {
                        "id": experiment_id,
                        "contract_id": self.contract.id,
                        "run_number": run_number,
                        "hypothesis": "",
                        "status": "running",
                        "metrics": {},
                        "verdict_reason": "",
                        "code_diff": "",
                        "timing_seconds": None,
                        "memory_peak_mb": None,
                        "lessons_learned": "",
                        "next_direction": "",
                        "search_queries": [],
                        "created_at": _utcnow().isoformat(),
                    },
                })

                await self._emit(
                    ActivityType.THINKING,
                    f"Generating code for experiment #{run_number}...",
                    experiment_id=experiment_id,
                    phase_name=PhaseName.TRAINING.value,
                )
                await self._emit_chat_progress(f"Generating code for experiment #{run_number}...", PhaseName.TRAINING.value)

                hypothesis = ""
                description = ""
                search_queries: list = []
                new_code = current_code
                coder_failed = False

                watchdog.heartbeat(f"generating_experiment_{run_number}")
                t0 = _time.monotonic()
                logger.info("Runner: calling coder.generate_iteration for experiment #%d ...", run_number)
                try:
                    iteration = await asyncio.wait_for(
                        coder.generate_iteration(
                            self.contract,
                            current_code,
                            journal,
                            self._hardware,
                            input_artifacts=[a.model_dump(mode="json") for a in featurization_artifacts],
                            watchdog=watchdog,
                        ),
                        timeout=180,
                    )
                    new_code = iteration.get("code", current_code)
                    hypothesis = iteration.get("hypothesis", "")
                    description = iteration.get("description", "")
                    search_queries = iteration.get("search_queries", [])
                    logger.info("Runner: coder returned for experiment #%d in %.1fs", run_number, _time.monotonic() - t0)

                    await self._broadcast({
                        "type": "experiment_updated",
                        "contract_id": self.contract.id,
                        "experiment_id": experiment_id,
                        "hypothesis": hypothesis,
                        "status": "running",
                    })
                except asyncio.TimeoutError:
                    logger.warning("Runner: coder TIMED OUT on experiment #%d (%.1fs)", run_number, _time.monotonic() - t0)
                    coder_failed = True
                    hypothesis = "Coder generation timed out"
                except asyncio.CancelledError:
                    logger.warning("Runner: coder CANCELLED on experiment #%d (%.1fs)", run_number, _time.monotonic() - t0)
                    store.save_journal(journal)
                    raise
                except Exception as exc:
                    logger.exception("Runner: coder FAILED on experiment #%d (%.1fs)", run_number, _time.monotonic() - t0)
                    coder_failed = True
                    hypothesis = f"Coder generation failed: {exc}"

                if coder_failed:
                    exp = ExperimentResult(
                        id=experiment_id,
                        contract_id=self.contract.id,
                        run_number=run_number,
                        hypothesis=hypothesis,
                        status=ExperimentStatus.CRASH,
                        verdict_reason=hypothesis,
                        lessons_learned=hypothesis,
                        next_direction="Retry code generation.",
                        phase_name=PhaseName.TRAINING.value,
                    )
                    store.save_experiment(exp)
                    journal.experiments.append(exp)
                    store.save_journal(journal)
                    consecutive_crashes += 1
                    if consecutive_crashes >= MAX_CONSECUTIVE_CRASHES:
                        await self._emit(
                            ActivityType.ERROR,
                            f"Pipeline halted: {consecutive_crashes} consecutive crashes.",
                            phase_name=PhaseName.TRAINING.value,
                        )
                        break
                    await self._emit(ActivityType.ERROR, hypothesis, experiment_id=experiment_id)
                    continue

                diff = _code_diff(current_code, new_code)

                await self._emit(
                    ActivityType.EDITING,
                    f"Training in progress... (experiment #{run_number})",
                    detail=f"{description[:100]}\n{diff[:2000]}",
                    experiment_id=experiment_id,
                    phase_name=PhaseName.TRAINING.value,
                )

                watchdog.heartbeat(f"running_script_{run_number}")
                train_result, payload, training_artifacts = await self._run_phase_script(
                    PhaseName.TRAINING.value,
                    new_code,
                    experiment_id=experiment_id,
                    input_artifacts=featurization_artifacts,
                )
                watchdog.heartbeat(f"evaluating_{run_number}")

                if train_result.exit_code != 0:
                    category = classify_error(
                        train_result.stderr, train_result.exit_code,
                        timed_out=train_result.timed_out,
                    )
                    cat_key = category.value
                    error_escalation[cat_key] = error_escalation.get(cat_key, 0) + 1
                    fix_model = select_model(category, error_escalation[cat_key], settings)

                    await self._emit(
                        ActivityType.THINKING,
                        f"Script crashed ({category.value}), attempting fix with {fix_model.split('/')[-1]}...",
                        detail=train_result.stderr[-500:],
                        experiment_id=experiment_id,
                        phase_name=PhaseName.TRAINING.value,
                    )

                    error_ctx = (
                        f"Error category: {category.value}\n"
                        f"Exit code: {train_result.exit_code}\n"
                        f"Stderr:\n{train_result.stderr[-2000:]}"
                    )
                    watchdog.heartbeat(f"fixing_experiment_{run_number}")
                    t0_fix = _time.monotonic()
                    try:
                        fix_iteration = await asyncio.wait_for(
                            coder.generate_iteration(
                                self.contract,
                                new_code,
                                journal,
                                self._hardware,
                                input_artifacts=[a.model_dump(mode="json") for a in featurization_artifacts],
                                model_override=fix_model,
                                error_context=error_ctx,
                                watchdog=watchdog,
                            ),
                            timeout=180,
                        )
                        logger.info("Coder fix for experiment #%d completed in %.1fs", run_number, _time.monotonic() - t0_fix)
                        fixed_code = fix_iteration.get("code", new_code)
                        if fixed_code != new_code:
                            new_code = fixed_code
                            hypothesis = fix_iteration.get("hypothesis", hypothesis)
                            description = fix_iteration.get("description", description)
                            diff = _code_diff(current_code, new_code)

                            train_result, payload, training_artifacts = await self._run_phase_script(
                                PhaseName.TRAINING.value,
                                new_code,
                                experiment_id=experiment_id,
                                input_artifacts=featurization_artifacts,
                            )
                            if train_result.exit_code == 0:
                                error_escalation.pop(cat_key, None)
                    except asyncio.TimeoutError:
                        logger.warning("Fix attempt timed out for run %d (%.1fs)", run_number, _time.monotonic() - t0_fix)
                    except Exception as fix_exc:
                        logger.exception("Fix attempt failed for run %d: %s", run_number, fix_exc)

                store.save_experiment_logs(self.contract.id, experiment_id, train_result.stdout, train_result.stderr)
                store.save_experiment_phase_outputs(
                    self.contract.id,
                    experiment_id,
                    {
                        PhaseName.TRAINING.value: {"stdout.log": train_result.stdout, "stderr.log": train_result.stderr},
                    },
                )

                metrics = payload.get("metrics", payload)
                if not isinstance(metrics, dict):
                    metrics = {}
                if train_result.exit_code != 0 and "_error" not in metrics:
                    metrics["_error"] = True
                    metrics["_exit_code"] = train_result.exit_code

                await self._emit(
                    ActivityType.EVALUATING,
                    f"Evaluating results for experiment #{run_number}...",
                    detail=json.dumps(metrics),
                    experiment_id=experiment_id,
                    phase_name=PhaseName.EVALUATION.value,
                )

                status, verdict_reason = self._decide(metrics, journal)

                lessons = ""
                next_dir = ""
                if status == ExperimentStatus.KEEP:
                    current_code = new_code
                    lessons = f"This approach worked. {description}"
                    next_dir = "Build on this improvement."
                elif status == ExperimentStatus.CRASH:
                    lessons = f"Training crashed. Stderr: {train_result.stderr[-2000:]}"
                    next_dir = "Fix the crash or try a different approach."
                    consecutive_crashes += 1
                else:
                    lessons = f"No improvement. {verdict_reason}"
                    next_dir = "Try a different approach."

                if status != ExperimentStatus.CRASH:
                    consecutive_crashes = 0

                exp = ExperimentResult(
                    id=experiment_id,
                    contract_id=self.contract.id,
                    run_number=run_number,
                    hypothesis=hypothesis,
                    code_diff=diff,
                    metrics=metrics,
                    timing_seconds=train_result.elapsed_seconds,
                    memory_peak_mb=train_result.peak_memory_mb,
                    status=status,
                    verdict_reason=verdict_reason,
                    lessons_learned=lessons,
                    next_direction=next_dir,
                    search_queries=search_queries,
                    phase_name=PhaseName.TRAINING.value,
                    phase_artifacts=training_artifacts,
                )
                store.save_experiment(exp)
                store.save_experiment_code(self.contract.id, experiment_id, new_code)
                if status == ExperimentStatus.KEEP:
                    store.save_phase_script(self.contract.id, PhaseName.TRAINING.value, new_code)

                journal.experiments.append(exp)
                journal.phase_history.append({
                    "phase": PhaseName.TRAINING.value,
                    "experiment_id": experiment_id,
                    "status": status.value,
                    "timestamp": _utcnow().isoformat(),
                    "metrics": metrics,
                })
                if status == ExperimentStatus.KEEP:
                    journal.best_experiment_id = experiment_id
                    primary = self.contract.eval_spec.primary_metric if self.contract.eval_spec else ""
                    if primary and primary in metrics:
                        journal.best_metric_value = metrics[primary]
                store.save_journal(journal)

                training_phase = self.contract.phases[PhaseName.TRAINING.value]
                training_phase.status = PhaseExecutionStatus.COMPLETED
                training_phase.last_run_at = _utcnow()
                training_phase.artifact_refs = training_artifacts
                store.save_phase_manifest(self.contract.id, training_phase)
                store.save_phase_artifacts(self.contract.id, PhaseName.TRAINING.value, training_artifacts)

                evaluation_phase = self.contract.phases[PhaseName.EVALUATION.value]
                evaluation_phase.status = PhaseExecutionStatus.COMPLETED
                evaluation_phase.last_run_at = training_phase.last_run_at
                evaluation_phase.metadata["last_metrics"] = metrics
                evaluation_phase.artifact_refs = [
                    PhaseArtifact(
                        name="evaluation_metrics",
                        path="inline",
                        artifact_type="metrics",
                        metadata=metrics,
                    )
                ]
                store.save_phase_manifest(self.contract.id, evaluation_phase)

                await self._emit(
                    ActivityType.DECIDING,
                    f"Comparing metrics and deciding... (experiment #{run_number}: {status.value})",
                    detail=verdict_reason,
                    experiment_id=experiment_id,
                    phase_name=PhaseName.ANALYSIS.value,
                )

                primary = self.contract.eval_spec.primary_metric if self.contract.eval_spec else ""
                primary_val = metrics.get(primary) if primary else None
                metric_str = f" — {primary}={primary_val}" if primary_val is not None else ""
                await self._emit_chat_progress(
                    f"Experiment #{run_number}: {status.value}{metric_str}. {verdict_reason}",
                    PhaseName.EVALUATION.value,
                )

                await self._broadcast({
                    "type": "experiment_completed",
                    "contract_id": self.contract.id,
                    "experiment": exp.model_dump(mode="json"),
                })

                if consecutive_crashes >= MAX_CONSECUTIVE_CRASHES:
                    await self._emit(
                        ActivityType.ERROR,
                        f"Pipeline halted: {consecutive_crashes} consecutive crashes.",
                        phase_name=PhaseName.TRAINING.value,
                    )
                    await self._emit_chat_progress(
                        f"Pipeline halted after {consecutive_crashes} consecutive crashes.",
                        PhaseName.TRAINING.value,
                    )
                    break

                if self._threshold_met(metrics):
                    logger.info("Runner: target threshold met at experiment #%d, stopping", run_number)
                    await self._emit(ActivityType.DECIDING, "Target threshold reached!", phase_name=PhaseName.ANALYSIS.value)
                    await self._emit_chat_progress("Target threshold reached!", PhaseName.ANALYSIS.value)
                    break

                logger.info("Runner: experiment #%d done (status=%s), looping to next", run_number, status.value)

            logger.info("Runner: training loop finished after %d experiments", run_number)

        # ── Phase: Analysis ──
        await self._emit_chat_progress("Running analysis...", PhaseName.ANALYSIS.value)
        watchdog.heartbeat("analysis")

        journal_path = workspace / "journal.json"
        journal_path.write_text(journal.model_dump_json(indent=2), encoding="utf-8")

        combined_artifacts = list(featurization_artifacts) + list(training_artifacts)

        analysis_artifacts, analysis_ok = await self._run_non_iterative_phase(
            PhaseName.ANALYSIS.value,
            journal,
            input_artifacts=combined_artifacts,
            ask_coder=True,
            max_retries=2,
        )

        if analysis_ok:
            analysis_output = self._read_phase_output(PhaseName.ANALYSIS.value)
            if analysis_output and isinstance(analysis_output, dict):
                journal.agent_commentary = analysis_output.get("summary", "")
        else:
            await self._emit(ActivityType.ERROR, "Analysis phase failed.", phase_name=PhaseName.ANALYSIS.value)
            journal.agent_commentary = self._generate_commentary(journal)

        journal.phase_history.append({
            "phase": PhaseName.ANALYSIS.value,
            "status": "completed" if analysis_ok else "failed",
            "timestamp": _utcnow().isoformat(),
            "commentary": journal.agent_commentary,
        })
        store.save_journal(journal)
        return journal

    def _decide(self, metrics: dict[str, Any], journal: ExperimentJournal) -> tuple[ExperimentStatus, str]:
        if metrics.get("_error"):
            return ExperimentStatus.CRASH, f"Training failed with exit code {metrics.get('_exit_code', -1)}"

        if not self.contract.eval_spec or not self.contract.eval_spec.primary_metric:
            return ExperimentStatus.KEEP, "No primary metric defined, keeping by default"

        primary = self.contract.eval_spec.primary_metric
        if primary not in metrics:
            return ExperimentStatus.DISCARD, f"Primary metric '{primary}' not found in output"

        current_val = metrics[primary]
        higher_better = self.contract.threshold_direction == "higher_is_better"

        best_id = journal.best_experiment_id
        best_label = f"best (#{best_id.replace('run_', '').lstrip('0') or '0'})" if best_id else None

        if journal.best_metric_value is None:
            return ExperimentStatus.KEEP, f"First successful run — {primary}: {current_val}"

        best = journal.best_metric_value
        if higher_better:
            if current_val > best:
                return ExperimentStatus.KEEP, f"{primary} improved: {best} \u2192 {current_val} (vs {best_label})"
            return ExperimentStatus.DISCARD, f"No improvement — {primary}: {current_val} (best is {best} from {best_label})"
        if current_val < best:
            return ExperimentStatus.KEEP, f"{primary} improved: {best} \u2192 {current_val} (vs {best_label})"
        return ExperimentStatus.DISCARD, f"No improvement — {primary}: {current_val} (best is {best} from {best_label})"

    def _threshold_met(self, metrics: dict[str, Any]) -> bool:
        if self.contract.target_threshold is None:
            return False
        if not self.contract.eval_spec or not self.contract.eval_spec.primary_metric:
            return False
        primary = self.contract.eval_spec.primary_metric
        if primary not in metrics:
            return False
        value = metrics[primary]
        if self.contract.threshold_direction == "higher_is_better":
            return value >= self.contract.target_threshold
        return value <= self.contract.target_threshold

    def _read_phase_output(self, phase_name: str) -> dict[str, Any] | None:
        """Read the structured JSON payload from a phase's latest run."""
        run_dir = self._workspace() / "runs" / phase_name / "latest" / "outputs"
        payload_file = run_dir / "phase_payload.json"
        if not payload_file.exists():
            return None
        try:
            return json.loads(payload_file.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            return None

    def _generate_commentary(self, journal: ExperimentJournal) -> str:
        kept = [experiment for experiment in journal.experiments if experiment.status == ExperimentStatus.KEEP]
        discarded = [experiment for experiment in journal.experiments if experiment.status == ExperimentStatus.DISCARD]
        crashed = [experiment for experiment in journal.experiments if experiment.status == ExperimentStatus.CRASH]
        total = len(journal.experiments)

        parts = [
            f"Completed {total} training/evaluation iterations after reusable discovery, preprocessing, and featurization phases.",
            f"Results: {len(kept)} kept, {len(discarded)} discarded, {len(crashed)} crashed.",
        ]
        if journal.best_metric_value is not None:
            parts.append(f"Best metric value: {journal.best_metric_value}")
        if kept:
            parts.append(f"Best run: #{kept[-1].run_number}")
        return " ".join(parts)
