import tempfile
import unittest
from unittest import mock
from contextlib import redirect_stdout
from io import StringIO
from pathlib import Path
import uuid
import os
import json
import urllib.request
import re
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import threading

from hiveos.intelligence import tracing as tracing_module
from hiveos import config as config_module
from hiveos import hive as hive_module
from hiveos.observe import (
    observe_agent_step_end,
    observe_agent_step_start,
    observe_branch_decision,
    observe_checkpoint,
    observe_rerun_request,
    observe_run,
    observe_proxy,
    observe_step,
    observe_step_retry,
    observe_tool_call_end,
    observe_tool_call_start,
    run_command_observed,
    main as observe_main,
)
from hiveos_trace import run_exec as run_exec_module


class ObserveHarnessTests(unittest.TestCase):
    def setUp(self):
        self._original_trace_db_async = tracing_module.TRACE_DB_ASYNC
        self._original_trace_sampling_enabled = tracing_module.TRACE_SAMPLING_ENABLED
        self._original_trace_log_path = tracing_module.TRACE_LOG_PATH
        self._original_trace_home = os.environ.get("HIVE_TRACE_HOME")
        self._original_disable_browser = os.environ.get("HIVE_TRACE_DISABLE_BROWSER")
        self._original_reconcile_enabled = config_module.TRACE_RECONCILE_ENABLED
        self._original_reconcile_auto_on_read = config_module.TRACE_RECONCILE_AUTO_ON_READ
        self._original_reconcile_stale_sec = config_module.TRACE_RECONCILE_STALE_SEC
        self._original_open_on_run = config_module.TRACE_OPEN_ON_RUN

        self._trace_log_path = str(Path(tempfile.gettempdir()) / f"hive-observe-{uuid.uuid4().hex}.log")
        self._trace_home = str(Path(tempfile.gettempdir()) / f"hive-observe-home-{uuid.uuid4().hex}")
        tracing_module.TRACE_LOG_PATH = self._trace_log_path
        tracing_module.TRACE_DB_ASYNC = False
        tracing_module.TRACE_SAMPLING_ENABLED = False
        os.environ["HIVE_TRACE_HOME"] = self._trace_home
        os.environ["HIVE_TRACE_DISABLE_BROWSER"] = "1"
        config_module.TRACE_RECONCILE_ENABLED = True
        config_module.TRACE_RECONCILE_AUTO_ON_READ = True
        config_module.TRACE_RECONCILE_STALE_SEC = 300
        config_module.TRACE_OPEN_ON_RUN = False

    def tearDown(self):
        tracing_module.TRACE_DB_ASYNC = self._original_trace_db_async
        tracing_module.TRACE_SAMPLING_ENABLED = self._original_trace_sampling_enabled
        tracing_module.TRACE_LOG_PATH = self._original_trace_log_path
        if self._original_trace_home is None:
            os.environ.pop("HIVE_TRACE_HOME", None)
        else:
            os.environ["HIVE_TRACE_HOME"] = self._original_trace_home
        if self._original_disable_browser is None:
            os.environ.pop("HIVE_TRACE_DISABLE_BROWSER", None)
        else:
            os.environ["HIVE_TRACE_DISABLE_BROWSER"] = self._original_disable_browser
        config_module.TRACE_RECONCILE_ENABLED = self._original_reconcile_enabled
        config_module.TRACE_RECONCILE_AUTO_ON_READ = self._original_reconcile_auto_on_read
        config_module.TRACE_RECONCILE_STALE_SEC = self._original_reconcile_stale_sec
        config_module.TRACE_OPEN_ON_RUN = self._original_open_on_run
        self._cleanup_trace_files()
        self._cleanup_trace_home()

    def _cleanup_trace_files(self):
        base = Path(self._trace_log_path)
        for path in [base] + list(base.parent.glob(f"{base.name}.*")):
            try:
                if path.exists():
                    path.unlink()
            except Exception:
                pass

    def _cleanup_trace_home(self):
        home = Path(self._trace_home)
        if not home.exists():
            return
        for path in sorted(home.rglob("*"), reverse=True):
            try:
                if path.is_file():
                    path.unlink()
                elif path.is_dir():
                    path.rmdir()
            except Exception:
                pass

    def test_observe_run_context_emits_start_step_finish(self):
        with observe_run("unit-test-run", metadata={"suite": "observe"}) as run_id:
            observe_step(run_id, "step-alpha", payload={"ok": True})

        started = tracing_module.read_trace_events(limit=200, filters={"event_type": "observe_run_started"})
        stepped = tracing_module.read_trace_events(limit=200, filters={"event_type": "observe_step"})
        finished = tracing_module.read_trace_events(limit=200, filters={"event_type": "observe_run_finished"})

        self.assertTrue(started, "expected observe_run_started event")
        self.assertTrue(stepped, "expected observe_step event")
        self.assertTrue(finished, "expected observe_run_finished event")
        self.assertEqual("success", finished[-1].get("outcome"))

    def test_run_command_observed_emits_cli_events(self):
        result = run_command_observed(
            ["python", "-c", "print('hello-observe')"],
            run_name="cli-test",
            open_ui_url="",
        )
        self.assertEqual(0, result["exit_code"])
        self.assertIn("run_id", result)

        started = tracing_module.read_trace_events(limit=200, filters={"event_type": "observe_run_started"})
        finished = tracing_module.read_trace_events(limit=200, filters={"event_type": "observe_run_finished"})
        steps = tracing_module.read_trace_events(limit=200, filters={"run_id": result["run_id"], "event_type": "observe_step"})
        self.assertTrue(started)
        self.assertTrue(finished)
        self.assertEqual("success", finished[-1].get("outcome"))
        finish_steps = [event for event in steps if str((event.get("payload") or {}).get("step_name") or "") == "command.finish"]
        self.assertTrue(finish_steps)
        finish_payload = finish_steps[-1].get("payload") or {}
        self.assertEqual("output.commit", finish_payload.get("canonical_anchor_type"))
        self.assertEqual("low", finish_payload.get("idempotency_hint"))
        self.assertFalse(bool(finish_payload.get("side_effect")))
        self.assertFalse(bool(finish_payload.get("external_write")))

    def test_run_command_observed_tolerates_non_utf_output_bytes(self):
        # Emit a raw byte sequence that can fail locale decoding on Windows.
        result = run_command_observed(
            ["python", "-c", "import sys; sys.stdout.buffer.write(bytes([0x8f, 0x0a])); sys.stdout.flush()"],
            run_name="decode-tolerance-test",
            open_ui_url="",
            timeout_sec=5,
        )
        self.assertEqual(0, result["exit_code"])

    def test_launch_error_windows_includes_cmd_fallback_hint(self):
        with mock.patch("hiveos_trace.run_exec.os.name", "nt"):
            text = run_exec_module._format_launch_error(["openclaw", "agent"], FileNotFoundError("[WinError 2]"))
        self.assertIn("Executable not found: openclaw", text)
        self.assertIn("Try: hive trace run -- cmd /c", text)

    def test_launch_error_shell_wrapper_does_not_hint_cmd_c(self):
        with mock.patch("hiveos_trace.run_exec.os.name", "nt"):
            text = run_exec_module._format_launch_error(["cmd", "/c", "openclaw"], FileNotFoundError("[WinError 2]"))
        self.assertIn("Executable not found: cmd", text)
        self.assertNotIn("Try: hive trace run -- cmd /c", text)

    def test_hive_module_alias_runs_trace_command(self):
        run_command_observed(["python", "-c", "pass"], run_name="hive-alias-test", open_ui_url="")

        buf = StringIO()
        with redirect_stdout(buf):
            code = hive_module.main(["trace", "ls", "--limit", "5"])
        self.assertEqual(0, code)
        self.assertIn("observe-run:", buf.getvalue())

    def test_checkpoint_and_rerun_sdk_events(self):
        with observe_run("checkpoint-test") as run_id:
            observe_checkpoint(run_id, "ckpt-1", step_name="stage.extract", state_ref="state://extract/1")
            observe_rerun_request(run_id, from_checkpoint_id="ckpt-1", reason="retry_with_override")

        checkpoint_events = tracing_module.read_trace_events(limit=200, filters={"event_type": "observe_checkpoint"})
        rerun_events = tracing_module.read_trace_events(limit=200, filters={"event_type": "observe_rerun_requested"})
        self.assertTrue(checkpoint_events)
        self.assertTrue(rerun_events)
        self.assertEqual("ckpt-1", (checkpoint_events[-1].get("payload") or {}).get("checkpoint_id"))
        self.assertEqual("ckpt-1", (rerun_events[-1].get("payload") or {}).get("from_checkpoint_id"))

    def test_trace_run_ls_and_open_flow(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('trace-run')"])
        self.assertEqual(0, code)
        run_output = run_buf.getvalue()
        self.assertIn("run_id=", run_output)
        run_id = ""
        for line in run_output.splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        ls_buf = StringIO()
        with redirect_stdout(ls_buf):
            ls_code = observe_main(["trace", "ls", "--limit", "5"])
        self.assertEqual(0, ls_code)
        self.assertIn(run_id, ls_buf.getvalue())

        open_buf = StringIO()
        with redirect_stdout(open_buf):
            open_code = observe_main(["trace", "open", run_id, "--ui-url", "http://localhost:5173"])
        self.assertEqual(0, open_code)
        self.assertIn("opened=http://localhost:5173", open_buf.getvalue())

        show_buf = StringIO()
        original_log_path = tracing_module.TRACE_LOG_PATH
        tracing_module.TRACE_LOG_PATH = str(Path(tempfile.gettempdir()) / f"hive-observe-mismatch-{uuid.uuid4().hex}.log")
        try:
            with redirect_stdout(show_buf):
                show_code = observe_main(["trace", "show", run_id, "--limit", "200"])
        finally:
            tracing_module.TRACE_LOG_PATH = original_log_path
        self.assertEqual(0, show_code)
        self.assertIn(run_id, show_buf.getvalue())

        show_json_buf = StringIO()
        with redirect_stdout(show_json_buf):
            show_json_code = observe_main(["trace", "show", run_id, "--limit", "200", "--json"])
        self.assertEqual(0, show_json_code)
        self.assertIn(f'"run_id":"{run_id}"', show_json_buf.getvalue())

    def test_trace_run_open_flag_behavior(self):
        default_buf = StringIO()
        with redirect_stdout(default_buf):
            default_code = observe_main(["trace", "run", "--", "python", "-c", "print('open-default')"])
        self.assertEqual(0, default_code)
        self.assertNotIn("ui=", default_buf.getvalue())

        explicit_buf = StringIO()
        with redirect_stdout(explicit_buf):
            explicit_code = observe_main(
                ["trace", "run", "--open", "--ui-url", "http://localhost:5173", "--", "python", "-c", "print('open-explicit')"]
            )
        self.assertEqual(0, explicit_code)
        self.assertIn("ui=http://localhost:5173", explicit_buf.getvalue())

        config_module.TRACE_OPEN_ON_RUN = True
        env_default_buf = StringIO()
        with redirect_stdout(env_default_buf):
            env_default_code = observe_main(["trace", "run", "--", "python", "-c", "print('open-env-default')"])
        self.assertEqual(0, env_default_code)
        self.assertIn("ui=http://localhost:5173", env_default_buf.getvalue())

        override_buf = StringIO()
        with redirect_stdout(override_buf):
            override_code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('open-no-override')"])
        self.assertEqual(0, override_code)
        self.assertNotIn("ui=", override_buf.getvalue())

    def test_trace_run_open_falls_back_to_local_status_page_when_ui_unreachable(self):
        os.environ.pop("HIVE_TRACE_DISABLE_BROWSER", None)
        captured = {"url": "", "html": ""}

        def _fake_browser_open(url, new=2):
            captured["url"] = str(url)
            with urllib.request.urlopen(str(url), timeout=2) as resp:
                captured["html"] = resp.read().decode("utf-8")
            return True

        with mock.patch("hiveos.observe._check_ui_reachable", return_value=(False, "ui unreachable")):
            with mock.patch("hiveos.observe.webbrowser.open", side_effect=_fake_browser_open):
                buf = StringIO()
                with redirect_stdout(buf):
                    code = observe_main(
                        [
                            "trace",
                            "run",
                            "--open",
                            "--ui-url",
                            "http://localhost:5173",
                            "--",
                            "python",
                            "-c",
                            "print('fallback-open')",
                        ]
                    )
        self.assertEqual(0, code)
        self.assertTrue(captured["url"].startswith("http://127.0.0.1:"))
        self.assertIn("Hive Trace Status", captured["html"])
        self.assertIn("http://localhost:5173", captured["html"])

    def test_trace_diff_json_reports_divergence(self):
        run_ids = []
        for text in ("alpha", "beta"):
            buf = StringIO()
            with redirect_stdout(buf):
                code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", f"print('{text}')"])
            self.assertEqual(0, code)
            rid = ""
            for line in buf.getvalue().splitlines():
                if line.startswith("run_id="):
                    rid = line.split()[0].split("=", 1)[1].strip()
                    break
            self.assertTrue(rid)
            run_ids.append(rid)

        diff_buf = StringIO()
        with redirect_stdout(diff_buf):
            diff_code = observe_main(
                ["trace", "diff", run_ids[0], run_ids[1], "--json", "--event-limit", "500", "--tail-lines", "10"]
            )
        self.assertEqual(0, diff_code)
        payload = json.loads(diff_buf.getvalue().strip())
        self.assertEqual(run_ids[0], payload["run_a"]["run_id"])
        self.assertEqual(run_ids[1], payload["run_b"]["run_id"])
        self.assertIn("execution", payload)
        self.assertGreaterEqual(int(payload["execution"]["first_divergence_index"]), 0)

    def test_trace_summary_json_for_run(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(
                [
                    "trace",
                    "run",
                    "--no-open",
                    "--",
                    "python",
                    "-c",
                    "import sys; print('summary-stdout'); sys.stderr.write('summary-stderr\\n')",
                ]
            )
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        summary_buf = StringIO()
        with redirect_stdout(summary_buf):
            summary_code = observe_main(["trace", "summary", run_id, "--json", "--event-limit", "500", "--tail-lines", "5"])
        self.assertEqual(0, summary_code)
        payload = json.loads(summary_buf.getvalue().strip())
        self.assertEqual(run_id, payload["run_id"])
        self.assertEqual("success", payload["status"])
        self.assertIn("summary-stderr", "\n".join(payload.get("stderr_tail") or []))
        self.assertIn("summary-stdout", "\n".join(payload.get("stdout_tail") or []))
        self.assertGreaterEqual(int(payload.get("event_count") or 0), 1)
        self.assertGreaterEqual(len(payload.get("suggestions") or []), 1)
        self.assertIn("runtime_state", payload)
        self.assertIn("last_activity_ms", payload)
        self.assertIn("last_activity_age_ms", payload)
        self.assertIn("last_heartbeat_ms", payload)
        self.assertIn("last_heartbeat_age_ms", payload)

    def test_trace_summary_missing_run_errors(self):
        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "summary", "observe-run:missing-id"])
        self.assertIn("run_id not found", str(ctx.exception))

    def test_trace_tail_json_with_run_filter(self):
        run_ids = []
        for label in ("tail-a", "tail-b"):
            run_buf = StringIO()
            with redirect_stdout(run_buf):
                code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", f"print('{label}')"])
            self.assertEqual(0, code)
            rid = ""
            for line in run_buf.getvalue().splitlines():
                if line.startswith("run_id="):
                    rid = line.split()[0].split("=", 1)[1].strip()
                    break
            self.assertTrue(rid)
            run_ids.append(rid)

        tail_buf = StringIO()
        with redirect_stdout(tail_buf):
            tail_code = observe_main(["trace", "tail", "--run-id", run_ids[0], "--limit", "200", "--json"])
        self.assertEqual(0, tail_code)
        lines = [line.strip() for line in tail_buf.getvalue().splitlines() if line.strip()]
        self.assertGreaterEqual(len(lines), 1)
        for line in lines:
            payload = json.loads(line)
            self.assertEqual(run_ids[0], payload.get("run_id"))

    def test_trace_tail_follow_exits_cleanly_on_keyboard_interrupt(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('tail-follow')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        tail_buf = StringIO()
        with mock.patch("hiveos.observe.time.sleep", side_effect=KeyboardInterrupt):
            with redirect_stdout(tail_buf):
                tail_code = observe_main(["trace", "tail", "--run-id", run_id, "--follow", "--limit", "20"])
        self.assertEqual(0, tail_code)
        self.assertIn(run_id, tail_buf.getvalue())

    def test_trace_export_import_round_trip_with_as_run_id(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(
                [
                    "trace",
                    "run",
                    "--no-open",
                    "--",
                    "python",
                    "-c",
                    "import sys; print('bundle-stdout'); sys.stderr.write('bundle-stderr\\n')",
                ]
            )
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        bundle_path = Path(self._trace_home) / "tmp" / "trace.bundle.json"
        export_buf = StringIO()
        with redirect_stdout(export_buf):
            export_code = observe_main(["trace", "ops", "export", source_run_id, "--bundle", str(bundle_path)])
        self.assertEqual(0, export_code)
        self.assertTrue(bundle_path.exists())
        self.assertIn("exported=", export_buf.getvalue())

        imported_run_id = "observe-run:imported-bundle"
        import_buf = StringIO()
        with redirect_stdout(import_buf):
            import_code = observe_main(
                ["trace", "ops", "import", "--bundle", str(bundle_path), "--as-run-id", imported_run_id]
            )
        self.assertEqual(0, import_code)
        self.assertIn(f"imported={imported_run_id}", import_buf.getvalue())

        record_path = Path(self._trace_home) / "runs" / f"{imported_run_id.replace(':', '_')}.json"
        self.assertTrue(record_path.exists())
        record = json.loads(record_path.read_text(encoding="utf-8"))
        self.assertEqual(imported_run_id, record.get("run_id"))
        self.assertEqual(source_run_id, record.get("import_source_run_id"))
        self.assertTrue(bool(record.get("imported_from_bundle")))

        show_buf = StringIO()
        with redirect_stdout(show_buf):
            show_code = observe_main(["trace", "show", imported_run_id, "--json", "--limit", "300"])
        self.assertEqual(0, show_code)
        lines = [line.strip() for line in show_buf.getvalue().splitlines() if line.strip()]
        self.assertGreaterEqual(len(lines), 1)
        payload = json.loads(lines[-1])
        self.assertEqual(imported_run_id, payload.get("run_id"))

    def test_trace_import_fails_on_existing_run_id_collision(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('collision-source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        bundle_path = Path(self._trace_home) / "tmp" / "collision.bundle.json"
        observe_main(["trace", "ops", "export", run_id, "--bundle", str(bundle_path)])
        self.assertTrue(bundle_path.exists())

        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "ops", "import", "--bundle", str(bundle_path)])
        self.assertIn("run_id already exists", str(ctx.exception))

    def test_trace_replay_creates_lineage_link(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay source text')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        replay_buf = StringIO()
        with redirect_stdout(replay_buf):
            replay_code = observe_main(["trace", "replay", source_run_id, "--no-open"])
        self.assertEqual(0, replay_code)
        replay_run_id = ""
        for line in replay_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                replay_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(replay_run_id)

        replay_record_path = Path(self._trace_home) / "runs" / f"{replay_run_id.replace(':', '_')}.json"
        replay_record = json.loads(replay_record_path.read_text(encoding="utf-8"))
        self.assertEqual(source_run_id, replay_record.get("replay_of_run_id"))
        self.assertEqual("replay", replay_record.get("source"))
        self.assertIsInstance(replay_record.get("argv"), list)
        self.assertEqual("python", str((replay_record.get("argv") or [""])[0]))
        self.assertEqual("success", replay_record.get("status"))

    def test_trace_replay_with_from_step_id_anchor(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay anchored source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "anchor.step"},
                flow_id="flow:replay-anchor",
                step_id="step:anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        replay_buf = StringIO()
        with redirect_stdout(replay_buf):
            replay_code = observe_main(
                [
                    "trace",
                    "replay",
                    source_run_id,
                    "--from-step-id",
                    "step:anchor-1",
                    "--validation-mode",
                    "deterministic",
                    "--no-open",
                ]
            )
        self.assertEqual(0, replay_code)
        replay_run_id = ""
        for line in replay_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                replay_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(replay_run_id)
        self.assertIn("from_step_id=step:anchor-1", replay_buf.getvalue())
        self.assertIn("validation_mode=deterministic", replay_buf.getvalue())

        replay_record_path = Path(self._trace_home) / "runs" / f"{replay_run_id.replace(':', '_')}.json"
        replay_record = json.loads(replay_record_path.read_text(encoding="utf-8"))
        self.assertEqual("step:anchor-1", replay_record.get("replay_from_step_id"))
        self.assertEqual("step_id", replay_record.get("replay_anchor_type"))
        self.assertEqual("deterministic", replay_record.get("replay_validation_mode"))

    def test_trace_replay_with_missing_from_step_id_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay bad anchor source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "replay", source_run_id, "--from-step-id", "step:not-there", "--no-open"])
        self.assertIn("step_id not found on source run", str(ctx.exception))

    def test_trace_replay_with_from_checkpoint_id_anchor(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay checkpoint source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        observe_checkpoint(source_run_id, "ckpt-anchor-1", step_name="anchor.step", state_ref="state://anchor/1")

        replay_buf = StringIO()
        with redirect_stdout(replay_buf):
            replay_code = observe_main(
                ["trace", "replay", source_run_id, "--from-checkpoint-id", "ckpt-anchor-1", "--no-open"]
            )
        self.assertEqual(0, replay_code)
        replay_run_id = ""
        for line in replay_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                replay_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(replay_run_id)
        self.assertIn("from_checkpoint_id=ckpt-anchor-1", replay_buf.getvalue())

        replay_record_path = Path(self._trace_home) / "runs" / f"{replay_run_id.replace(':', '_')}.json"
        replay_record = json.loads(replay_record_path.read_text(encoding="utf-8"))
        self.assertEqual("ckpt-anchor-1", replay_record.get("replay_from_checkpoint_id"))
        self.assertEqual("checkpoint_id", replay_record.get("replay_anchor_type"))

    def test_trace_replay_with_both_anchor_flags_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay both anchor source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        with self.assertRaises(SystemExit) as ctx:
            observe_main(
                [
                    "trace",
                    "replay",
                    source_run_id,
                    "--from-step-id",
                    "step:any",
                    "--from-checkpoint-id",
                    "ckpt:any",
                    "--no-open",
                ]
            )
        self.assertIn("Choose only one of --from-step-id or --from-checkpoint-id", str(ctx.exception))

    def test_trace_replay_plan_json_with_step_anchor(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "plan.anchor"},
                flow_id="flow:replay-plan",
                step_id="step:plan-anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                ["trace", "replay-plan", source_run_id, "--from-step-id", "step:plan-anchor-1", "--json"]
            )
        self.assertEqual(0, plan_code)
        payload = json.loads(plan_buf.getvalue().strip())
        self.assertEqual(source_run_id, payload.get("replay_of_run_id"))
        self.assertEqual("step:plan-anchor-1", payload.get("from_step_id"))
        self.assertEqual("tolerant", payload.get("validation_mode"))
        checks = payload.get("validation_checks") or {}
        self.assertEqual("pass", checks.get("status"))
        self.assertEqual("step.generic", payload.get("anchor_type"))
        source_window = payload.get("source_window") or {}
        self.assertEqual(source_run_id, source_window.get("source_run_id"))
        self.assertEqual("tolerant", source_window.get("validation_mode"))
        anchor = source_window.get("anchor") or {}
        self.assertEqual("step_id", anchor.get("anchor_type"))
        self.assertEqual("step:plan-anchor-1", anchor.get("anchor_id"))
        self.assertEqual("step.generic", anchor.get("canonical_anchor_type"))
        resume_envelope = payload.get("resume_envelope") or {}
        self.assertEqual("1.0", resume_envelope.get("resume_contract_version"))
        self.assertEqual(source_run_id, resume_envelope.get("parent_run_id"))
        self.assertEqual("step:plan-anchor-1", resume_envelope.get("anchor_id"))
        self.assertEqual("pass", ((resume_envelope.get("historical_closure") or {}).get("status")))
        self.assertEqual("partial", ((resume_envelope.get("state") or {}).get("state_sufficiency")))
        self.assertTrue(bool((resume_envelope.get("admission") or {}).get("allowed")))
        branch_preview = payload.get("branch_preview") or {}
        self.assertEqual("full_replay_preview", branch_preview.get("mode"))
        self.assertFalse(bool(branch_preview.get("preview_only")))
        self.assertFalse(bool(payload.get("does_execute")))
        self.assertIsNotNone(payload.get("replay_command"))
        self.assertNotIn("artifact_path", payload)

    def test_trace_replay_plan_json_emit_artifact_default_path(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan artifact source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "artifact.anchor"},
                flow_id="flow:replay-plan-artifact",
                step_id="step:artifact-anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:artifact-anchor-1",
                    "--emit-artifact",
                    "--json",
                ]
            )
        self.assertEqual(0, plan_code)
        payload = json.loads(plan_buf.getvalue().strip())
        artifact_path = str(payload.get("artifact_path") or "").strip()
        self.assertTrue(artifact_path)
        self.assertTrue(Path(artifact_path).exists())
        artifact_payload = json.loads(Path(artifact_path).read_text(encoding="utf-8"))
        self.assertEqual("replay_plan", artifact_payload.get("artifact_type"))
        self.assertEqual(1, int(artifact_payload.get("artifact_version") or 0))
        self.assertTrue(str(artifact_payload.get("artifact_id") or "").startswith("replay-plan:"))
        integrity = artifact_payload.get("integrity") or {}
        self.assertTrue(str(integrity.get("plan_hash") or "").startswith("sha256:"))
        self.assertEqual("plan", integrity.get("hash_scope"))
        self.assertEqual(source_run_id, ((artifact_payload.get("plan") or {}).get("replay_of_run_id")))
        source_window = ((artifact_payload.get("plan") or {}).get("source_window") or {})
        self.assertEqual(source_run_id, source_window.get("source_run_id"))

    def test_trace_replay_plan_json_emit_artifact_custom_path(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan custom artifact source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "artifact.custom.anchor"},
                flow_id="flow:replay-plan-artifact-custom",
                step_id="step:artifact-custom-anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )
        custom_path = Path(self._trace_home) / "tmp" / "replay-plan.custom.json"

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:artifact-custom-anchor-1",
                    "--artifact-path",
                    str(custom_path),
                    "--json",
                ]
            )
        self.assertEqual(0, plan_code)
        payload = json.loads(plan_buf.getvalue().strip())
        artifact_path = str(payload.get("artifact_path") or "").strip()
        self.assertTrue(artifact_path)
        self.assertEqual(str(custom_path.resolve()), artifact_path)
        self.assertTrue(custom_path.exists())

    def test_trace_replay_plan_recommended_selects_admissible_anchor(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan recommended source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "recommended.anchor"},
                flow_id="flow:replay-plan-recommended",
                step_id="step:recommended-anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )
        observe_checkpoint(source_run_id, "ckpt-recommended-1", step_name="recommended.anchor", state_ref="state://recommended/1")

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(["trace", "replay-plan", source_run_id, "--recommended", "--json"])
        self.assertEqual(0, plan_code)
        payload = json.loads(plan_buf.getvalue().strip())
        self.assertEqual("ckpt-recommended-1", payload.get("from_checkpoint_id"))
        recommendation = payload.get("recommendation_used") or {}
        self.assertEqual("ckpt-recommended-1", recommendation.get("anchor_id"))
        self.assertEqual("stable_anchor", recommendation.get("stability_rank"))
        self.assertTrue(list(recommendation.get("recommendation_rationale") or []))
        recommendation_context = payload.get("recommendation_context") or {}
        self.assertEqual("auto", recommendation_context.get("mode"))
        self.assertGreaterEqual(int(recommendation_context.get("evaluated_count") or 0), 1)
        self.assertIn("selected", recommendation_context)

    def test_trace_replay_plan_recommended_text_includes_selection_and_skips(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan recommended text source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "tool.request", "phase": "start"},
                flow_id="flow:replay-plan-recommended-text",
                step_id="step:tool-request-1",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )
        observe_checkpoint(source_run_id, "ckpt-recommended-text-1", step_name="recommended.text.anchor", state_ref="state://recommended/text/1")

        text_buf = StringIO()
        with redirect_stdout(text_buf):
            plan_code = observe_main(["trace", "replay-plan", source_run_id, "--recommended", "--explain"])
        self.assertEqual(0, plan_code)
        text = text_buf.getvalue()
        self.assertIn("recommendation_explain_mode=auto", text)
        self.assertIn("recommended_anchor=ckpt-recommended-text-1", text)
        self.assertIn("recommended_reason=", text)
        self.assertIn("recommended_skipped:", text)
        self.assertIn("selected_anchor=ckpt-recommended-text-1", text)

    def test_trace_replay_plan_explain_with_explicit_anchor_shows_advisory_context(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan explain explicit source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "tool.result", "phase": "end"},
                flow_id="flow:replay-plan-explain-explicit",
                step_id="step:explicit-anchor-1",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )

        text_buf = StringIO()
        with redirect_stdout(text_buf):
            plan_code = observe_main(
                ["trace", "replay-plan", source_run_id, "--from-step-id", "step:explicit-anchor-1", "--explain"]
            )
        self.assertEqual(0, plan_code)
        text = text_buf.getvalue()
        self.assertIn("recommendation_explain_mode=advisory", text)
        self.assertIn("selected_anchor=step:explicit-anchor-1", text)

    def test_trace_replay_plan_recommended_and_explicit_anchor_conflict(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan recommended conflict source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(
                ["trace", "replay-plan", source_run_id, "--recommended", "--from-step-id", "step:any", "--json"]
            )
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("--recommended cannot be combined with explicit anchor flags", str(error_payload.get("error") or ""))

    def test_trace_replay_plan_recommended_without_admissible_anchor_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan recommended missing source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(["trace", "replay-plan", source_run_id, "--recommended", "--json"])
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("no admissible recommended replay anchors found on source run", str(error_payload.get("error") or ""))

    def test_trace_replay_plan_recommended_respects_deterministic_admission(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan recommended deterministic source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "weak.anchor"},
                flow_id="flow:replay-plan-recommended-deterministic",
                step_id="step:weak-deterministic-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        with self.assertRaises(SystemExit) as ctx:
            observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--recommended",
                    "--validation-mode",
                    "deterministic",
                    "--explain",
                ]
            )
        self.assertIn("no admissible recommended replay anchors found on source run", str(ctx.exception))

    def test_trace_artifacts_ls_and_show_by_id_json(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('artifact registry source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "artifact.registry.anchor"},
                flow_id="flow:artifact-registry",
                step_id="step:artifact-registry-anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:artifact-registry-anchor-1",
                    "--emit-artifact",
                    "--json",
                ]
            )
        self.assertEqual(0, plan_code)
        plan_payload = json.loads(plan_buf.getvalue().strip())
        artifact_path = str(plan_payload.get("artifact_path") or "").strip()
        self.assertTrue(artifact_path)
        artifact_payload = json.loads(Path(artifact_path).read_text(encoding="utf-8"))
        artifact_id = str(artifact_payload.get("artifact_id") or "").strip()
        self.assertTrue(artifact_id)

        ls_buf = StringIO()
        with redirect_stdout(ls_buf):
            ls_code = observe_main(["trace", "artifacts", "ls", "--type", "replay_plan", "--limit", "20", "--json"])
        self.assertEqual(0, ls_code)
        rows = [json.loads(line) for line in ls_buf.getvalue().splitlines() if line.strip()]
        self.assertTrue(any(str(row.get("artifact_id") or "") == artifact_id for row in rows))

        show_buf = StringIO()
        with redirect_stdout(show_buf):
            show_code = observe_main(["trace", "artifacts", "show", artifact_id, "--json"])
        self.assertEqual(0, show_code)
        shown = json.loads(show_buf.getvalue().strip())
        self.assertEqual(artifact_id, shown.get("artifact_id"))
        self.assertEqual("replay_plan", shown.get("artifact_type"))

    def test_trace_artifacts_show_missing_fails(self):
        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "artifacts", "show", "replay-plan:missing"])
        self.assertIn("artifact not found", str(ctx.exception))

    def test_trace_artifacts_replay_executes_from_artifact_id(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('artifact replay source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "artifact.replay.anchor"},
                flow_id="flow:artifact-replay",
                step_id="step:artifact-replay-anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )
        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:artifact-replay-anchor-1",
                    "--emit-artifact",
                    "--json",
                ]
            )
        self.assertEqual(0, plan_code)
        plan_payload = json.loads(plan_buf.getvalue().strip())
        artifact_path = str(plan_payload.get("artifact_path") or "").strip()
        artifact_payload = json.loads(Path(artifact_path).read_text(encoding="utf-8"))
        artifact_id = str(artifact_payload.get("artifact_id") or "").strip()
        self.assertTrue(artifact_id)

        replay_buf = StringIO()
        with redirect_stdout(replay_buf):
            replay_code = observe_main(["trace", "artifacts", "replay", artifact_id, "--no-open"])
        self.assertEqual(0, replay_code)
        self.assertIn("replay_of=", replay_buf.getvalue())
        self.assertIn("validation_mode=tolerant", replay_buf.getvalue())

    def test_trace_artifacts_replay_rejects_preview_only_plan(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('artifact replay preview source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        for step_id in ("step:artifact-preview-a", "step:artifact-preview-b"):
            tracing_module.emit_trace(
                tracing_module.build_trace_event(
                    event_type="observe_step",
                    run_id=source_run_id,
                    outcome="success",
                    payload={"step_name": "artifact.preview.select"},
                    flow_id="flow:artifact-preview",
                    step_id=step_id,
                    attempt=1,
                    step_type="reason",
                ),
                persist_db=False,
                persist_file=True,
            )

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:artifact-preview-a",
                    "--select-step",
                    "step:artifact-preview-a",
                    "--emit-artifact",
                    "--json",
                ]
            )
        self.assertEqual(0, plan_code)
        plan_payload = json.loads(plan_buf.getvalue().strip())
        artifact_path = str(plan_payload.get("artifact_path") or "").strip()
        artifact_payload = json.loads(Path(artifact_path).read_text(encoding="utf-8"))
        artifact_id = str(artifact_payload.get("artifact_id") or "").strip()
        self.assertTrue(artifact_id)

        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "artifacts", "replay", artifact_id, "--no-open"])
        self.assertIn("preview-only branch selection", str(ctx.exception))

    def test_trace_artifacts_replay_rejects_non_admitted_resume(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('artifact replay admission source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(["trace", "replay-plan", source_run_id, "--emit-artifact", "--json"])
        self.assertEqual(0, plan_code)
        plan_payload = json.loads(plan_buf.getvalue().strip())
        artifact_path = str(plan_payload.get("artifact_path") or "").strip()
        artifact_payload = json.loads(Path(artifact_path).read_text(encoding="utf-8"))
        artifact_id = str(artifact_payload.get("artifact_id") or "").strip()
        self.assertTrue(artifact_id)

        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "artifacts", "replay", artifact_id, "--no-open"])
        self.assertIn("artifact replay denied", str(ctx.exception))

    def test_trace_replay_plan_with_missing_checkpoint_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan bad checkpoint')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(
                ["trace", "replay-plan", source_run_id, "--from-checkpoint-id", "ckpt:not-there", "--json"]
            )
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("checkpoint_id not found on source run", str(error_payload.get("error") or ""))

    def test_trace_replay_plan_apply_executes_replay(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan apply source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "tool.result", "phase": "end"},
                flow_id="flow:replay-plan-apply",
                step_id="step:apply-anchor-1",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )

        apply_buf = StringIO()
        with redirect_stdout(apply_buf):
            apply_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:apply-anchor-1",
                    "--validation-mode",
                    "deterministic",
                    "--apply",
                    "--no-open",
                ]
            )
        self.assertEqual(0, apply_code)
        self.assertIn("replay_of=", apply_buf.getvalue())
        self.assertIn("from_step_id=step:apply-anchor-1", apply_buf.getvalue())
        self.assertIn("anchor_type=tool.result", apply_buf.getvalue())
        self.assertIn("validation_mode=deterministic", apply_buf.getvalue())
        self.assertIn("validation_status=pass", apply_buf.getvalue())

    def test_trace_replay_plan_apply_without_anchor_denied_by_resume_admission(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay apply no anchor source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "replay-plan", source_run_id, "--apply", "--no-open"])
        self.assertIn("replay admission denied", str(ctx.exception))

    def test_trace_replay_plan_apply_deterministic_with_weak_anchor_denied(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay apply weak deterministic source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "weak.anchor"},
                flow_id="flow:replay-plan-apply-weak",
                step_id="step:apply-weak-anchor-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        with self.assertRaises(SystemExit) as ctx:
            observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:apply-weak-anchor-1",
                    "--validation-mode",
                    "deterministic",
                    "--apply",
                    "--no-open",
                ]
            )
        self.assertIn("replay admission denied", str(ctx.exception))

    def test_trace_replay_plan_deterministic_without_anchor_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay deterministic no-anchor source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(["trace", "replay-plan", source_run_id, "--validation-mode", "deterministic", "--json"])
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn(
            "deterministic validation-mode requires --from-step-id or --from-checkpoint-id",
            str(error_payload.get("error") or ""),
        )

    def test_trace_replay_plan_select_step_preview_json(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan select source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        for step_id in ("step:select-a", "step:select-b"):
            tracing_module.emit_trace(
                tracing_module.build_trace_event(
                    event_type="observe_step",
                    run_id=source_run_id,
                    outcome="success",
                    payload={"step_name": "plan.select"},
                    flow_id="flow:replay-plan-select",
                    step_id=step_id,
                    attempt=1,
                    step_type="reason",
                ),
                persist_db=False,
                persist_file=True,
            )

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:select-a",
                    "--select-step",
                    "step:select-a",
                    "--select-step",
                    "step:select-b",
                    "--json",
                ]
            )
        self.assertEqual(0, code)
        payload = json.loads(plan_buf.getvalue().strip())
        branch = payload.get("branch_preview") or {}
        self.assertTrue(bool(branch.get("preview_only")))
        self.assertEqual("selected_subgraph_preview", branch.get("mode"))
        self.assertEqual(["step:select-a", "step:select-b"], branch.get("selected_step_ids"))
        sel = branch.get("selection_validation") or {}
        self.assertEqual("step:select-a", sel.get("anchor_step_id"))
        self.assertEqual(2, int(sel.get("selected_count") or 0))

    def test_trace_replay_plan_select_step_apply_rejected(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan select apply source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "plan.select.apply"},
                flow_id="flow:replay-plan-select-apply",
                step_id="step:select-only",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        with self.assertRaises(SystemExit) as ctx:
            observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:select-only",
                    "--select-step",
                    "step:select-only",
                    "--apply",
                ]
            )
        self.assertIn("--select-step is preview-only", str(ctx.exception))

    def test_trace_replay_plan_select_step_missing_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan select missing source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(["trace", "replay-plan", source_run_id, "--select-step", "step:not-there", "--json"])
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("--select-step currently requires --from-step-id", str(error_payload.get("error") or ""))

    def test_trace_replay_plan_select_step_requires_from_step_id(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan select require anchor source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(["trace", "replay-plan", source_run_id, "--select-step", "step:any", "--json"])
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("--select-step currently requires --from-step-id", str(error_payload.get("error") or ""))

    def test_trace_replay_plan_select_step_with_checkpoint_anchor_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan select checkpoint source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)
        observe_checkpoint(source_run_id, "ckpt-select-1", step_name="select.ckpt", state_ref="state://select/1")

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-checkpoint-id",
                    "ckpt-select-1",
                    "--select-step",
                    "step:any",
                    "--json",
                ]
            )
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("--select-step cannot be used with --from-checkpoint-id", str(error_payload.get("error") or ""))

    def test_trace_replay_plan_select_step_missing_after_anchor_validation_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan select missing anchored source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "select.anchor"},
                flow_id="flow:replay-plan-select-missing",
                step_id="step:anchor-ok",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )

        error_buf = StringIO()
        with redirect_stdout(error_buf):
            error_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--from-step-id",
                    "step:anchor-ok",
                    "--select-step",
                    "step:not-there",
                    "--json",
                ]
            )
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("selected step_id not found on source run", str(error_payload.get("error") or ""))

    def test_trace_replay_plan_apply_and_json_flags_conflict(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan conflict source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "replay-plan", source_run_id, "--apply", "--json"])
        self.assertIn("Choose only one of --apply or --json", str(ctx.exception))

    def test_trace_replay_plan_invalid_validation_mode_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan invalid mode source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        with self.assertRaises(SystemExit) as ctx:
            observe_main(["trace", "replay-plan", source_run_id, "--validation-mode", "strict", "--json"])
        self.assertEqual(2, int(getattr(ctx.exception, "code", 0) or 0))

    def test_trace_anchors_lists_step_and_checkpoint_ids(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('anchors source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "anchors.step"},
                flow_id="flow:anchors",
                step_id="step:anchors-1",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )
        observe_checkpoint(run_id, "ckpt-anchors-1", step_name="anchors.step", state_ref="state://anchors/1")

        anchors_buf = StringIO()
        with redirect_stdout(anchors_buf):
            anchors_code = observe_main(["trace", "anchors", run_id, "--json"])
        self.assertEqual(0, anchors_code)
        payload = json.loads(anchors_buf.getvalue().strip())
        self.assertEqual(run_id, payload.get("run_id"))
        self.assertIn("step:anchors-1", payload.get("step_ids") or [])
        self.assertIn("ckpt-anchors-1", payload.get("checkpoint_ids") or [])
        self.assertGreaterEqual(int(payload.get("anchor_count") or 0), 2)
        self.assertIn("anchors", payload)
        self.assertIn("step_anchor_types", payload)
        self.assertIn("anchor_type_counts", payload)
        self.assertIn("recommended_replay_anchors", payload)
        step_types = (payload.get("step_anchor_types") or {}).get("step:anchors-1") or []
        self.assertIn("step.generic", step_types)
        self.assertIn("checkpoint.state", set((payload.get("anchor_type_counts") or {}).keys()))
        recommended = list(payload.get("recommended_replay_anchors") or [])
        self.assertTrue(recommended)
        self.assertTrue(any(bool(item.get("admission_allowed")) for item in recommended))
        self.assertTrue(any(str(item.get("stability_rank") or "") == "stable_anchor" for item in recommended))
        first = recommended[0]
        self.assertIn("boundary_strength", first)
        self.assertIn("idempotency_risk", first)
        self.assertIn("side_effect", first)
        self.assertIn("external_write", first)
        self.assertIn("failure_proximity", first)
        self.assertIn("failure_proximity_score", first)
        self.assertIn("replay_utility_score", first)
        self.assertIn("continuation_viability", first)
        self.assertIn("replay_utility_rationale", first)
        self.assertIn("recommendation_rationale", first)

    def test_trace_anchors_recommendations_include_failure_proximity(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('anchors failure proximity source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "anchors.before_failure"},
                flow_id="flow:anchors-proximity",
                step_id="step:before-failure",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )
        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="failed",
                payload={"step_name": "anchors.failure"},
                flow_id="flow:anchors-proximity",
                step_id="step:failure",
                attempt=1,
                step_type="reason",
            ),
            persist_db=False,
            persist_file=True,
        )
        observe_checkpoint(run_id, "ckpt-after-failure", step_name="anchors.after_failure", state_ref="state://anchors/failure")

        anchors_buf = StringIO()
        with redirect_stdout(anchors_buf):
            anchors_code = observe_main(["trace", "anchors", run_id, "--json"])
        self.assertEqual(0, anchors_code)
        payload = json.loads(anchors_buf.getvalue().strip())
        recommended = list(payload.get("recommended_replay_anchors") or [])
        self.assertTrue(recommended)
        self.assertTrue(any(item.get("failure_proximity_score") is not None for item in recommended))

    def test_trace_anchors_side_effect_signals_influence_recommendations(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('anchors side effect source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={
                    "step_name": "tool.result",
                    "phase": "end",
                    "side_effect": True,
                    "external_write": True,
                    "idempotency_hint": "high",
                },
                flow_id="flow:anchors-side-effects",
                step_id="step:side-effect-1",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )

        anchors_buf = StringIO()
        with redirect_stdout(anchors_buf):
            anchors_code = observe_main(["trace", "anchors", run_id, "--json"])
        self.assertEqual(0, anchors_code)
        payload = json.loads(anchors_buf.getvalue().strip())
        recommended = list(payload.get("recommended_replay_anchors") or [])
        self.assertTrue(recommended)
        hit = next((item for item in recommended if str(item.get("from_step_id") or "") == "step:side-effect-1"), {})
        self.assertTrue(hit)
        self.assertEqual("high", str(hit.get("idempotency_risk") or ""))
        self.assertTrue(bool(hit.get("side_effect")))
        self.assertTrue(bool(hit.get("external_write")))
        self.assertFalse(bool(hit.get("admission_allowed")))
        self.assertTrue(
            any("external writes require checkpoint-level state" in str(reason) for reason in (hit.get("admission_reasons") or []))
        )

    def test_trace_anchors_invocation_failure_lowers_replay_utility(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('anchors invocation-failure source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_output",
                run_id=run_id,
                outcome="failed",
                payload={
                    "step_name": "command.output",
                    "stream": "stderr",
                    "seq": 1,
                    "text": "error: required option '-m, --message <text>' not specified",
                },
                step_id="step:openclaw.output",
                step_type="output",
            ),
            persist_db=False,
            persist_file=True,
        )
        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="failed",
                payload={
                    "step_name": "command.finish",
                    "exit_code": 1,
                    "canonical_anchor_type": "output.commit",
                },
                step_id="step:openclaw.command",
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )

        anchors_buf = StringIO()
        with redirect_stdout(anchors_buf):
            anchors_code = observe_main(["trace", "anchors", run_id, "--json"])
        self.assertEqual(0, anchors_code)
        payload = json.loads(anchors_buf.getvalue().strip())
        recommended = list(payload.get("recommended_replay_anchors") or [])
        self.assertTrue(recommended)
        weak = next((item for item in recommended if str(item.get("from_step_id") or "") == "step:openclaw.output"), {})
        self.assertTrue(weak)
        self.assertLess(float(weak.get("replay_utility_score") or 1.0), 0.5)
        self.assertIn(str(weak.get("continuation_viability") or ""), {"minimal", "low"})
        self.assertTrue(
            any("invocation failure signals detected" in str(reason) for reason in (weak.get("replay_utility_rationale") or []))
        )

    def test_trace_anchors_checkpoint_preferred_for_side_effecting_paths(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('anchors checkpoint utility source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={
                    "step_name": "tool.result",
                    "phase": "end",
                    "side_effect": True,
                    "external_write": True,
                    "idempotency_hint": "medium",
                },
                flow_id="flow:anchors-checkpoint-utility",
                step_id="step:side-effect-path",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )
        observe_checkpoint(
            run_id,
            "ckpt-side-effect-path",
            step_name="tool.result",
            state_ref="state://side-effect/path/1",
        )

        anchors_buf = StringIO()
        with redirect_stdout(anchors_buf):
            anchors_code = observe_main(["trace", "anchors", run_id, "--json"])
        self.assertEqual(0, anchors_code)
        payload = json.loads(anchors_buf.getvalue().strip())
        recommended = list(payload.get("recommended_replay_anchors") or [])
        self.assertTrue(recommended)

        checkpoint_item = next(
            (item for item in recommended if str(item.get("from_checkpoint_id") or "") == "ckpt-side-effect-path"),
            {},
        )
        step_item = next(
            (
                item
                for item in recommended
                if str(item.get("from_step_id") or "") == "step:side-effect-path"
                and not str(item.get("from_checkpoint_id") or "")
            ),
            {},
        )
        self.assertTrue(checkpoint_item)
        self.assertTrue(step_item)
        self.assertGreater(
            float(checkpoint_item.get("replay_utility_score") or 0.0),
            float(step_item.get("replay_utility_score") or 0.0),
        )
        self.assertIn(
            str(checkpoint_item.get("continuation_viability") or ""),
            {"high", "medium"},
        )

    def test_trace_replay_plan_recommended_deterministic_prefers_checkpoint_on_side_effecting_paths(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(
                ["trace", "run", "--no-open", "--", "python", "-c", "print('replay deterministic checkpoint source')"]
            )
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={
                    "step_name": "tool.result",
                    "phase": "end",
                    "side_effect": True,
                    "external_write": True,
                    "idempotency_hint": "medium",
                },
                flow_id="flow:replay-plan-deterministic-checkpoint",
                step_id="step:deterministic-side-effect",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )
        observe_checkpoint(
            source_run_id,
            "ckpt-deterministic-side-effect",
            step_name="tool.result",
            state_ref="state://deterministic/side-effect/1",
        )

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--recommended",
                    "--validation-mode",
                    "deterministic",
                    "--json",
                ]
            )
        self.assertEqual(0, plan_code)
        payload = json.loads(plan_buf.getvalue().strip())
        self.assertEqual("ckpt-deterministic-side-effect", str(payload.get("from_checkpoint_id") or ""))
        recommendation = payload.get("recommendation_used") if isinstance(payload.get("recommendation_used"), dict) else {}
        self.assertEqual("checkpoint_id", str(recommendation.get("anchor_ref_type") or ""))
        self.assertGreater(float(recommendation.get("replay_utility_score") or 0.0), 0.5)

    def test_trace_replay_plan_recommended_deterministic_side_effecting_without_checkpoint_fails(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(
                ["trace", "run", "--no-open", "--", "python", "-c", "print('replay deterministic side-effect no-checkpoint source')"]
            )
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={
                    "step_name": "tool.result",
                    "phase": "end",
                    "side_effect": True,
                    "external_write": True,
                    "idempotency_hint": "medium",
                },
                flow_id="flow:replay-plan-deterministic-no-checkpoint",
                step_id="step:deterministic-no-checkpoint",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )

        error_json_buf = StringIO()
        with redirect_stdout(error_json_buf):
            error_code = observe_main(
                [
                    "trace",
                    "replay-plan",
                    source_run_id,
                    "--recommended",
                    "--validation-mode",
                    "deterministic",
                    "--json",
                ]
            )
        self.assertEqual(1, error_code)
        error_payload = json.loads(error_json_buf.getvalue().strip())
        self.assertFalse(bool(error_payload.get("ok")))
        self.assertEqual("replay_plan_resolution_failed", str(error_payload.get("reason") or ""))
        self.assertIn("requires checkpoint anchor", str(error_payload.get("error") or ""))
        self.assertIn("hive trace checkpoint", str(error_payload.get("suggested_checkpoint_command") or ""))

    def test_trace_checkpoint_emits_anchor_for_existing_run(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('checkpoint command source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        checkpoint_buf = StringIO()
        with redirect_stdout(checkpoint_buf):
            checkpoint_code = observe_main(
                [
                    "trace",
                    "checkpoint",
                    source_run_id,
                    "ckpt:cli:1",
                    "--step-name",
                    "cli.checkpoint",
                    "--state-ref",
                    "state://cli/checkpoint/1",
                    "--json",
                ]
            )
        self.assertEqual(0, checkpoint_code)
        payload = json.loads(checkpoint_buf.getvalue().strip())
        self.assertTrue(payload.get("ok"))
        self.assertEqual(source_run_id, payload.get("run_id"))
        self.assertEqual("ckpt:cli:1", payload.get("checkpoint_id"))

        anchors_buf = StringIO()
        with redirect_stdout(anchors_buf):
            anchors_code = observe_main(["trace", "anchors", source_run_id, "--json"])
        self.assertEqual(0, anchors_code)
        anchors_payload = json.loads(anchors_buf.getvalue().strip())
        self.assertIn("ckpt:cli:1", list(anchors_payload.get("checkpoint_ids") or []))

    def test_trace_checkpoint_auto_id_from_step_id_derives_context(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('checkpoint auto-id source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "tool.result", "phase": "end"},
                flow_id="flow:checkpoint-auto-id",
                step_id="step:auto-anchor",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )

        checkpoint_buf = StringIO()
        with redirect_stdout(checkpoint_buf):
            checkpoint_code = observe_main(
                [
                    "trace",
                    "checkpoint",
                    source_run_id,
                    "--auto-id",
                    "--from-step-id",
                    "step:auto-anchor",
                    "--json",
                ]
            )
        self.assertEqual(0, checkpoint_code)
        payload = json.loads(checkpoint_buf.getvalue().strip())
        checkpoint_id = str(payload.get("checkpoint_id") or "")
        self.assertTrue(checkpoint_id.startswith("ckpt:"))
        self.assertEqual("step:auto-anchor", str(payload.get("from_step_id") or ""))
        self.assertEqual("tool.result", str(payload.get("step_name") or ""))
        self.assertEqual("step://step:auto-anchor", str(payload.get("state_ref") or ""))

    def test_trace_replay_plan_infers_strong_anchor_type_from_step(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('replay plan anchor infer source')"])
        self.assertEqual(0, code)
        source_run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                source_run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(source_run_id)

        tracing_module.emit_trace(
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=source_run_id,
                outcome="success",
                payload={"step_name": "tool.result", "phase": "end"},
                flow_id="flow:replay-plan-infer",
                step_id="step:tool-infer-1",
                attempt=1,
                step_type="tool_call",
            ),
            persist_db=False,
            persist_file=True,
        )

        plan_buf = StringIO()
        with redirect_stdout(plan_buf):
            plan_code = observe_main(
                ["trace", "replay-plan", source_run_id, "--from-step-id", "step:tool-infer-1", "--json"]
            )
        self.assertEqual(0, plan_code)
        payload = json.loads(plan_buf.getvalue().strip())
        self.assertEqual("tool.result", payload.get("anchor_type"))
        stability = ((payload.get("resume_envelope") or {}).get("stability") or {})
        self.assertEqual("stable_anchor", stability.get("rank"))
        self.assertGreater(float(stability.get("boundary_strength") or 0.0), 0.7)

    def test_trace_diagnose_json_for_failed_run(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(
                [
                    "trace",
                    "run",
                    "--no-open",
                    "--",
                    "python",
                    "-c",
                    "import sys; sys.stderr.write('boom\\n'); raise SystemExit(3)",
                ]
            )
        self.assertEqual(3, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        diag_buf = StringIO()
        with redirect_stdout(diag_buf):
            diag_code = observe_main(["trace", "diagnose", run_id, "--json"])
        self.assertEqual(0, diag_code)
        payload = json.loads(diag_buf.getvalue().strip())
        self.assertEqual(run_id, payload["run_id"])
        self.assertEqual("failed", payload["status"])
        self.assertIn(payload["category"], {"nonzero_exit", "stderr_heavy"})
        self.assertGreaterEqual(len(payload.get("recommendations") or []), 1)

    def test_trace_diagnose_json_for_success_run(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('healthy')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        diag_buf = StringIO()
        with redirect_stdout(diag_buf):
            diag_code = observe_main(["trace", "diagnose", run_id, "--json"])
        self.assertEqual(0, diag_code)
        payload = json.loads(diag_buf.getvalue().strip())
        self.assertEqual("success", payload["status"])
        self.assertEqual("success", payload["category"])

    def test_trace_reconcile_marks_stale_running_run(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('reconcile-source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        run_file = Path(self._trace_home) / "runs" / f"{run_id.replace(':', '_')}.json"
        record = json.loads(run_file.read_text(encoding="utf-8"))
        old_ms = int(record.get("started_at_ms") or 0) - (10 * 60 * 1000)
        record["status"] = "running"
        record["exit_code"] = None
        record["finished_at_ms"] = None
        record["duration_ms"] = None
        record["last_update_ms"] = old_ms
        run_file.write_text(json.dumps(record), encoding="utf-8")

        reconcile_buf = StringIO()
        with redirect_stdout(reconcile_buf):
            reconcile_code = observe_main(["trace", "ops", "reconcile", "--stale-after", "1s", "--json"])
        self.assertEqual(0, reconcile_code)
        payload = json.loads(reconcile_buf.getvalue().strip())
        self.assertGreaterEqual(int(payload.get("reconciled_runs") or 0), 1)
        self.assertIn(run_id, list(payload.get("reconciled_run_ids") or []))

        refreshed = json.loads(run_file.read_text(encoding="utf-8"))
        self.assertEqual("failed", refreshed.get("status"))
        self.assertTrue(bool(refreshed.get("stale_reconciled")))

    def test_trace_ls_auto_reconciles_stale_running_run(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('auto-reconcile-source')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        run_file = Path(self._trace_home) / "runs" / f"{run_id.replace(':', '_')}.json"
        record = json.loads(run_file.read_text(encoding="utf-8"))
        old_ms = int(record.get("started_at_ms") or 0) - (10 * 60 * 1000)
        record["status"] = "running"
        record["exit_code"] = None
        record["finished_at_ms"] = None
        record["duration_ms"] = None
        record["last_update_ms"] = old_ms
        run_file.write_text(json.dumps(record), encoding="utf-8")
        config_module.TRACE_RECONCILE_STALE_SEC = 1

        ls_buf = StringIO()
        with redirect_stdout(ls_buf):
            ls_code = observe_main(["trace", "ls", "--status", "failed", "--limit", "20"])
        self.assertEqual(0, ls_code)
        self.assertIn(run_id, ls_buf.getvalue())

    def test_trace_ls_includes_liveness_for_running_records(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('ls-liveness')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        run_file = Path(self._trace_home) / "runs" / f"{run_id.replace(':', '_')}.json"
        record = json.loads(run_file.read_text(encoding="utf-8"))
        now_ms = int(time.time() * 1000)
        record["status"] = "running"
        record["exit_code"] = None
        record["finished_at_ms"] = None
        record["duration_ms"] = None
        record["last_update_ms"] = now_ms
        run_file.write_text(json.dumps(record), encoding="utf-8")
        config_module.TRACE_RECONCILE_STALE_SEC = 3600

        ls_buf = StringIO()
        with redirect_stdout(ls_buf):
            ls_code = observe_main(["trace", "ls", "--status", "running", "--limit", "20"])
        self.assertEqual(0, ls_code)
        text = ls_buf.getvalue()
        self.assertIn(run_id, text)
        self.assertIn("active", text)

        ls_json_buf = StringIO()
        with redirect_stdout(ls_json_buf):
            ls_json_code = observe_main(["trace", "ls", "--status", "running", "--limit", "20", "--json"])
        self.assertEqual(0, ls_json_code)
        rows = [json.loads(line) for line in ls_json_buf.getvalue().splitlines() if line.strip()]
        self.assertGreaterEqual(len(rows), 1)
        row = next((item for item in rows if str(item.get("run_id") or "") == run_id), None)
        self.assertIsNotNone(row)
        self.assertEqual("running_active", row.get("runtime_state"))
        self.assertIsNotNone(row.get("last_activity_ms"))
        self.assertIsNotNone(row.get("last_activity_age_ms"))
        self.assertIsNotNone(row.get("last_heartbeat_ms"))
        self.assertIsNotNone(row.get("last_heartbeat_age_ms"))

    def test_trace_archive_unarchive_and_status_filter(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('archive-test')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        archive_buf = StringIO()
        with redirect_stdout(archive_buf):
            archive_code = observe_main(["trace", "ops", "archive", run_id, "--reason", "unit-test"])
        self.assertEqual(0, archive_code)
        self.assertIn(f"archived={run_id}", archive_buf.getvalue())

        ls_archived = StringIO()
        with redirect_stdout(ls_archived):
            ls_archived_code = observe_main(["trace", "ls", "--status", "archived", "--limit", "20"])
        self.assertEqual(0, ls_archived_code)
        self.assertIn(run_id, ls_archived.getvalue())

        ls_active = StringIO()
        with redirect_stdout(ls_active):
            ls_active_code = observe_main(["trace", "ls", "--status", "active", "--limit", "20"])
        self.assertEqual(0, ls_active_code)
        self.assertNotIn(run_id, ls_active.getvalue())

        unarchive_buf = StringIO()
        with redirect_stdout(unarchive_buf):
            unarchive_code = observe_main(["trace", "ops", "unarchive", run_id])
        self.assertEqual(0, unarchive_code)
        self.assertIn(f"unarchived={run_id}", unarchive_buf.getvalue())

    def test_trace_prune_removes_old_archived_runs(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('prune-test')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        archive_buf = StringIO()
        with redirect_stdout(archive_buf):
            observe_main(["trace", "ops", "archive", run_id, "--reason", "prune-window"])
        run_file = Path(self._trace_home) / "runs" / f"{run_id.replace(':', '_')}.json"
        record = json.loads(run_file.read_text(encoding="utf-8"))
        old_ms = int(record.get("started_at_ms") or 0) - (3 * 24 * 60 * 60 * 1000)
        record["started_at_ms"] = old_ms
        record["finished_at_ms"] = old_ms
        record["archived_at_ms"] = old_ms
        run_file.write_text(json.dumps(record), encoding="utf-8")

        prune_buf = StringIO()
        with redirect_stdout(prune_buf):
            prune_code = observe_main(["trace", "ops", "prune", "--older-than", "1d"])
        self.assertEqual(0, prune_code)
        self.assertIn("pruned_runs=", prune_buf.getvalue())
        self.assertFalse(run_file.exists())

    def test_quickstart_command(self):
        buf = StringIO()
        with redirect_stdout(buf):
            code = observe_main(["quickstart", "--no-open"])
        self.assertEqual(0, code)
        text = buf.getvalue()
        self.assertIn("quickstart_complete=true", text)
        self.assertIn("run_id=observe-run:", text)
        self.assertIn("next=hive trace ls", text)

    def test_doctor_command(self):
        buf = StringIO()
        with redirect_stdout(buf):
            code = observe_main(["doctor"])
        self.assertIn(code, (0, 2))
        text = buf.getvalue()
        self.assertIn("[", text)
        self.assertIn("python", text)
        self.assertIn("trace_home", text)

    def test_version_flag(self):
        buf = StringIO()
        with self.assertRaises(SystemExit) as ctx:
            with redirect_stdout(buf):
                observe_main(["--version"])
        self.assertEqual(0, int(ctx.exception.code))
        self.assertRegex(buf.getvalue().strip(), r"^hive\s+\S+$")

    def test_help_verbose_for_replay_plan_without_run_id(self):
        buf = StringIO()
        with redirect_stdout(buf):
            code = observe_main(["trace", "replay-plan", "--help-verbose"])
        self.assertEqual(0, code)
        text = buf.getvalue()
        self.assertIn("hive trace replay-plan (verbose)", text)
        self.assertIn("Preview replay intent", text)
        self.assertIn("--apply", text)

    def test_help_verbose_alias_with_help_and_v(self):
        buf = StringIO()
        with redirect_stdout(buf):
            code = observe_main(["trace", "replay-plan", "--help", "-v"])
        self.assertEqual(0, code)
        text = buf.getvalue()
        self.assertIn("hive trace replay-plan (verbose)", text)
        self.assertIn("anchors command", text)

    def test_help_verbose_for_trace_insight_namespace(self):
        buf = StringIO()
        with redirect_stdout(buf):
            code = observe_main(["trace", "insight", "--help-verbose"])
        self.assertEqual(0, code)
        text = buf.getvalue()
        self.assertIn("hive trace insight (verbose)", text)
        self.assertIn("composes primitives", text.lower())

    def test_help_verbose_for_trace_artifacts_namespace(self):
        buf = StringIO()
        with redirect_stdout(buf):
            code = observe_main(["trace", "artifacts", "--help-verbose"])
        self.assertEqual(0, code)
        text = buf.getvalue()
        self.assertIn("hive trace artifacts (verbose)", text)
        self.assertIn("replay-plan manifests", text)

    def test_trace_insight_explain_macro_output(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('insight-macro')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        explain_buf = StringIO()
        with redirect_stdout(explain_buf):
            explain_code = observe_main(["trace", "insight", "explain", run_id, "--json"])
        self.assertEqual(0, explain_code)
        payload = json.loads(explain_buf.getvalue().strip())
        self.assertEqual("macro", payload.get("mode"))
        self.assertEqual("trace.insight.explain", payload.get("macro"))
        self.assertEqual(run_id, payload.get("run_id"))
        self.assertTrue(payload.get("derived_from"))
        self.assertTrue(payload.get("actions"))

    def test_trace_insight_drift_macro_output(self):
        run_ids = []
        for label in ("drift-a", "drift-b"):
            run_buf = StringIO()
            with redirect_stdout(run_buf):
                code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", f"print('{label}')"])
            self.assertEqual(0, code)
            rid = ""
            for line in run_buf.getvalue().splitlines():
                if line.startswith("run_id="):
                    rid = line.split()[0].split("=", 1)[1].strip()
                    break
            self.assertTrue(rid)
            run_ids.append(rid)

        drift_buf = StringIO()
        with redirect_stdout(drift_buf):
            drift_code = observe_main(["trace", "insight", "drift", run_ids[0], run_ids[1], "--json"])
        self.assertEqual(0, drift_code)
        payload = json.loads(drift_buf.getvalue().strip())
        self.assertEqual("macro", payload.get("mode"))
        self.assertEqual("trace.insight.drift", payload.get("macro"))
        self.assertEqual(run_ids[0], payload.get("run_id_a"))
        self.assertEqual(run_ids[1], payload.get("run_id_b"))
        self.assertTrue(payload.get("answer"))
        self.assertTrue(payload.get("derived_from"))

    def test_trace_insight_health_macro_output(self):
        for snippet in ("print('health-ok')", "import sys; sys.stderr.write('health-fail\\n'); raise SystemExit(2)"):
            run_buf = StringIO()
            with redirect_stdout(run_buf):
                code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", snippet])
            if "SystemExit(2)" in snippet:
                self.assertEqual(2, code)
            else:
                self.assertEqual(0, code)

        health_buf = StringIO()
        with redirect_stdout(health_buf):
            health_code = observe_main(["trace", "insight", "health", "--window", "24h", "--json"])
        self.assertEqual(0, health_code)
        payload = json.loads(health_buf.getvalue().strip())
        self.assertEqual("macro", payload.get("mode"))
        self.assertEqual("trace.insight.health", payload.get("macro"))
        self.assertIn("summary", payload)
        self.assertIn("status_counts", payload.get("summary") or {})
        self.assertTrue(payload.get("derived_from"))
        self.assertTrue(payload.get("actions"))
        self.assertIn("failed_run_ids", payload)
        self.assertIn("failure_inspect_commands", payload)
        self.assertIn("top_failed_commands", payload)
        self.assertGreaterEqual(len(payload.get("failed_run_ids") or []), 1)
        self.assertTrue(any(str(cmd).startswith("hive trace diagnose ") for cmd in (payload.get("failure_inspect_commands") or [])))
        self.assertGreaterEqual(len(payload.get("top_failed_commands") or []), 1)

    def test_trace_ops_archive_alias(self):
        run_buf = StringIO()
        with redirect_stdout(run_buf):
            code = observe_main(["trace", "run", "--no-open", "--", "python", "-c", "print('ops-alias')"])
        self.assertEqual(0, code)
        run_id = ""
        for line in run_buf.getvalue().splitlines():
            if line.startswith("run_id="):
                run_id = line.split()[0].split("=", 1)[1].strip()
                break
        self.assertTrue(run_id)

        archive_buf = StringIO()
        with redirect_stdout(archive_buf):
            archive_code = observe_main(["trace", "ops", "archive", run_id, "--reason", "ops-alias-test"])
        self.assertEqual(0, archive_code)
        self.assertIn(f"archived={run_id}", archive_buf.getvalue())

    def test_proxy_passthrough_emits_proxy_events(self):
        class UpstreamHandler(BaseHTTPRequestHandler):
            def do_POST(self):  # noqa: N802
                length = int(self.headers.get("Content-Length") or 0)
                body = self.rfile.read(length) if length > 0 else b""
                payload = json.dumps({"ok": True, "size": len(body)}).encode("utf-8")
                self.send_response(200)
                self.send_header("Content-Type", "application/json")
                self.send_header("Content-Length", str(len(payload)))
                self.end_headers()
                self.wfile.write(payload)

            def log_message(self, fmt, *args):  # noqa: A003
                return

        upstream = ThreadingHTTPServer(("127.0.0.1", 0), UpstreamHandler)
        thread = threading.Thread(target=upstream.serve_forever, daemon=True)
        thread.start()
        host, port = upstream.server_address
        upstream_base = f"http://{host}:{port}/v1"
        run_id = f"observe-run:{uuid.uuid4().hex[:8]}"
        try:
            with observe_proxy(run_id=run_id, upstream_base=upstream_base) as proxy:
                body = json.dumps({"hello": "proxy"}).encode("utf-8")
                req = urllib.request.Request(
                    url=f"{proxy.url}/chat/completions",
                    data=body,
                    headers={"Content-Type": "application/json"},
                    method="POST",
                )
                with urllib.request.urlopen(req, timeout=5) as resp:
                    data = json.loads(resp.read().decode("utf-8"))
                    self.assertTrue(data.get("ok"))
        finally:
            upstream.shutdown()
            upstream.server_close()

        req_events = tracing_module.read_trace_events(
            limit=200,
            filters={"run_id": run_id, "event_type": "observe_proxy_request"},
        )
        res_events = tracing_module.read_trace_events(
            limit=200,
            filters={"run_id": run_id, "event_type": "observe_proxy_response"},
        )
        self.assertTrue(req_events)
        self.assertTrue(res_events)

    def test_trace_run_proxy_mode_prints_proxy_url(self):
        class UpstreamHandler(BaseHTTPRequestHandler):
            def do_POST(self):  # noqa: N802
                payload = json.dumps({"ok": True}).encode("utf-8")
                self.send_response(200)
                self.send_header("Content-Type", "application/json")
                self.send_header("Content-Length", str(len(payload)))
                self.end_headers()
                self.wfile.write(payload)

            def log_message(self, fmt, *args):  # noqa: A003
                return

        upstream = ThreadingHTTPServer(("127.0.0.1", 0), UpstreamHandler)
        thread = threading.Thread(target=upstream.serve_forever, daemon=True)
        thread.start()
        host, port = upstream.server_address
        upstream_base = f"http://{host}:{port}/v1"
        try:
            buf = StringIO()
            with redirect_stdout(buf):
                code = observe_main(
                    [
                        "trace",
                        "run",
                        "--no-open",
                        "--proxy",
                        "--proxy-upstream",
                        upstream_base,
                        "--",
                        "python",
                        "-c",
                        "import os; print('base_set', bool(os.getenv('OPENAI_BASE_URL')))",
                    ]
                )
            self.assertEqual(0, code)
            out = buf.getvalue()
            self.assertIn("proxy=http://127.0.0.1:", out)
            self.assertIn("base_set True", out)
        finally:
            upstream.shutdown()
            upstream.server_close()

    def test_trace_run_timeout_enforced_when_process_stays_alive(self):
        buf = StringIO()
        started = time.time()
        with redirect_stdout(buf):
            code = observe_main(
                [
                    "trace",
                    "run",
                    "--timeout-sec",
                    "1",
                    "--no-open",
                    "--",
                    "python",
                    "-c",
                    "import time; print('timeout-check'); time.sleep(10)",
                ]
            )
        elapsed = time.time() - started
        self.assertEqual(124, code)
        self.assertLess(elapsed, 6.0)
        text = buf.getvalue()
        self.assertIn("run_id=observe-run:", text)
        self.assertIn("status=failed", text)

    def test_trace_event_lineage_fields_are_accepted_and_filterable(self):
        run_id = f"observe-run:{uuid.uuid4().hex[:8]}"
        event = tracing_module.build_trace_event(
            event_type="observe_step",
            run_id=run_id,
            outcome="success",
            payload={"step_name": "lineage.step"},
            flow_id="flow:test-1",
            step_id="step:test-1",
            parent_step_id="step:root",
            tool_call_id="tool:test-1",
            attempt=2,
            step_type="tool_call",
        )
        tracing_module.emit_trace(event, persist_db=False, persist_file=True)
        matched = tracing_module.read_trace_events(
            limit=50,
            filters={"flow_id": "flow:test-1", "step_id": "step:test-1", "attempt": 2},
        )
        self.assertGreaterEqual(len(matched), 1)
        found = matched[-1]
        self.assertEqual("flow:test-1", found.get("flow_id"))
        self.assertEqual("step:test-1", found.get("step_id"))
        self.assertEqual(2, int(found.get("attempt") or 0))

    def test_trace_event_lineage_attempt_must_be_positive_int(self):
        with self.assertRaises(ValueError):
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=f"observe-run:{uuid.uuid4().hex[:8]}",
                outcome="success",
                payload={"step_name": "bad-attempt"},
                flow_id="flow:test-invalid",
                step_id="step:test-invalid",
                attempt=0,
                step_type="tool_call",
            )

    def test_structured_agent_tool_helper_events_emit_lineage(self):
        run_id = f"observe-run:{uuid.uuid4().hex[:8]}"
        flow_id = "flow:test-helper-events"
        observe_agent_step_start(
            run_id,
            "step:plan",
            flow_id=flow_id,
            step_name="planner",
            attempt=1,
            payload={"note": "planning"},
        )
        observe_tool_call_start(
            run_id,
            "tool:search-1",
            flow_id=flow_id,
            step_id="step:tool.search",
            parent_step_id="step:plan",
            tool_name="web.search",
            attempt=1,
        )
        observe_tool_call_end(
            run_id,
            "tool:search-1",
            flow_id=flow_id,
            step_id="step:tool.search",
            parent_step_id="step:plan",
            tool_name="web.search",
            attempt=1,
            outcome="success",
            payload={"result_count": 3},
        )
        observe_step_retry(
            run_id,
            "step:tool.search",
            flow_id=flow_id,
            parent_step_id="step:plan",
            attempt=2,
            reason="insufficient_result_quality",
        )
        observe_branch_decision(
            run_id,
            "step:branch.next",
            flow_id=flow_id,
            parent_step_id="step:plan",
            selected_branch="summarize",
            branches=["summarize", "retry_search"],
        )
        observe_agent_step_end(
            run_id,
            "step:plan",
            flow_id=flow_id,
            step_name="planner",
            attempt=1,
            outcome="success",
        )

        events = tracing_module.read_trace_events(limit=200, filters={"run_id": run_id, "flow_id": flow_id})
        self.assertGreaterEqual(len(events), 6)

        tool_events = [e for e in events if str(e.get("step_type") or "") == "tool_call"]
        self.assertGreaterEqual(len(tool_events), 2)
        phases = [str((e.get("payload") or {}).get("phase") or "") for e in tool_events]
        self.assertIn("start", phases)
        self.assertIn("end", phases)
        self.assertTrue(any(str(e.get("tool_call_id") or "") == "tool:search-1" for e in tool_events))
        start_payload = next((e.get("payload") or {} for e in tool_events if str((e.get("payload") or {}).get("phase") or "") == "start"), {})
        end_payload = next((e.get("payload") or {} for e in tool_events if str((e.get("payload") or {}).get("phase") or "") == "end"), {})
        self.assertEqual("tool.request", start_payload.get("canonical_anchor_type"))
        self.assertEqual("high", start_payload.get("idempotency_hint"))
        self.assertTrue(bool(start_payload.get("side_effect")))
        self.assertTrue(bool(start_payload.get("external_write")))
        self.assertEqual("tool.result", end_payload.get("canonical_anchor_type"))
        self.assertEqual("medium", end_payload.get("idempotency_hint"))
        self.assertTrue(bool(end_payload.get("side_effect")))
        self.assertTrue(bool(end_payload.get("external_write")))

        retry_events = [
            e
            for e in events
            if str((e.get("payload") or {}).get("phase") or "") == "retry" and str(e.get("step_id") or "") == "step:tool.search"
        ]
        self.assertGreaterEqual(len(retry_events), 1)
        self.assertEqual(2, int(retry_events[-1].get("attempt") or 0))

        branch_events = [e for e in events if str(e.get("step_type") or "") == "branch"]
        self.assertGreaterEqual(len(branch_events), 1)
        branch_payload = branch_events[-1].get("payload") or {}
        self.assertEqual("summarize", branch_payload.get("selected_branch"))
        self.assertEqual(["summarize", "retry_search"], branch_payload.get("branches"))
        self.assertEqual("decision.branch", branch_payload.get("canonical_anchor_type"))

    def test_trace_flow_show_steps_and_ls_json(self):
        flow_id = "flow:test-show-1"
        run_id = f"observe-run:{uuid.uuid4().hex[:8]}"
        events = [
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "plan"},
                flow_id=flow_id,
                step_id="step:plan",
                attempt=1,
                step_type="plan",
            ),
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="failed",
                payload={"step_name": "tool.call"},
                flow_id=flow_id,
                step_id="step:tool.call",
                parent_step_id="step:plan",
                tool_call_id="tool:alpha",
                attempt=1,
                step_type="tool_call",
            ),
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "tool.call"},
                flow_id=flow_id,
                step_id="step:tool.call",
                parent_step_id="step:plan",
                tool_call_id="tool:alpha",
                attempt=2,
                step_type="tool_call",
            ),
        ]
        for event in events:
            tracing_module.emit_trace(event, persist_db=False, persist_file=True)

        show_buf = StringIO()
        with redirect_stdout(show_buf):
            show_code = observe_main(["trace", "flow", "show", flow_id, "--json", "--event-limit", "200"])
        self.assertEqual(0, show_code)
        show_payload = json.loads(show_buf.getvalue().strip())
        self.assertEqual(flow_id, (show_payload.get("flow") or {}).get("flow_id"))
        self.assertGreaterEqual(int((show_payload.get("flow") or {}).get("retry_steps") or 0), 1)
        self.assertGreaterEqual(len(show_payload.get("events") or []), 3)
        self.assertIn("lineage_integrity", show_payload)
        self.assertEqual("ok", str((show_payload.get("lineage_integrity") or {}).get("status") or ""))

        show_text_buf = StringIO()
        with redirect_stdout(show_text_buf):
            show_text_code = observe_main(["trace", "flow", "show", flow_id, "--event-limit", "200"])
        self.assertEqual(0, show_text_code)
        show_text = show_text_buf.getvalue()
        self.assertIn("lineage_warnings=0", show_text)
        self.assertIn("step_id=step:tool.call", show_text)
        self.assertIn("type=tool_call", show_text)
        self.assertIn("attempt=2", show_text)
        self.assertIn("tool=tool:alpha", show_text)
        self.assertIn("anchor=tool.result", show_text)

        steps_buf = StringIO()
        with redirect_stdout(steps_buf):
            steps_code = observe_main(["trace", "flow", "steps", flow_id, "--json", "--event-limit", "200"])
        self.assertEqual(0, steps_code)
        steps_payload = json.loads(steps_buf.getvalue().strip())
        steps = steps_payload.get("steps") or []
        self.assertGreaterEqual(len(steps), 2)
        self.assertIn("lineage_integrity", steps_payload)
        self.assertEqual("ok", str((steps_payload.get("lineage_integrity") or {}).get("status") or ""))
        tool_step = next((item for item in steps if item.get("step_id") == "step:tool.call"), None)
        self.assertIsNotNone(tool_step)
        self.assertEqual(2, int(tool_step.get("attempts") or 0))

        steps_text_buf = StringIO()
        with redirect_stdout(steps_text_buf):
            steps_text_code = observe_main(["trace", "flow", "steps", flow_id, "--event-limit", "200"])
        self.assertEqual(0, steps_text_code)
        steps_text = steps_text_buf.getvalue()
        self.assertIn("lineage_warnings=0", steps_text)
        self.assertIn("parent=step:plan", steps_text)
        self.assertIn("tools=tool:alpha", steps_text)
        self.assertIn("anchors=tool.result", steps_text)
        self.assertIn("duration=", steps_text)

        ls_buf = StringIO()
        with redirect_stdout(ls_buf):
            ls_code = observe_main(["trace", "flow", "ls", "--limit", "20", "--json"])
        self.assertEqual(0, ls_code)
        ls_lines = [line.strip() for line in ls_buf.getvalue().splitlines() if line.strip()]
        self.assertGreaterEqual(len(ls_lines), 1)
        flow_ids = [str((json.loads(line) or {}).get("flow_id") or "") for line in ls_lines]
        self.assertIn(flow_id, flow_ids)

    def test_trace_flow_lineage_integrity_warnings_are_reported(self):
        flow_id = "flow:test-integrity-1"
        run_id = f"observe-run:{uuid.uuid4().hex[:8]}"
        events = [
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "plan"},
                flow_id=flow_id,
                step_id="step:plan",
                attempt=1,
                step_type="plan",
            ),
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="failed",
                payload={"step_name": "tool"},
                flow_id=flow_id,
                step_id="step:tool",
                parent_step_id="step:missing",
                attempt=1,
                step_type="tool_call",
            ),
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "tool"},
                flow_id=flow_id,
                step_id="step:tool",
                parent_step_id="step:plan",
                attempt=2,
                step_type="tool_call",
            ),
        ]
        for event in events:
            tracing_module.emit_trace(event, persist_db=False, persist_file=True)

        show_buf = StringIO()
        with redirect_stdout(show_buf):
            show_code = observe_main(["trace", "flow", "show", flow_id, "--json", "--event-limit", "200"])
        self.assertEqual(0, show_code)
        show_payload = json.loads(show_buf.getvalue().strip())
        lineage = show_payload.get("lineage_integrity") or {}
        self.assertEqual("warning", str(lineage.get("status") or ""))
        self.assertGreaterEqual(int(lineage.get("warning_count") or 0), 1)
        warning_codes = {str(item.get("code") or "") for item in (lineage.get("warnings") or [])}
        self.assertIn("parent_step_missing", warning_codes)
        self.assertIn("conflicting_parent_step_id", warning_codes)
        self.assertGreaterEqual(int((show_payload.get("flow") or {}).get("lineage_warning_count") or 0), 1)

        steps_buf = StringIO()
        with redirect_stdout(steps_buf):
            steps_code = observe_main(["trace", "flow", "steps", flow_id, "--event-limit", "200"])
        self.assertEqual(0, steps_code)
        steps_text = steps_buf.getvalue()
        self.assertIn("lineage_warnings=", steps_text)
        self.assertIn("warning[parent_step_missing]", steps_text)

    def test_trace_flow_diff_json_reports_step_divergence(self):
        flow_a = "flow:test-diff-a"
        flow_b = "flow:test-diff-b"
        run_id = f"observe-run:{uuid.uuid4().hex[:8]}"
        events = [
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "plan"},
                flow_id=flow_a,
                step_id="step:plan",
                attempt=1,
                step_type="plan",
            ),
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "plan"},
                flow_id=flow_b,
                step_id="step:plan",
                attempt=1,
                step_type="plan",
            ),
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="success",
                payload={"step_name": "tool"},
                flow_id=flow_a,
                step_id="step:tool",
                attempt=1,
                step_type="tool_call",
            ),
            tracing_module.build_trace_event(
                event_type="observe_step",
                run_id=run_id,
                outcome="failed",
                payload={"step_name": "tool"},
                flow_id=flow_b,
                step_id="step:tool",
                attempt=1,
                step_type="tool_call",
            ),
        ]
        for event in events:
            tracing_module.emit_trace(event, persist_db=False, persist_file=True)

        diff_buf = StringIO()
        with redirect_stdout(diff_buf):
            diff_code = observe_main(["trace", "flow", "diff", flow_a, flow_b, "--json", "--event-limit", "200"])
        self.assertEqual(0, diff_code)
        payload = json.loads(diff_buf.getvalue().strip())
        self.assertEqual(flow_a, ((payload.get("flow_a") or {}).get("flow_id") or ""))
        self.assertEqual(flow_b, ((payload.get("flow_b") or {}).get("flow_id") or ""))
        self.assertGreaterEqual(int(payload.get("first_divergence_index") or -1), 1)


if __name__ == "__main__":
    unittest.main()
