import concurrent.futures
import contextlib
import json
import sys
import threading
import time
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any, Callable, Dict, FrozenSet, List, Optional, Sequence, Set, Tuple, Union, cast

from .._internal.utils.json_like import ensure_json_like as _ensure_json_like_ssot
from ..events import (
    EVENT_WORKFLOW_NODE_CANCELLED,
    EVENT_WORKFLOW_NODE_END,
    EVENT_WORKFLOW_NODE_START,
    EVENT_WORKFLOW_RESOURCE_COMMIT,
    WORKFLOW_NODE_CANCELLED_REASON_DEPENDENCY_FAILED,
    WORKFLOW_NODE_CANCELLED_REASON_POLICY_ALL_FAIL,
    WORKFLOW_NODE_END_STATUS_ERROR,
    WORKFLOW_NODE_END_STATUS_OK,
    Event,
    generate_run_id,
)
from ..events._events import (
    WorkflowNodeCancelledEvent,
    WorkflowNodeEndEvent,
    WorkflowNodeStartEvent,
    WorkflowResourceCommitEvent,
)
from ..exceptions import ScalimWorkflowError, safe_error_message, safe_error_type
from ..execution.adaptive.capture import HookCaptureManager, HookRecordedEvent
from ..execution.engine import ScalimEngine
from ..execution.run_ir import ExecutionRequest, ExecutionResult, run_ir, run_ir_capture_events
from ..execution.workflow_cache_pool import ScalimWorkflowCachePoolError, WorkflowCachePool
from ..hooks import HookManager
from ..ob.components import split_components
from ..ob.hub import InstrumentationHub
from ..ob.manager import ObserverManager
from ..ob.observability import Observability
from ..ob.observer import Observer
from ..ob.presets._internal import viz_config as viz_config_module
from ..ob.presets._internal.viz_config import normalize_output_dir as _normalize_viz_output_dir
from ..ob.presets.viz import (
    VizObserverConfig,
    WorkflowVizObserver,
    build_workflow_viz_graph_snapshot,
)
from ..sinks.rows import InMemoryRows, iter_in_memory_rows_as_main_rows
from ..spec.ir import DemandIr
from ..spec.ir._workflow import (
    AppendSheetNodeIr,
    WorkflowAnyNodeIr,
    WorkflowCtxOptionsIr,
    WorkflowIr,
    WorkflowNodeIr,
    WorkflowNodeType,
    WriteSheetNodeIr,
)
from ..vendor.compact.typing_extensionsx import TypeGuard
from ..vendor.dataclassesx import dataclass, replace
from ._internal.request_overrides import WorkflowNodeRequestOverrides, merge_workflow_node_request
from .artifacts import WorkflowArtifactsDirectory
from .errors import ScalimWorkflowConfigError
from .input_artifacts import (
    resolve_workflow_input_csv as _resolve_workflow_input_csv,
)
from .input_artifacts import (
    resolve_workflow_input_tabular as _resolve_workflow_input_tabular,
)
from .input_artifacts import (
    resolve_workflow_output_export_header as _resolve_workflow_output_export_header,
)
from .loaders import workflow_loader_context
from .report import WorkflowResult, WorkflowRunError, WorkflowRunOutcome
from .resources import ScalimWorkflowWriteError, SheetBookDef, WorkflowResourceManager
from .resources_base import WorkflowResourceWaitDiagnostics
from .visibility_index import WorkflowVisibilityIndex


class _CompilationLike(ABC):
    @property
    @abstractmethod
    def request(self) -> object:
        raise NotImplementedError  # pragma: no cover  # pragma: allow-no-cover abstract property


def _workflow_error_diff(exc: BaseException) -> Optional[List[str]]:
    if isinstance(exc, ScalimWorkflowWriteError):
        return exc.diff
    return None


def ensure_json_like(value: object, *, path: str) -> object:
    return _ensure_json_like_ssot(
        value,
        path=path,
        value_name="ctx value",
        allowed_types_desc="None/bool/int/float/str/list/dict[str, ...]",
        dict_key_desc="non-empty str",
        require_nonempty_dict_key=True,
        error_cls=ScalimWorkflowConfigError,
    )


def _json_value_size_bytes(value: object) -> int:
    payload = json.dumps(
        value,
        ensure_ascii=False,
        sort_keys=True,
        separators=(",", ":"),
    ).encode("utf-8")
    return len(payload)


class WorkflowCtxStore:
    _guardrails: WorkflowCtxOptionsIr
    _visible_by_consumer_node_id: Dict[str, FrozenSet[str]]
    _values_by_producer_node_id: Dict[str, Dict[str, object]]
    _value_bytes_by_producer_node_id: Dict[str, Dict[str, int]]
    _total_bytes: int
    _lock: threading.Lock

    def __init__(self, workflow_ir: WorkflowIr) -> None:
        self._guardrails = workflow_ir.options.ctx
        visibility = WorkflowVisibilityIndex.from_workflow_ir(workflow_ir)
        self._visible_by_consumer_node_id = visibility.visible_by_consumer_node_id
        self._values_by_producer_node_id = {}
        self._value_bytes_by_producer_node_id = {}
        self._total_bytes = 0
        self._lock = threading.Lock()

    def visible_producer_node_ids(self, consumer_node_id: str) -> FrozenSet[str]:
        return self._visible_by_consumer_node_id.get(str(consumer_node_id), frozenset())

    def publish_default_summary(self, producer_node_id: str, result: ExecutionResult) -> None:
        node_id = str(producer_node_id)
        self.publish(node_id, "output_path", result.output_path, path="workflow.options.ctx")
        self.publish(node_id, "total_rows", int(result.total_rows), path="workflow.options.ctx")
        self.publish(node_id, "duration_secs", float(result.duration), path="workflow.options.ctx")

    def publish(self, producer_node_id: str, key: str, value: object, *, path: str) -> None:
        node_id = str(producer_node_id)
        ctx_key = str(key)
        ctx_value = ensure_json_like(value, path=path)

        value_bytes = _json_value_size_bytes(ctx_value)
        max_value_bytes = int(self._guardrails.max_value_bytes)
        if value_bytes > max_value_bytes:
            msg = "ctx value too large: node={}, key={}, bytes={} > max_value_bytes={}".format(
                node_id, ctx_key, value_bytes, max_value_bytes
            )
            raise ScalimWorkflowConfigError(msg, path="workflow.options.ctx.max_value_bytes")

        with self._lock:
            by_key = self._values_by_producer_node_id.setdefault(node_id, {})
            by_key_bytes = self._value_bytes_by_producer_node_id.setdefault(node_id, {})
            prev_bytes = int(by_key_bytes.get(ctx_key, 0))

            next_total = int(self._total_bytes) - prev_bytes + int(value_bytes)
            max_bytes = int(self._guardrails.max_bytes)
            if next_total > max_bytes:
                msg = "ctx total bytes exceeded: adding node={}, key={} would make total_bytes={} > max_bytes={}".format(
                    node_id,
                    ctx_key,
                    next_total,
                    max_bytes,
                )
                raise ScalimWorkflowConfigError(msg, path="workflow.options.ctx.max_bytes")

            by_key[ctx_key] = ctx_value
            by_key_bytes[ctx_key] = int(value_bytes)
            self._total_bytes = int(next_total)

    def resolve(self, consumer_node_id: str, *, node: str, key: str, path: str) -> object:
        consumer = str(consumer_node_id)
        producer = str(node)
        ctx_key = str(key)

        if producer == consumer:
            msg = "$ctx does not allow node=self (node_id={})".format(consumer)
            raise ScalimWorkflowConfigError(msg, path=path)

        visible = self.visible_producer_node_ids(consumer)
        if producer not in visible:
            msg = "ctx key '{}' from node '{}' is not visible to node '{}' (declare depends_on)".format(ctx_key, producer, consumer)
            raise ScalimWorkflowConfigError(msg, path=path)

        with self._lock:
            by_key = self._values_by_producer_node_id.get(producer) or {}
            if ctx_key not in by_key:
                msg = "Unknown ctx key '{}' for node '{}'".format(ctx_key, producer)
                raise ScalimWorkflowConfigError(msg, path=path)
            return by_key[ctx_key]


def _is_list(value: object) -> TypeGuard[List[object]]:
    return isinstance(value, list)


def _is_dict(value: object) -> TypeGuard[Dict[object, object]]:
    return isinstance(value, dict)


def _is_dict_str_any(value: object) -> TypeGuard[Dict[str, Any]]:
    return isinstance(value, dict)


def iter_ctx_directives(value: object, *, path: str) -> List[Tuple[str, str]]:
    out: List[Tuple[str, str]] = []
    if _is_dict(value):
        mapping = value
        if len(mapping) == 1 and "$ctx" in mapping:
            directive = mapping.get("$ctx")
            if not _is_dict(directive):
                msg = "$ctx directive must be a mapping"
                raise ScalimWorkflowConfigError(msg, path=path)
            node_raw = directive.get("node")
            key_raw = directive.get("key")
            node_id = str(node_raw or "").strip() if isinstance(node_raw, str) else ""
            ctx_key = str(key_raw or "").strip() if isinstance(key_raw, str) else ""
            if not node_id:
                msg = "$ctx.node must be a non-empty string"
                raise ScalimWorkflowConfigError(msg, path=path)
            if not ctx_key:
                msg = "$ctx.key must be a non-empty string"
                raise ScalimWorkflowConfigError(msg, path=path)
            out.append((node_id, ctx_key))
            return out
        for raw_key, raw_value in mapping.items():
            child_path = "{}.{}".format(path, str(raw_key))
            out.extend(iter_ctx_directives(raw_value, path=child_path))
        return out

    if _is_list(value):
        for idx, item in enumerate(value):
            child_path = "{}.{}".format(path, idx)
            out.extend(iter_ctx_directives(item, path=child_path))
    return out


def render_ctx_directives(value: object, *, consumer_node_id: str, ctx_store: WorkflowCtxStore, path: str) -> object:
    if _is_dict(value):
        mapping = value
        if len(mapping) == 1 and "$ctx" in mapping:
            directive = mapping.get("$ctx")
            if not _is_dict(directive):
                msg = "$ctx directive must be a mapping"
                raise ScalimWorkflowConfigError(msg, path=path)
            node_raw = directive.get("node")
            key_raw = directive.get("key")
            node_id = str(node_raw or "").strip() if isinstance(node_raw, str) else ""
            ctx_key = str(key_raw or "").strip() if isinstance(key_raw, str) else ""
            if not node_id:
                msg = "$ctx.node must be a non-empty string"
                raise ScalimWorkflowConfigError(msg, path=path)
            if not ctx_key:
                msg = "$ctx.key must be a non-empty string"
                raise ScalimWorkflowConfigError(msg, path=path)
            value = ctx_store.resolve(str(consumer_node_id), node=node_id, key=ctx_key, path=path)
            return ensure_json_like(value, path=path)

        out: Dict[object, object] = {}
        for raw_key, raw_value in mapping.items():
            child_path = "{}.{}".format(path, str(raw_key))
            out[raw_key] = render_ctx_directives(raw_value, consumer_node_id=consumer_node_id, ctx_store=ctx_store, path=child_path)
        return out

    if _is_list(value):
        return [
            render_ctx_directives(item, consumer_node_id=consumer_node_id, ctx_store=ctx_store, path="{}.{}".format(path, idx))
            for idx, item in enumerate(value)
        ]

    return value


def _render_workflow_init_vars(
    init_vars: Optional[Dict[str, object]],
    *,
    consumer_node_id: str,
    ctx_store: WorkflowCtxStore,
    path_prefix: str,
) -> Dict[str, object]:
    if init_vars is None:
        return {}
    out: Dict[str, object] = {}
    for key, value in init_vars.items():
        item_path = "{}.{}".format(path_prefix, key)
        rendered = render_ctx_directives(value, consumer_node_id=consumer_node_id, ctx_store=ctx_store, path=item_path)
        out[str(key)] = ensure_json_like(rendered, path=item_path)
    return out


def _validate_workflow_ctx_refs(workflow_ir: WorkflowIr, *, ctx_store: WorkflowCtxStore) -> None:
    node_ids = {str(node.node_id) for node in workflow_ir.nodes}
    for node in workflow_ir.nodes:
        if not isinstance(node, WorkflowNodeIr):
            continue
        node_id = str(node.node_id)
        init_vars = node.init_vars
        if not init_vars:
            continue
        visible = ctx_store.visible_producer_node_ids(node_id)
        prefix = "workflow.runs.{}.init_vars".format(int(node.decl_order))
        for key, value in init_vars.items():
            item_path = "{}.{}".format(prefix, key)
            for ref_node_id, _ref_key in iter_ctx_directives(value, path=item_path):
                if ref_node_id == node_id:
                    msg = "$ctx does not allow node=self (node_id={})".format(node_id)
                    raise ScalimWorkflowConfigError(msg, path=item_path)
                if ref_node_id not in node_ids:
                    msg = "Unknown ctx node '{}'".format(ref_node_id)
                    raise ScalimWorkflowConfigError(msg, path=item_path)
                if ref_node_id not in visible:
                    msg = "ctx reference to node '{}' is not visible to node '{}' (declare depends_on)".format(ref_node_id, node_id)
                    raise ScalimWorkflowConfigError(msg, path=item_path)


class ScalimWorkflowRunFailedError(ScalimWorkflowError):
    run_id: str
    demand_path: str

    def __init__(self, message: str, *, run_id: str, demand_path: str) -> None:
        super(ScalimWorkflowRunFailedError, self).__init__(message)
        self.run_id = str(run_id)
        self.demand_path = str(demand_path)


def _run_workflow_write_sheet_node(
    node: WriteSheetNodeIr,
    *,
    artifacts_dir: WorkflowArtifactsDirectory,
    resource_manager: WorkflowResourceManager,
) -> None:
    if str(node.resource_type) == "book":
        book_kind = resource_manager.get_book_kind(str(node.resource_id))
        if book_kind == "xlsx_memory":
            input_csv = _resolve_workflow_input_tabular(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
                error_prefix="write node",
            )
        else:
            input_csv = _resolve_workflow_input_csv(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
                error_prefix="write node",
            )
        resource_manager.apply_book_sheet(
            workflow_node_id=str(node.node_id),
            decl_order=int(node.decl_order),
            book_id=str(node.resource_id),
            sheet=str(node.sheet),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            input_csv=input_csv,
            export_header=_resolve_workflow_output_export_header(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
            ),
            on_conflict=str(node.on_conflict or "error"),
        )
        return

    if str(node.resource_type) == "workbook":
        input_csv = _resolve_workflow_input_csv(
            artifacts_dir=artifacts_dir,
            consumer_node_id=str(node.node_id),
            consumer_decl_order=int(node.decl_order),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            error_prefix="write node",
        )
        resource_manager.apply_workbook_sheet(
            workflow_node_id=str(node.node_id),
            decl_order=int(node.decl_order),
            workbook_id=str(node.resource_id),
            sheet=str(node.sheet),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            input_csv=input_csv,
            on_conflict=str(node.on_conflict or "error"),
        )
        return

    if str(node.resource_type) == "sheetbook":
        input_tabular = _resolve_workflow_input_tabular(
            artifacts_dir=artifacts_dir,
            consumer_node_id=str(node.node_id),
            consumer_decl_order=int(node.decl_order),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            error_prefix="write node",
        )
        resource_manager.apply_sheetbook_sheet(
            workflow_node_id=str(node.node_id),
            decl_order=int(node.decl_order),
            sheetbook_id=str(node.resource_id),
            sheet=str(node.sheet),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            input_csv=input_tabular,
            export_header=_resolve_workflow_output_export_header(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
            ),
            on_conflict=str(node.on_conflict or "error"),
        )
        return

    msg = "Unsupported write_sheet resource_type: {!r}".format(
        str(node.resource_type)
    )  # pragma: no cover  # pragma: allow-no-cover unreachable: IR validated
    raise ScalimWorkflowWriteError(msg)  # pragma: no cover  # pragma: allow-no-cover unreachable: IR validated


def _run_workflow_append_sheet_node(
    node: AppendSheetNodeIr,
    *,
    artifacts_dir: WorkflowArtifactsDirectory,
    resource_manager: WorkflowResourceManager,
) -> None:
    if str(node.resource_type) == "book":
        book_kind = resource_manager.get_book_kind(str(node.resource_id))
        if book_kind == "xlsx_memory":
            input_csv = _resolve_workflow_input_tabular(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
                error_prefix="append node",
            )
        else:
            input_csv = _resolve_workflow_input_csv(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
                error_prefix="append node",
            )
        if not node.sheet:  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
            msg = "append_sheet requires sheet for book resource (resource_id={!r})".format(
                str(node.resource_id)
            )  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
            raise ScalimWorkflowWriteError(msg)  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
        resource_manager.apply_book_append(
            workflow_node_id=str(node.node_id),
            decl_order=int(node.decl_order),
            book_id=str(node.resource_id),
            sheet=str(node.sheet),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            input_csv=input_csv,
            export_header=_resolve_workflow_output_export_header(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
            ),
            align_by=str(node.align_by or "field_id"),
            header_policy=str(node.header_policy or "once"),
            on_mismatch=str(node.on_mismatch or "error"),
        )
        return

    if str(node.resource_type) == "workbook":
        input_csv = _resolve_workflow_input_csv(
            artifacts_dir=artifacts_dir,
            consumer_node_id=str(node.node_id),
            consumer_decl_order=int(node.decl_order),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            error_prefix="append node",
        )
        if not node.sheet:  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
            msg = "append_sheet requires sheet for workbook resource (resource_id={!r})".format(
                str(node.resource_id)
            )  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
            raise ScalimWorkflowWriteError(msg)  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
        resource_manager.apply_workbook_append(
            workflow_node_id=str(node.node_id),
            decl_order=int(node.decl_order),
            workbook_id=str(node.resource_id),
            sheet=str(node.sheet),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            input_csv=input_csv,
            align_by=str(node.align_by or "field_id"),
            header_policy=str(node.header_policy or "once"),
            on_mismatch=str(node.on_mismatch or "error"),
        )
        return

    if str(node.resource_type) == "csv":
        input_csv = _resolve_workflow_input_csv(
            artifacts_dir=artifacts_dir,
            consumer_node_id=str(node.node_id),
            consumer_decl_order=int(node.decl_order),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            error_prefix="append node",
        )
        resource_manager.apply_csv_append(
            workflow_node_id=str(node.node_id),
            decl_order=int(node.decl_order),
            csv_id=str(node.resource_id),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            input_csv=input_csv,
            header_policy=str(node.header_policy or "once"),
            on_mismatch=str(node.on_mismatch or "error"),
        )
        return

    if str(node.resource_type) == "sheetbook":
        input_tabular = _resolve_workflow_input_tabular(
            artifacts_dir=artifacts_dir,
            consumer_node_id=str(node.node_id),
            consumer_decl_order=int(node.decl_order),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            error_prefix="append node",
        )
        if not node.sheet:  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
            msg = "append_sheet requires sheet for sheetbook resource (resource_id={!r})".format(
                str(node.resource_id)
            )  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
            raise ScalimWorkflowWriteError(msg)  # pragma: no cover  # pragma: allow-no-cover invariant: sheet required by IR
        resource_manager.apply_sheetbook_append(
            workflow_node_id=str(node.node_id),
            decl_order=int(node.decl_order),
            sheetbook_id=str(node.resource_id),
            sheet=str(node.sheet),
            input_node_id=str(node.input_node_id),
            input_output_id=str(node.input_output_id),
            input_csv=input_tabular,
            export_header=_resolve_workflow_output_export_header(
                artifacts_dir=artifacts_dir,
                consumer_node_id=str(node.node_id),
                consumer_decl_order=int(node.decl_order),
                input_node_id=str(node.input_node_id),
                input_output_id=str(node.input_output_id),
            ),
            align_by=str(node.align_by or "field_id"),
            header_policy=str(node.header_policy or "once"),
            on_mismatch=str(node.on_mismatch or "error"),
        )
        return

    msg = "Unsupported append_sheet resource_type: {!r}".format(
        str(node.resource_type)
    )  # pragma: no cover  # pragma: allow-no-cover unreachable: IR validated
    raise ScalimWorkflowWriteError(msg)  # pragma: no cover  # pragma: allow-no-cover unreachable: IR validated


def _run_workflow_write_node(
    node: WorkflowAnyNodeIr,
    *,
    artifacts_dir: WorkflowArtifactsDirectory,
    resource_manager: WorkflowResourceManager,
) -> None:
    if isinstance(node, WriteSheetNodeIr):
        _run_workflow_write_sheet_node(
            node,
            artifacts_dir=artifacts_dir,
            resource_manager=resource_manager,
        )
        return

    if isinstance(node, AppendSheetNodeIr):
        _run_workflow_append_sheet_node(
            node,
            artifacts_dir=artifacts_dir,
            resource_manager=resource_manager,
        )
        return

    msg = "Unsupported workflow node type: {}".format(
        type(node).__name__
    )  # pragma: no cover  # pragma: allow-no-cover unreachable: IR validated
    raise ScalimWorkflowWriteError(msg)  # pragma: no cover  # pragma: allow-no-cover unreachable: IR validated


def _bundle_run_dir(config: VizObserverConfig, run_id: str) -> Path:
    base_dir = config.output_dir
    if base_dir is None:
        base_dir = viz_config_module.default_viz_dir()
    output_dir = _normalize_viz_output_dir(str(base_dir))
    return Path(output_dir) / str(run_id)


def _bundle_has_child_replay(config: VizObserverConfig, run_id: str) -> bool:
    run_dir = _bundle_run_dir(config, run_id)
    snapshot_path = run_dir / str(config.snapshot_filename)
    events_path = run_dir / str(config.events_filename)
    return snapshot_path.exists() and events_path.exists()


@dataclass(frozen=True)
class _CapturedNodeRun:
    core: ExecutionResult
    captured_hook_events: List[HookRecordedEvent]
    captured_events: List[Event]
    viz_observer: Optional[Observer]


_NodeRunResult = Union[ExecutionResult, _CapturedNodeRun]


@dataclass
class _PreparedWorkflowRun:
    workflow_path: str
    workflow_exec_id: str
    workflow_components: Tuple[object, ...]
    workflow_ir: WorkflowIr
    artifacts_dir: WorkflowArtifactsDirectory
    ctx_store: WorkflowCtxStore
    max_concurrency: int
    failure_policy: str
    workflow_wall_start_ts: float
    bundle_viz_base_config: Optional[VizObserverConfig]
    workflow_observer_manager: ObserverManager
    workflow_viz_observer: Optional[WorkflowVizObserver]
    workflow_instrumentation: InstrumentationHub
    workflow_cache_pool: Optional[WorkflowCachePool]
    resource_manager: WorkflowResourceManager
    write_output_ids_by_run_id: Dict[str, FrozenSet[str]]
    write_consumers_remaining_by_output_key: Dict[Tuple[str, str], int]
    main_rows_consumers_remaining_by_run_id: Dict[str, int]
    capture_observability: bool
    workflow_replay_instrumentation: Optional[InstrumentationHub]
    captured_demand_events_by_node_id: Dict[str, List[Event]]
    captured_demand_hook_events_by_node_id: Dict[str, List[HookRecordedEvent]]
    captured_demand_viz_observer_by_node_id: Dict[str, Optional[Observer]]
    captured_demand_request_by_node_id: Dict[str, object]


def _build_workflow_instrumentation(
    *,
    workflow_exec_id: str,
    workflow_path: str,
    workflow_ir: WorkflowIr,
    components: Optional[Sequence[object]],
    bundle_viz_base_config: Optional[VizObserverConfig],
) -> Tuple[ObserverManager, Optional[WorkflowVizObserver], InstrumentationHub]:
    # 工作流层事件:复用 `hooks`/`observers` 分发通道,并以 `workflow_exec_id` 作为 `run_id` 分区.
    component_observers, component_hooks = split_components(components)
    workflow_observer_manager = Observability().build_manager(run_id=workflow_exec_id)
    try:
        for observer in component_observers:
            workflow_observer_manager.register(observer)

        workflow_viz_observer: Optional[WorkflowVizObserver] = None
        if bundle_viz_base_config is not None:
            workflow_viz_config = replace(bundle_viz_base_config, run_id="workflow")
            workflow_snapshot = build_workflow_viz_graph_snapshot(
                workflow_ir,
                demand_run_id_by_workflow_node_id={},
                workflow_yaml_path=workflow_path,
            )
            workflow_viz_observer = WorkflowVizObserver(config=workflow_viz_config, snapshot=workflow_snapshot)
            workflow_observer_manager.register(workflow_viz_observer)

        workflow_hook_manager = HookManager()
        for hook in component_hooks:
            workflow_hook_manager.register(hook)

        workflow_instrumentation = InstrumentationHub(
            hook_manager=workflow_hook_manager,
            observer_manager=workflow_observer_manager,
        )
    except Exception:
        with contextlib.suppress(Exception):
            workflow_observer_manager.close()
        raise
    else:
        return workflow_observer_manager, workflow_viz_observer, workflow_instrumentation


def _maybe_build_workflow_cache_pool(
    *,
    workflow_exec_id: str,
    workflow_ir: WorkflowIr,
    workflow_instrumentation: InstrumentationHub,
    logical_keys_by_node_id: Optional[Dict[str, FrozenSet[Tuple[str, str]]]],
    consumers_by_logical_key: Optional[Dict[Tuple[str, str], FrozenSet[str]]],
) -> Optional[WorkflowCachePool]:
    cache_pool_ir = workflow_ir.options.cache_pool
    if cache_pool_ir is None:
        return None

    if logical_keys_by_node_id is None or consumers_by_logical_key is None:
        msg = "workflow cache_pool requires derived consumers mapping"
        raise ScalimWorkflowConfigError(msg, path="workflow.options.cache_pool")
    return WorkflowCachePool(
        workflow_exec_id=workflow_exec_id,
        instrumentation=workflow_instrumentation,
        config=cache_pool_ir,
        logical_keys_by_node_id=logical_keys_by_node_id,
        consumers_by_logical_key=consumers_by_logical_key,
    )


def _build_workflow_resource_defs(  # noqa: PLR0915
    workflow_ir: WorkflowIr,
) -> Tuple[Dict[str, str], Dict[str, bool], Dict[str, bool], Dict[str, str], Dict[str, SheetBookDef]]:
    workbook_defs: Dict[str, str] = {}
    workbook_allow_formulas_by_id: Dict[str, bool] = {}
    workbook_write_lock_by_id: Dict[str, bool] = {}
    csv_defs: Dict[str, str] = {}
    sheetbook_defs: Dict[str, SheetBookDef] = {}

    for res in workflow_ir.resources:
        res_type = str(res.resource_type)
        if res_type == "book":
            opts = res.options or {}
            if not _is_dict_str_any(opts):
                msg = "Invalid workflow resource options for book: resource_id={!r}".format(str(res.resource_id))
                raise ScalimWorkflowConfigError(msg, path="workflow.resources.books")
            kind = str(opts.get("kind") or "").strip()
            if kind == "xlsx_file":
                workbook_defs[str(res.resource_id)] = str(res.path)
                workbook_allow_formulas_by_id[str(res.resource_id)] = bool(opts.get("allow_formulas", False))
                workbook_write_lock_by_id[str(res.resource_id)] = bool(opts.get("write_lock", False))
                continue
            if kind == "xlsx_memory":
                budget_obj = opts.get("budget")
                budget_dict: Dict[str, Any] = budget_obj if _is_dict_str_any(budget_obj) else {}
                max_sheets = int(budget_dict.get("max_sheets") or 0)
                max_total_cells = int(budget_dict.get("max_total_cells") or 0)

                export_cfg_obj = opts.get("export_xlsx")
                export_cfg_dict: Dict[str, Any] = export_cfg_obj if _is_dict_str_any(export_cfg_obj) else {}
                export_write_lock = bool(export_cfg_dict.get("write_lock", False))
                export_allow_formulas = bool(export_cfg_dict.get("allow_formulas", False))
                export_path = str(res.path or "").strip() or None
                sheetbook_defs[str(res.resource_id)] = SheetBookDef(
                    resource_id=str(res.resource_id),
                    budget_max_sheets=int(max_sheets),
                    budget_max_total_cells=int(max_total_cells),
                    export_path=str(export_path) if export_path is not None else None,
                    export_write_lock=bool(export_write_lock),
                    export_allow_formulas=bool(export_allow_formulas),
                )
                continue

            msg = "Unknown book kind: {!r} (book_id={!r})".format(kind, str(res.resource_id))
            raise ScalimWorkflowConfigError(msg, path="workflow.resources.books.{}".format(str(res.resource_id)))

        if res_type == "workbook":
            workbook_defs[str(res.resource_id)] = str(res.path)
            opts = res.options or {}
            allow_formulas = False
            if isinstance(opts, dict):
                allow_formulas = bool(opts.get("allow_formulas", False))
            workbook_allow_formulas_by_id[str(res.resource_id)] = bool(allow_formulas)
            workbook_write_lock_by_id[str(res.resource_id)] = True
            continue

        if res_type == "csv":
            csv_defs[str(res.resource_id)] = str(res.path)
            continue

        if res_type == "sheetbook":
            opts = res.options or {}
            budget_obj = opts.get("budget")
            sheetbook_budget: Dict[str, Any] = budget_obj if _is_dict_str_any(budget_obj) else {}
            max_sheets = int(sheetbook_budget.get("max_sheets") or 0)
            max_total_cells = int(sheetbook_budget.get("max_total_cells") or 0)

            export_cfg_obj = opts.get("export_xlsx")
            sheetbook_export_cfg: Dict[str, Any] = export_cfg_obj if _is_dict_str_any(export_cfg_obj) else {}
            export_write_lock = bool(sheetbook_export_cfg.get("write_lock", False))
            export_allow_formulas = bool(sheetbook_export_cfg.get("allow_formulas", False))
            export_path = str(res.path or "").strip() or None
            sheetbook_defs[str(res.resource_id)] = SheetBookDef(
                resource_id=str(res.resource_id),
                budget_max_sheets=int(max_sheets),
                budget_max_total_cells=int(max_total_cells),
                export_path=str(export_path) if export_path is not None else None,
                export_write_lock=bool(export_write_lock),
                export_allow_formulas=bool(export_allow_formulas),
            )
            continue

    return workbook_defs, workbook_allow_formulas_by_id, workbook_write_lock_by_id, csv_defs, sheetbook_defs


def _build_write_output_ids_by_run_id(workflow_ir: WorkflowIr) -> Dict[str, FrozenSet[str]]:
    tmp_write_output_ids: Dict[str, Set[str]] = {}
    for node in workflow_ir.nodes:
        if isinstance(node, (WriteSheetNodeIr, AppendSheetNodeIr)):
            tmp_write_output_ids.setdefault(str(node.input_node_id), set()).add(str(node.input_output_id))
    return {run_id: frozenset(sorted(ids)) for run_id, ids in tmp_write_output_ids.items()}


def _build_write_consumers_remaining_by_output_key(workflow_ir: WorkflowIr) -> Dict[Tuple[str, str], int]:
    counts: Dict[Tuple[str, str], int] = {}
    for node in workflow_ir.nodes:
        if isinstance(node, (WriteSheetNodeIr, AppendSheetNodeIr)):
            key = (str(node.input_node_id), str(node.input_output_id))
            counts[key] = int(counts.get(key, 0)) + 1
    return counts


def _build_main_rows_consumers_remaining_by_run_id(workflow_ir: WorkflowIr) -> Dict[str, int]:
    counts: Dict[str, int] = {}
    for node in workflow_ir.nodes:
        if not isinstance(node, WorkflowNodeIr):
            continue
        producer_run_id = str(node.main_rows_from_run_id or "").strip()
        if not producer_run_id:
            continue
        counts[producer_run_id] = int(counts.get(producer_run_id, 0)) + 1
    return counts


def _prepare_workflow_run_ir(
    workflow_path: str,
    workflow_ir: WorkflowIr,
    *,
    components: Optional[Sequence[object]],
    bundle_viz_base_config: Optional[VizObserverConfig],
    cache_pool_logical_keys_by_node_id: Optional[Dict[str, FrozenSet[Tuple[str, str]]]],
    cache_pool_consumers_by_logical_key: Optional[Dict[Tuple[str, str], FrozenSet[str]]],
) -> _PreparedWorkflowRun:
    workflow_exec_id = generate_run_id(prefix="wf")
    artifacts_dir = WorkflowArtifactsDirectory(workflow_ir)
    ctx_store = WorkflowCtxStore(workflow_ir)
    _validate_workflow_ctx_refs(workflow_ir, ctx_store=ctx_store)

    max_concurrency = int(workflow_ir.options.max_concurrency)
    failure_policy = str(workflow_ir.options.failure_policy or "all_fail")
    workflow_wall_start_ts = time.time()

    workflow_observer_manager: Optional[ObserverManager] = None
    workflow_cache_pool: Optional[WorkflowCachePool] = None
    try:
        workflow_observer_manager, workflow_viz_observer, workflow_instrumentation = _build_workflow_instrumentation(
            workflow_exec_id=workflow_exec_id,
            workflow_path=workflow_path,
            workflow_ir=workflow_ir,
            components=components,
            bundle_viz_base_config=bundle_viz_base_config,
        )

        component_observers, component_hooks = split_components(components)
        capture_observability = int(max_concurrency) > 1 and bool(component_observers or component_hooks)
        workflow_replay_instrumentation: Optional[InstrumentationHub] = None
        if capture_observability:
            workflow_replay_instrumentation = workflow_instrumentation
            capture_hook_manager = HookCaptureManager(workflow_instrumentation.hook_manager)
            capture_observer_manager = workflow_observer_manager.create_capture_manager()
            capture_observer_manager.max_recorded_events = None
            workflow_instrumentation = InstrumentationHub(
                hook_manager=capture_hook_manager,
                observer_manager=capture_observer_manager,
            )

        if workflow_viz_observer is not None:
            _ = workflow_instrumentation.emit(
                "workflow_started",
                {
                    "workflow_id": str(Path(workflow_path).name),
                    "workflow_exec_id": str(workflow_exec_id),
                    "max_concurrency": int(max_concurrency),
                },
            )
        workflow_cache_pool = _maybe_build_workflow_cache_pool(
            workflow_exec_id=workflow_exec_id,
            workflow_ir=workflow_ir,
            workflow_instrumentation=workflow_instrumentation,
            logical_keys_by_node_id=cache_pool_logical_keys_by_node_id,
            consumers_by_logical_key=cache_pool_consumers_by_logical_key,
        )
        (
            workbook_defs,
            workbook_allow_formulas_by_id,
            workbook_write_lock_by_id,
            csv_defs,
            sheetbook_defs,
        ) = _build_workflow_resource_defs(workflow_ir)
        diagnostics_ir = workflow_ir.options.resources_wait.diagnostics
        wait_diagnostics = WorkflowResourceWaitDiagnostics(
            enabled=bool(diagnostics_ir.enabled),
            warn_after_s=float(diagnostics_ir.warn_after_s),
            repeat_every_s=float(diagnostics_ir.repeat_every_s) if diagnostics_ir.repeat_every_s is not None else None,
            capture_owner_callsite=bool(diagnostics_ir.capture_owner_callsite),
        )
        resource_manager = WorkflowResourceManager(
            workflow_exec_id=workflow_exec_id,
            instrumentation=workflow_instrumentation,
            workbook_defs=workbook_defs,
            workbook_allow_formulas=workbook_allow_formulas_by_id,
            workbook_write_lock=workbook_write_lock_by_id,
            csv_defs=csv_defs,
            sheetbook_defs=sheetbook_defs,
            wait_diagnostics=wait_diagnostics,
            max_wait_s=float(workflow_ir.options.resources_wait.max_wait_s),
            output_staging_dir_name=str(workflow_ir.options.output_staging.dir_name),
            output_staging_keep_on_success=bool(workflow_ir.options.output_staging.keep_on_success),
            output_staging_keep_on_failure=bool(workflow_ir.options.output_staging.keep_on_failure),
        )
        write_output_ids_by_run_id = _build_write_output_ids_by_run_id(workflow_ir)
        write_consumers_remaining_by_output_key = _build_write_consumers_remaining_by_output_key(workflow_ir)
        main_rows_consumers_remaining_by_run_id = _build_main_rows_consumers_remaining_by_run_id(workflow_ir)

        return _PreparedWorkflowRun(
            workflow_path=workflow_path,
            workflow_exec_id=workflow_exec_id,
            workflow_components=tuple(components or ()),
            workflow_ir=workflow_ir,
            artifacts_dir=artifacts_dir,
            ctx_store=ctx_store,
            max_concurrency=int(max_concurrency),
            failure_policy=str(failure_policy),
            workflow_wall_start_ts=float(workflow_wall_start_ts),
            bundle_viz_base_config=bundle_viz_base_config,
            workflow_observer_manager=workflow_observer_manager,
            workflow_viz_observer=workflow_viz_observer,
            workflow_instrumentation=workflow_instrumentation,
            workflow_cache_pool=workflow_cache_pool,
            resource_manager=resource_manager,
            write_output_ids_by_run_id=write_output_ids_by_run_id,
            write_consumers_remaining_by_output_key=write_consumers_remaining_by_output_key,
            main_rows_consumers_remaining_by_run_id=main_rows_consumers_remaining_by_run_id,
            capture_observability=bool(capture_observability),
            workflow_replay_instrumentation=workflow_replay_instrumentation,
            captured_demand_events_by_node_id={},
            captured_demand_hook_events_by_node_id={},
            captured_demand_viz_observer_by_node_id={},
            captured_demand_request_by_node_id={},
        )
    except Exception:
        if workflow_cache_pool is not None:
            with contextlib.suppress(Exception):
                workflow_cache_pool.close()
        if workflow_observer_manager is not None:
            with contextlib.suppress(Exception):
                workflow_observer_manager.close()
        raise


def _compile_demand_node(
    node: WorkflowNodeIr,
    *,
    workflow_exec_id: str,
    ctx_store: WorkflowCtxStore,
    compile_demand_fn: Callable[..., object],
    bundle_viz_base_config: Optional[VizObserverConfig],
    write_output_ids_by_run_id: Dict[str, FrozenSet[str]],
    main_rows_consumers_remaining_by_run_id: Dict[str, int],
    artifacts_dir: WorkflowArtifactsDirectory,
) -> Tuple[object, ExecutionRequest]:
    node_id = str(node.node_id)
    demand_path = str(node.demand_path or "")

    node_init_vars = _render_workflow_init_vars(
        node.init_vars,
        consumer_node_id=node_id,
        ctx_store=ctx_store,
        path_prefix="workflow.runs.{}.init_vars".format(int(node.decl_order)),
    )

    managed_output_ids = write_output_ids_by_run_id.get(node_id)

    node_viz_config = None
    if bundle_viz_base_config is not None:
        node_viz_config = replace(bundle_viz_base_config, run_id=str(node_id))

    compilation = compile_demand_fn(
        demand_path,
        workflow_exec_id=str(workflow_exec_id),
        workflow_node_id=str(node_id),
        workflow_node_decl_order=int(node.decl_order),
        node_init_vars=node_init_vars,
        managed_output_ids=managed_output_ids,
        viz_config=node_viz_config,
    )
    comp = cast("_CompilationLike", compilation)  # pragma: allow-cast compile_demand typed narrowing
    base_request = cast("ExecutionRequest", comp.request)  # pragma: allow-cast compilation.request narrow
    overrides = WorkflowNodeRequestOverrides()
    if node_id in main_rows_consumers_remaining_by_run_id:
        overrides = replace(overrides, capture_in_memory_rows=True)

    producer_run_id = str(node.main_rows_from_run_id or "").strip()
    if producer_run_id:
        try:
            typed_rows_obj = artifacts_dir.get(str(node_id), producer_run_id, "in_memory_rows")
        except ValueError as exc:
            path = "workflow.runs.{}.main_rows_from_run_id".format(int(node.decl_order))
            raise ScalimWorkflowConfigError(str(exc), path=path) from exc
        if not isinstance(typed_rows_obj, InMemoryRows):
            msg = "Missing workflow-managed typed rows artifact: producer_node_id={!r}".format(producer_run_id)
            raise ScalimWorkflowWriteError(msg)
        overrides = replace(overrides, main_rows=iter_in_memory_rows_as_main_rows(typed_rows_obj))

    request_for_run = merge_workflow_node_request(base_request, overrides)
    return compilation, request_for_run


def _workflow_try_submit_ready(  # noqa: PLR0913
    executor: concurrent.futures.ThreadPoolExecutor,
    *,
    ready_queue: List[str],
    submitted: Dict[
        "concurrent.futures.Future[Any]",
        Tuple[str, WorkflowAnyNodeIr, Optional[str], Optional[object], Optional[ExecutionRequest]],
    ],
    max_concurrency: int,
    failure_policy: str,
    failed_outcome_holder: List[Optional[WorkflowRunOutcome]],
    failed_exc_holder: List[Optional[BaseException]],
    node_by_id: Dict[str, WorkflowAnyNodeIr],
    node_state: Dict[str, str],
    index_by_node_id: Dict[str, int],
    outcomes: List[Optional[WorkflowRunOutcome]],
    workflow_exec_id: str,
    workflow_cache_pool: Optional[WorkflowCachePool],
    compile_demand_fn: Callable[..., object],
    artifacts_dir: WorkflowArtifactsDirectory,
    ctx_store: WorkflowCtxStore,
    bundle_viz_base_config: Optional[VizObserverConfig],
    write_output_ids_by_run_id: Dict[str, FrozenSet[str]],
    main_rows_consumers_remaining_by_run_id: Dict[str, int],
    resource_manager: WorkflowResourceManager,
    run_one: Callable[[DemandIr, ExecutionRequest, str], _NodeRunResult],
    emit_workflow_node_start: Callable[[WorkflowAnyNodeIr], None],
    emit_workflow_node_end: Callable[..., None],
    maybe_release_workflow_main_rows_artifact: Callable[[WorkflowAnyNodeIr], None],
    on_terminal: Callable[..., None],
    cancel_all_not_started_due_to_all_fail: Callable[[], None],
) -> None:
    while ready_queue and len(submitted) < max_concurrency and (failed_outcome_holder[0] is None or failure_policy != "all_fail"):
        node_id = ready_queue.pop(0)
        node = node_by_id[str(node_id)]
        node_state[str(node_id)] = "running"
        emit_workflow_node_start(node)

        if isinstance(node, WorkflowNodeIr):
            demand_path = str(node.demand_path or "")
            try:
                compilation, request_for_run = _compile_demand_node(
                    node,
                    workflow_exec_id=str(workflow_exec_id),
                    ctx_store=ctx_store,
                    compile_demand_fn=compile_demand_fn,
                    bundle_viz_base_config=bundle_viz_base_config,
                    write_output_ids_by_run_id=write_output_ids_by_run_id,
                    main_rows_consumers_remaining_by_run_id=main_rows_consumers_remaining_by_run_id,
                    artifacts_dir=artifacts_dir,
                )
            except ScalimWorkflowConfigError:
                raise
            except Exception as exc:  # noqa: BLE001
                err = WorkflowRunError(
                    run_id=str(node_id),
                    demand_path=demand_path,
                    exc_type=safe_error_type(exc),
                    message=str(safe_error_message(exc) or ""),
                    diff=_workflow_error_diff(exc),
                )
                outcome = WorkflowRunOutcome(run_id=str(node_id), demand_path=demand_path, result=None, error=err)
                outcomes[index_by_node_id[str(node_id)]] = outcome
                node_state[str(node_id)] = "failed"
                emit_workflow_node_end(node, status=WORKFLOW_NODE_END_STATUS_ERROR, exc=exc)
                maybe_release_workflow_main_rows_artifact(node)
                if workflow_cache_pool is not None:
                    workflow_cache_pool.on_workflow_node_done(str(node_id))
                on_terminal(str(node_id), ok=False)
                if failure_policy == "all_fail" and failed_outcome_holder[0] is None:
                    failed_outcome_holder[0] = outcome
                    failed_exc_holder[0] = exc
                    cancel_all_not_started_due_to_all_fail()
                continue

            comp = cast("Any", compilation)  # pragma: allow-cast compilation runtime boundary
            demand_ir = cast("DemandIr", comp.demand_ir)  # pragma: allow-cast compilation.demand_ir narrow
            fut = executor.submit(run_one, demand_ir, request_for_run, str(node_id))
            submitted[fut] = (str(node_id), node, str(demand_path), compilation, request_for_run)
            continue

        fut = executor.submit(
            _run_workflow_write_node,
            node,
            artifacts_dir=artifacts_dir,
            resource_manager=resource_manager,
        )
        submitted[fut] = (str(node_id), node, None, None, None)


def _workflow_process_completed_future(  # noqa: C901, PLR0912, PLR0913, PLR0915
    fut: "concurrent.futures.Future[Any]",
    *,
    node_id: str,
    node: WorkflowAnyNodeIr,
    demand_path: Optional[str],
    compilation: Optional[object],
    request: Optional[ExecutionRequest],
    idx: int,
    outcomes: List[Optional[WorkflowRunOutcome]],
    node_state: Dict[str, str],
    workflow_exec_id: str,
    artifacts_dir: WorkflowArtifactsDirectory,
    ctx_store: WorkflowCtxStore,
    build_demand_run_result_fn: Optional[Callable[..., object]],
    workflow_cache_pool: Optional[WorkflowCachePool],
    failure_policy: str,
    failed_outcome_holder: List[Optional[WorkflowRunOutcome]],
    failed_exc_holder: List[Optional[BaseException]],
    emit_workflow_node_end: Callable[..., None],
    maybe_release_workflow_managed_in_memory_output: Callable[[WorkflowAnyNodeIr], None],
    maybe_release_workflow_main_rows_artifact: Callable[[WorkflowAnyNodeIr], None],
    on_terminal: Callable[..., None],
    cancel_all_not_started_due_to_all_fail: Callable[[], None],
    captured_demand_events_by_node_id: Dict[str, List[Event]],
    captured_demand_hook_events_by_node_id: Dict[str, List[HookRecordedEvent]],
    captured_demand_viz_observer_by_node_id: Dict[str, Optional[Observer]],
    captured_demand_request_by_node_id: Dict[str, object],
) -> None:
    try:
        result_obj = fut.result()
        capture_obj: Optional[_CapturedNodeRun] = None
        if isinstance(result_obj, _CapturedNodeRun):
            capture_obj = result_obj
            result_obj = capture_obj.core

        if isinstance(node, WorkflowNodeIr):
            core = cast("ExecutionResult", result_obj)  # pragma: allow-cast future result typed narrowing
            if capture_obj is not None:
                captured_demand_events_by_node_id[str(node_id)] = list(capture_obj.captured_events)
                captured_demand_hook_events_by_node_id[str(node_id)] = list(capture_obj.captured_hook_events)
                captured_demand_viz_observer_by_node_id[str(node_id)] = capture_obj.viz_observer
                if request is not None:
                    captured_demand_request_by_node_id[str(node_id)] = request
            demand_yaml_path = str(demand_path or "")
            artifacts_dir.publish(str(node_id), "output_path", core.output_path)
            artifacts_dir.publish(str(node_id), "outputs", core.outputs)
            artifacts_dir.publish(str(node_id), "in_memory_csv_outputs", core.in_memory_csv_outputs or {})
            artifacts_dir.publish(str(node_id), "in_memory_rows_outputs", core.in_memory_rows_outputs or {})
            artifacts_dir.publish(str(node_id), "in_memory_csv_export_headers", core.workflow_managed_output_export_headers or {})
            if core.in_memory_rows is not None:
                artifacts_dir.publish(str(node_id), "in_memory_rows", core.in_memory_rows)
            ctx_store.publish_default_summary(str(node_id), core)
            if build_demand_run_result_fn is None:
                node_result: object = core
            else:
                node_result = build_demand_run_result_fn(
                    core,
                    compilation=compilation,
                    demand_yaml_path=demand_yaml_path,
                    workflow_exec_id=str(workflow_exec_id),
                    workflow_node_id=str(node_id),
                )
            outcomes[idx] = WorkflowRunOutcome(
                run_id=str(node_id),
                demand_path=demand_yaml_path,
                result=node_result,
                error=None,
            )
        else:
            maybe_release_workflow_managed_in_memory_output(node)
            outcomes[idx] = WorkflowRunOutcome(run_id=str(node_id), demand_path="", result=None, error=None)

        maybe_release_workflow_main_rows_artifact(node)
        node_state[str(node_id)] = "done"
        emit_workflow_node_end(node, status=WORKFLOW_NODE_END_STATUS_OK, exc=None)
        if workflow_cache_pool is not None:
            workflow_cache_pool.on_workflow_node_done(str(node_id))
        on_terminal(str(node_id), ok=True)
    except Exception as exc:
        if isinstance(exc, ScalimWorkflowCachePoolError):
            raise ScalimWorkflowConfigError(str(exc), path=exc.path) from exc
        err = WorkflowRunError(
            run_id=str(node_id),
            demand_path=str(demand_path or ""),
            exc_type=safe_error_type(exc),
            message=str(safe_error_message(exc) or ""),
            diff=_workflow_error_diff(exc),
        )
        outcome = WorkflowRunOutcome(run_id=str(node_id), demand_path=str(demand_path or ""), result=None, error=err)
        outcomes[idx] = outcome
        node_state[str(node_id)] = "failed"
        emit_workflow_node_end(node, status=WORKFLOW_NODE_END_STATUS_ERROR, exc=exc)
        maybe_release_workflow_main_rows_artifact(node)
        if workflow_cache_pool is not None:
            workflow_cache_pool.on_workflow_node_done(str(node_id))
        on_terminal(str(node_id), ok=False)
        if failure_policy == "all_fail" and failed_outcome_holder[0] is None:
            failed_outcome_holder[0] = outcome
            failed_exc_holder[0] = exc
            cancel_all_not_started_due_to_all_fail()


def _execute_workflow_run(  # noqa: C901, PLR0915
    prepared: _PreparedWorkflowRun,
    *,
    compile_demand_fn: Callable[..., object],
    build_demand_run_result_fn: Optional[Callable[..., object]],
    run_ir_fn: Callable[..., ExecutionResult],
) -> Tuple[List[WorkflowRunOutcome], Optional[WorkflowRunOutcome], Optional[BaseException]]:
    workflow_exec_id = prepared.workflow_exec_id
    workflow_ir = prepared.workflow_ir
    artifacts_dir = prepared.artifacts_dir
    ctx_store = prepared.ctx_store
    max_concurrency = int(prepared.max_concurrency)
    failure_policy = str(prepared.failure_policy or "all_fail")
    bundle_viz_base_config = prepared.bundle_viz_base_config
    workflow_instrumentation = prepared.workflow_instrumentation
    workflow_cache_pool = prepared.workflow_cache_pool
    resource_manager = prepared.resource_manager
    write_output_ids_by_run_id = prepared.write_output_ids_by_run_id
    write_consumers_remaining_by_output_key = prepared.write_consumers_remaining_by_output_key
    main_rows_consumers_remaining_by_run_id = prepared.main_rows_consumers_remaining_by_run_id

    outcomes: List[Optional[WorkflowRunOutcome]] = [None for _ in range(len(workflow_ir.nodes))]
    failed_outcome_holder: List[Optional[WorkflowRunOutcome]] = [None]
    failed_exc_holder: List[Optional[BaseException]] = [None]

    def _maybe_release_workflow_managed_in_memory_output(node: WorkflowAnyNodeIr) -> None:
        if not isinstance(node, (WriteSheetNodeIr, AppendSheetNodeIr)):
            return  # pragma: no cover  # pragma: allow-no-cover invariant: helper called only for write nodes
        producer_node_id = str(node.input_node_id)
        output_id = str(node.input_output_id)
        key = (producer_node_id, output_id)
        remaining = write_consumers_remaining_by_output_key.get(key)
        if remaining is None:
            return
        next_remaining = int(remaining) - 1
        if next_remaining < 0:
            msg = "workflow internal error: negative write consumer count: producer_node_id={!r}, output_id={!r}".format(
                producer_node_id,
                output_id,
            )
            raise RuntimeError(msg)
        if next_remaining == 0:
            _ = write_consumers_remaining_by_output_key.pop(key, None)
            artifacts_dir.discard_in_memory_csv_output(producer_node_id, output_id)
            artifacts_dir.discard_in_memory_rows_output(producer_node_id, output_id)
        else:
            write_consumers_remaining_by_output_key[key] = int(next_remaining)

    def _maybe_release_workflow_main_rows_artifact(node: WorkflowAnyNodeIr) -> None:
        if not isinstance(node, WorkflowNodeIr):
            return
        producer_node_id = str(node.main_rows_from_run_id or "").strip()
        if not producer_node_id:
            return
        remaining = main_rows_consumers_remaining_by_run_id.get(producer_node_id)
        if remaining is None:
            return
        next_remaining = int(remaining) - 1
        if next_remaining < 0:
            msg = "workflow internal error: negative main_rows consumer count: producer_node_id={!r}".format(producer_node_id)
            raise RuntimeError(msg)
        if next_remaining == 0:
            _ = main_rows_consumers_remaining_by_run_id.pop(producer_node_id, None)
            artifacts_dir.discard(producer_node_id, "in_memory_rows")
        else:
            main_rows_consumers_remaining_by_run_id[producer_node_id] = int(next_remaining)

    def _node_type_str(node: WorkflowAnyNodeIr) -> str:
        raw = node.node_type
        return str(raw.value if isinstance(raw, WorkflowNodeType) else raw)

    def _node_demand_path(node: WorkflowAnyNodeIr) -> Optional[str]:
        if isinstance(node, WorkflowNodeIr):
            return node.demand_path
        return None

    def _emit_workflow_node_start(node: WorkflowAnyNodeIr) -> None:
        node_id = str(node.node_id)
        demand_path = _node_demand_path(node)
        _ = workflow_instrumentation.emit(
            EVENT_WORKFLOW_NODE_START,
            WorkflowNodeStartEvent(
                workflow_exec_id=workflow_exec_id,
                workflow_node_id=str(node_id),
                node_type=_node_type_str(node),
                demand_path=str(demand_path) if demand_path is not None else None,
            ),
            meta={
                "workflow_exec_id": workflow_exec_id,
                "workflow_node_id": str(node_id),
            },
        )

    def _emit_workflow_node_end(node: WorkflowAnyNodeIr, *, status: str, exc: Optional[BaseException]) -> None:
        node_id = str(node.node_id)
        demand_path = _node_demand_path(node)
        error_type = None
        error_message = None
        if status != WORKFLOW_NODE_END_STATUS_OK and exc is not None:
            error_type = safe_error_type(exc)
            error_message = safe_error_message(exc)
        _ = workflow_instrumentation.emit(
            EVENT_WORKFLOW_NODE_END,
            WorkflowNodeEndEvent(
                workflow_exec_id=workflow_exec_id,
                workflow_node_id=str(node_id),
                node_type=_node_type_str(node),
                status=str(status),
                demand_path=str(demand_path) if demand_path is not None else None,
                error_type=error_type,
                error_message=error_message,
            ),
            meta={
                "workflow_exec_id": workflow_exec_id,
                "workflow_node_id": str(node_id),
            },
        )

    def _emit_workflow_node_cancelled(node: WorkflowAnyNodeIr, *, reason: str, message: str) -> None:
        node_id = str(node.node_id)
        demand_path = _node_demand_path(node)
        _ = workflow_instrumentation.emit(
            EVENT_WORKFLOW_NODE_CANCELLED,
            WorkflowNodeCancelledEvent(
                workflow_exec_id=workflow_exec_id,
                workflow_node_id=str(node_id),
                node_type=_node_type_str(node),
                reason=str(reason),
                message=str(message),
                demand_path=str(demand_path) if demand_path is not None else None,
            ),
            meta={
                "workflow_exec_id": workflow_exec_id,
                "workflow_node_id": str(node_id),
            },
        )

    def _run_one(demand_ir: DemandIr, request: ExecutionRequest, workflow_node_id: str) -> _NodeRunResult:
        def _engine_factory(**kwargs: object) -> ScalimEngine:
            engine_kwargs = cast("Any", kwargs)  # pragma: allow-cast engine kwargs typed narrowing
            return ScalimEngine(
                **engine_kwargs,
                workflow_cache_pool=workflow_cache_pool,
                workflow_node_id=str(workflow_node_id),
            )

        visible = ctx_store.visible_producer_node_ids(str(workflow_node_id))
        decl_order = int(index_by_node_id.get(str(workflow_node_id), 0))
        event_meta_defaults = {
            "workflow_exec_id": workflow_exec_id,
            "workflow_node_id": str(workflow_node_id),
            "workflow_node_decl_order": int(decl_order),
        }
        with workflow_loader_context(
            workflow_exec_id=workflow_exec_id,
            workflow_node_id=str(workflow_node_id),
            visible_producer_node_ids=visible,
            resource_manager=resource_manager,
        ):
            if prepared.capture_observability and run_ir_fn is run_ir:
                core, captured_hook_events, captured_events, viz_observer = run_ir_capture_events(
                    demand_ir,
                    request,
                    engine_factory=_engine_factory,
                    event_meta_defaults=event_meta_defaults,
                )
                return _CapturedNodeRun(
                    core=core,
                    captured_hook_events=list(captured_hook_events),
                    captured_events=list(captured_events),
                    viz_observer=viz_observer,
                )
            return run_ir_fn(
                demand_ir,
                request,
                engine_factory=_engine_factory,
                event_meta_defaults=event_meta_defaults,
            )

    node_by_id: Dict[str, WorkflowAnyNodeIr] = {node.node_id: node for node in workflow_ir.nodes}
    index_by_node_id: Dict[str, int] = {node.node_id: int(node.decl_order) for node in workflow_ir.nodes}

    dependents_by_node_id: Dict[str, List[str]] = {}
    for node in workflow_ir.nodes:
        for dep_id in node.deps:
            dependents_by_node_id.setdefault(str(dep_id), []).append(node.node_id)
    for children in dependents_by_node_id.values():
        children.sort(key=lambda nid: index_by_node_id.get(str(nid), 0))

    node_state: Dict[str, str] = {node.node_id: "pending" for node in workflow_ir.nodes}
    remaining_prereqs: Dict[str, int] = {node.node_id: len(node.deps) for node in workflow_ir.nodes}
    prereq_failed: Dict[str, bool] = {node.node_id: False for node in workflow_ir.nodes}

    ready_queue: List[str] = []
    for node in workflow_ir.nodes:
        if remaining_prereqs.get(node.node_id, 0) == 0:
            node_state[node.node_id] = "ready"
            ready_queue.append(node.node_id)
    ready_queue.sort(key=lambda nid: index_by_node_id.get(str(nid), 0))

    submitted: Dict[
        "concurrent.futures.Future[Any]",
        Tuple[str, WorkflowAnyNodeIr, Optional[str], Optional[object], Optional[ExecutionRequest]],
    ] = {}

    def _cancel_node(node_id: str, *, reason: str, message: str) -> None:
        idx = index_by_node_id[str(node_id)]
        node = node_by_id[str(node_id)]
        demand_path = str(_node_demand_path(node) or "")
        outcomes[idx] = WorkflowRunOutcome(
            run_id=str(node_id),
            demand_path=demand_path,
            result=None,
            error=WorkflowRunError(
                run_id=str(node_id),
                demand_path=demand_path,
                exc_type="WorkflowCancelled",
                message=str(message),
            ),
        )
        node_state[str(node_id)] = "cancelled"
        _emit_workflow_node_cancelled(node, reason=reason, message=message)
        _maybe_release_workflow_main_rows_artifact(node)
        if workflow_cache_pool is not None:
            workflow_cache_pool.on_workflow_node_done(str(node_id))

    def _on_terminal(node_id: str, *, ok: bool) -> None:
        for child_id in dependents_by_node_id.get(str(node_id), []):
            if node_state.get(str(child_id)) in {"done", "failed", "cancelled"}:
                continue
            remaining_prereqs[str(child_id)] -= 1
            if not ok:
                prereq_failed[str(child_id)] = True
            if remaining_prereqs[str(child_id)] == 0:
                if prereq_failed[str(child_id)]:
                    _cancel_node(
                        str(child_id),
                        reason=WORKFLOW_NODE_CANCELLED_REASON_DEPENDENCY_FAILED,
                        message="Cancelled due to dependency failure",
                    )
                    _on_terminal(str(child_id), ok=False)
                else:
                    node_state[str(child_id)] = "ready"
                    ready_queue.append(str(child_id))
                    ready_queue.sort(key=lambda nid: index_by_node_id.get(str(nid), 0))

    def _cancel_all_not_started_due_to_all_fail() -> None:
        nonlocal ready_queue
        for node in workflow_ir.nodes:
            if node_state.get(node.node_id) in {"pending", "ready"}:
                _cancel_node(
                    node.node_id,
                    reason=WORKFLOW_NODE_CANCELLED_REASON_POLICY_ALL_FAIL,
                    message="Cancelled due to failure_policy=all_fail",
                )
        ready_queue = []

    with concurrent.futures.ThreadPoolExecutor(max_workers=max_concurrency) as executor:

        def _try_submit_ready() -> None:
            _workflow_try_submit_ready(
                executor,
                ready_queue=ready_queue,
                submitted=submitted,
                max_concurrency=max_concurrency,
                failure_policy=failure_policy,
                failed_outcome_holder=failed_outcome_holder,
                failed_exc_holder=failed_exc_holder,
                node_by_id=node_by_id,
                node_state=node_state,
                index_by_node_id=index_by_node_id,
                outcomes=outcomes,
                workflow_exec_id=str(workflow_exec_id),
                workflow_cache_pool=workflow_cache_pool,
                compile_demand_fn=compile_demand_fn,
                artifacts_dir=artifacts_dir,
                ctx_store=ctx_store,
                bundle_viz_base_config=bundle_viz_base_config,
                write_output_ids_by_run_id=write_output_ids_by_run_id,
                main_rows_consumers_remaining_by_run_id=main_rows_consumers_remaining_by_run_id,
                resource_manager=resource_manager,
                run_one=_run_one,
                emit_workflow_node_start=_emit_workflow_node_start,
                emit_workflow_node_end=_emit_workflow_node_end,
                maybe_release_workflow_main_rows_artifact=_maybe_release_workflow_main_rows_artifact,
                on_terminal=_on_terminal,
                cancel_all_not_started_due_to_all_fail=_cancel_all_not_started_due_to_all_fail,
            )

        _try_submit_ready()

        while submitted:
            done, _pending = concurrent.futures.wait(submitted.keys(), return_when=concurrent.futures.FIRST_COMPLETED)
            for fut in done:
                node_id, node, demand_path, compilation, request = submitted.pop(fut)
                idx = index_by_node_id.get(str(node_id), 0)
                _workflow_process_completed_future(
                    fut,
                    node_id=str(node_id),
                    node=node,
                    demand_path=demand_path,
                    compilation=compilation,
                    request=request,
                    idx=idx,
                    outcomes=outcomes,
                    node_state=node_state,
                    workflow_exec_id=str(workflow_exec_id),
                    artifacts_dir=artifacts_dir,
                    ctx_store=ctx_store,
                    build_demand_run_result_fn=build_demand_run_result_fn,
                    workflow_cache_pool=workflow_cache_pool,
                    failure_policy=failure_policy,
                    failed_outcome_holder=failed_outcome_holder,
                    failed_exc_holder=failed_exc_holder,
                    emit_workflow_node_end=_emit_workflow_node_end,
                    maybe_release_workflow_managed_in_memory_output=_maybe_release_workflow_managed_in_memory_output,
                    maybe_release_workflow_main_rows_artifact=_maybe_release_workflow_main_rows_artifact,
                    on_terminal=_on_terminal,
                    cancel_all_not_started_due_to_all_fail=_cancel_all_not_started_due_to_all_fail,
                    captured_demand_events_by_node_id=prepared.captured_demand_events_by_node_id,
                    captured_demand_hook_events_by_node_id=prepared.captured_demand_hook_events_by_node_id,
                    captured_demand_viz_observer_by_node_id=prepared.captured_demand_viz_observer_by_node_id,
                    captured_demand_request_by_node_id=prepared.captured_demand_request_by_node_id,
                )

            _try_submit_ready()

    final_outcomes: List[WorkflowRunOutcome] = []
    for idx, outcome in enumerate(outcomes):
        if outcome is None:  # pragma: no cover  # pragma: allow-no-cover unreachable: outcome always set
            node_id = str(workflow_ir.nodes[idx].node_id)  # pragma: no cover  # pragma: allow-no-cover unreachable: outcome always set
            demand_path = str(
                _node_demand_path(workflow_ir.nodes[idx]) or ""
            )  # pragma: no cover  # pragma: allow-no-cover unreachable: outcome always set
            missing = WorkflowRunOutcome(  # pragma: no cover  # pragma: allow-no-cover unreachable: outcome always set
                run_id=node_id,
                demand_path=demand_path,
                result=None,
                error=WorkflowRunError(run_id=node_id, demand_path=demand_path, exc_type="Unknown", message="Missing outcome"),
            )
            final_outcomes.append(missing)  # pragma: no cover  # pragma: allow-no-cover unreachable: outcome always set
            continue  # pragma: no cover  # pragma: allow-no-cover unreachable: outcome always set
        final_outcomes.append(outcome)

    return final_outcomes, failed_outcome_holder[0], failed_exc_holder[0]


def _commit_workflow_resources(
    *,
    resource_manager: WorkflowResourceManager,
    outcomes: List[WorkflowRunOutcome],
    failed: Optional[WorkflowRunOutcome],
) -> None:
    try:
        has_errors = any(o.error is not None for o in outcomes)
        if has_errors:
            discard_node_id = failed.run_id if failed is not None else "__wf__discard"
            resource_manager.discard_all(workflow_node_id=str(discard_node_id), reason="workflow_failed")
        else:
            resource_manager.commit_all()
    except ScalimWorkflowWriteError as exc:
        with contextlib.suppress(Exception):
            resource_manager.discard_all(workflow_node_id="__wf__discard", reason="resource_commit_failed")
        raise ScalimWorkflowConfigError(str(exc), path="workflow.resources") from exc


def _report_workflow_viz_finished(prepared: _PreparedWorkflowRun) -> None:
    if prepared.workflow_viz_observer is None or prepared.bundle_viz_base_config is None:
        return

    total_duration_ms = int(max(0.0, time.time() - prepared.workflow_wall_start_ts) * 1000)
    status = "error" if sys.exc_info()[1] is not None else "ok"
    with contextlib.suppress(Exception):
        _ = prepared.workflow_instrumentation.emit(
            "workflow_finished",
            {
                "workflow_id": str(Path(prepared.workflow_path).name),
                "workflow_exec_id": str(prepared.workflow_exec_id),
                "status": status,
                "total_duration_ms": total_duration_ms,
            },
        )

    # 子运行完成后重写工作流快照,避免生成指向缺失子运行的下钻链接.
    with contextlib.suppress(Exception):
        demand_run_id_by_workflow_node_id: Dict[str, str] = {}
        for node in prepared.workflow_ir.nodes:
            if not isinstance(node, WorkflowNodeIr):
                continue
            node_id = str(node.node_id or "").strip()
            if not node_id:
                continue  # pragma: no cover  # pragma: allow-no-cover unreachable: node_id required by IR
            if _bundle_has_child_replay(prepared.bundle_viz_base_config, node_id):
                demand_run_id_by_workflow_node_id[node_id] = node_id

        workflow_run_dir = _bundle_run_dir(prepared.bundle_viz_base_config, "workflow")
        workflow_run_dir.mkdir(parents=True, exist_ok=True)
        workflow_snapshot = build_workflow_viz_graph_snapshot(
            prepared.workflow_ir,
            demand_run_id_by_workflow_node_id=demand_run_id_by_workflow_node_id,
            workflow_yaml_path=prepared.workflow_path,
        )
        snapshot_path = workflow_run_dir / str(prepared.bundle_viz_base_config.snapshot_filename)
        with snapshot_path.open("w", encoding="utf-8") as handle:
            json.dump(workflow_snapshot, handle, ensure_ascii=False, indent=2, default=str)


def _build_demand_replay_instrumentation(  # noqa: C901
    request: Optional[object],
    viz_observer: Optional[Observer],
    *,
    workflow_components: Tuple[object, ...],
) -> Optional[InstrumentationHub]:
    if request is None and viz_observer is None:
        return None

    fallback_logger_enabled = False
    components: List[object] = []
    if request is not None:
        req = cast("ExecutionRequest", request)  # pragma: allow-cast request runtime boundary
        obs_spec = req.observability
        if obs_spec is not None:
            fallback_logger_enabled = bool(obs_spec.fallback_logger_enabled)
        req_components = req.components
        if req_components:
            workflow_component_ids = {id(c) for c in workflow_components}
            for component in list(req_components):
                if id(component) in workflow_component_ids:
                    continue
                components.append(component)

    observers, hooks = split_components(components)
    if viz_observer is None and not observers and not hooks:
        return None

    observer_manager = Observability(fallback_logger_enabled=fallback_logger_enabled).build_manager()
    for observer in observers:
        observer_manager.register(observer)
    if viz_observer is not None:
        observer_manager.register(viz_observer)

    hook_manager = HookManager(fallback_logger_enabled=fallback_logger_enabled)
    for hook in hooks:
        hook_manager.register(hook)

    return InstrumentationHub(hook_manager=hook_manager, observer_manager=observer_manager)


def _replay_captured_workflow_observability(prepared: _PreparedWorkflowRun) -> None:  # noqa: C901, PLR0912, PLR0915
    if not prepared.capture_observability:
        return
    replay = prepared.workflow_replay_instrumentation
    if replay is None:
        return

    capture = prepared.workflow_instrumentation
    workflow_hook_events: List[HookRecordedEvent] = []
    with contextlib.suppress(Exception):
        workflow_hook_events = cast("Any", capture.hook_manager).drain_events()  # pragma: allow-cast capture hook manager drain
    for event in workflow_hook_events:
        replay.hook_manager.emit_typed(str(event.event_type), event.payload)

    workflow_events: List[Event] = []
    with contextlib.suppress(Exception):
        workflow_events = cast("Any", capture.observer_manager).drain_events()  # pragma: allow-cast capture observer manager drain

    known_node_ids: Set[str] = {node.node_id for node in prepared.workflow_ir.nodes}
    started_events: List[Event] = []
    finished_events: List[Event] = []
    other_global_events: List[Event] = []
    unknown_node_events: List[Event] = []

    node_start_events_by_node_id: Dict[str, List[Event]] = {}
    node_end_events_by_node_id: Dict[str, List[Event]] = {}
    node_cancelled_events_by_node_id: Dict[str, List[Event]] = {}
    node_other_events_by_node_id: Dict[str, List[Event]] = {}
    resource_commit_events: List[Event] = []

    for event in workflow_events:
        if str(event.event_type) == "workflow_started":
            started_events.append(event)
            continue
        if str(event.event_type) == "workflow_finished":
            finished_events.append(event)
            continue
        if str(event.event_type) == EVENT_WORKFLOW_RESOURCE_COMMIT:
            resource_commit_events.append(event)
            continue

        node_id = ""
        meta = event.meta
        if meta and isinstance(meta, dict):
            raw_node_id = meta.get("workflow_node_id")
            if raw_node_id is not None:
                node_id = str(raw_node_id)
        node_id = str(node_id or "").strip()
        if not node_id:
            other_global_events.append(event)
            continue
        if node_id not in known_node_ids:
            unknown_node_events.append(event)
            continue

        if str(event.event_type) == EVENT_WORKFLOW_NODE_START:
            node_start_events_by_node_id.setdefault(node_id, []).append(event)
            continue
        if str(event.event_type) == EVENT_WORKFLOW_NODE_END:
            node_end_events_by_node_id.setdefault(node_id, []).append(event)
            continue
        if str(event.event_type) == EVENT_WORKFLOW_NODE_CANCELLED:
            node_cancelled_events_by_node_id.setdefault(node_id, []).append(event)
            continue
        node_other_events_by_node_id.setdefault(node_id, []).append(event)

    workflow_seq = 0

    def _emit_workflow_event(event: Event) -> None:
        nonlocal workflow_seq
        workflow_seq += 1
        meta = dict(event.meta) if event.meta else {}
        replay.emit_recorded_event(
            Event(
                event_type=str(event.event_type),
                timestamp=float(event.timestamp),
                run_id=str(prepared.workflow_exec_id),
                payload=event.payload,
                meta=meta,
                seq=int(workflow_seq),
            )
        )

    def _replay_demand_node(node_id: str) -> None:
        hook_events = prepared.captured_demand_hook_events_by_node_id.get(str(node_id), [])
        observer_events = prepared.captured_demand_events_by_node_id.get(str(node_id), [])
        if observer_events:
            observer_events = sorted(observer_events, key=lambda e: int(e.seq))

        viz_observer = prepared.captured_demand_viz_observer_by_node_id.get(str(node_id))
        request = prepared.captured_demand_request_by_node_id.get(str(node_id))
        node_replay = _build_demand_replay_instrumentation(
            request,
            viz_observer,
            workflow_components=prepared.workflow_components,
        )
        for event in hook_events:
            replay.hook_manager.emit_typed(str(event.event_type), event.payload)
            if node_replay is not None:
                node_replay.hook_manager.emit_typed(str(event.event_type), event.payload)
        for event in observer_events:
            replay.emit_recorded_event(event)
            if node_replay is not None:
                node_replay.emit_recorded_event(event)
        if node_replay is not None:
            with contextlib.suppress(Exception):
                node_replay.observer_manager.close()

    for event in started_events:
        _emit_workflow_event(event)

    nodes_in_order = sorted(prepared.workflow_ir.nodes, key=lambda n: int(n.decl_order))
    for node in nodes_in_order:
        node_id = str(node.node_id)
        for event in node_start_events_by_node_id.get(node_id, []):
            _emit_workflow_event(event)
        if isinstance(node, WorkflowNodeIr) and node_id in prepared.captured_demand_events_by_node_id:
            _replay_demand_node(node_id)
        for event in node_other_events_by_node_id.get(node_id, []):
            _emit_workflow_event(event)
        for event in node_cancelled_events_by_node_id.get(node_id, []):
            _emit_workflow_event(event)
        for event in node_end_events_by_node_id.get(node_id, []):
            _emit_workflow_event(event)

    def _resource_commit_sort_key(event: Event) -> Tuple[str, str, str]:
        payload = cast("WorkflowResourceCommitEvent", event.payload)  # pragma: allow-cast resource commit payload boundary
        return (
            str(payload.resource_type),
            str(payload.resource_id),
            str(payload.path),
        )

    for event in sorted(resource_commit_events, key=_resource_commit_sort_key):
        _emit_workflow_event(event)
    for event in unknown_node_events:
        _emit_workflow_event(event)
    for event in other_global_events:
        _emit_workflow_event(event)
    for event in finished_events:
        _emit_workflow_event(event)


def _cleanup_workflow_finally(prepared: _PreparedWorkflowRun, *, resources_finalized: bool) -> None:
    if not resources_finalized:
        with contextlib.suppress(Exception):
            prepared.resource_manager.discard_all(workflow_node_id="__wf__discard", reason="workflow_finally")
    with contextlib.suppress(Exception):
        prepared.artifacts_dir.discard_all_in_memory_csv_outputs()
    with contextlib.suppress(Exception):
        prepared.artifacts_dir.discard_all_in_memory_rows_outputs()
    with contextlib.suppress(Exception):
        prepared.artifacts_dir.discard_all_in_memory_rows()
    if prepared.workflow_cache_pool is not None:
        with contextlib.suppress(Exception):
            prepared.workflow_cache_pool.close()
    with contextlib.suppress(Exception):
        cast("Any", prepared.workflow_observer_manager).close()  # pragma: allow-cast observer manager close boundary


def run_workflow_ir(
    workflow_path: str,
    workflow_ir: WorkflowIr,
    *,
    compile_demand_fn: Callable[..., object],
    build_demand_run_result_fn: Optional[Callable[..., object]] = None,
    run_ir_fn: Optional[Callable[..., ExecutionResult]] = None,
    components: Optional[Sequence[object]] = None,
    bundle_viz_base_config: Optional[VizObserverConfig] = None,
    cache_pool_logical_keys_by_node_id: Optional[Dict[str, FrozenSet[Tuple[str, str]]]] = None,
    cache_pool_consumers_by_logical_key: Optional[Dict[Tuple[str, str], FrozenSet[str]]] = None,
) -> WorkflowResult:
    prepared: Optional[_PreparedWorkflowRun] = None
    resources_finalized = False
    try:
        prepared = _prepare_workflow_run_ir(
            workflow_path,
            workflow_ir,
            components=components,
            bundle_viz_base_config=bundle_viz_base_config,
            cache_pool_logical_keys_by_node_id=cache_pool_logical_keys_by_node_id,
            cache_pool_consumers_by_logical_key=cache_pool_consumers_by_logical_key,
        )
        final_outcomes, failed, failed_exc = _execute_workflow_run(
            prepared,
            compile_demand_fn=compile_demand_fn,
            build_demand_run_result_fn=build_demand_run_result_fn,
            run_ir_fn=run_ir_fn or run_ir,
        )
        try:
            _commit_workflow_resources(
                resource_manager=prepared.resource_manager,
                outcomes=final_outcomes,
                failed=failed,
            )
            resources_finalized = True
        except ScalimWorkflowConfigError:
            resources_finalized = True
            raise

        result = WorkflowResult(outcomes=tuple(final_outcomes))
        if failed is not None and prepared.failure_policy == "all_fail":
            msg = "工作流运行失败(run_id={}, demand_path={})".format(failed.run_id, failed.demand_path)
            exc = ScalimWorkflowRunFailedError(msg, run_id=failed.run_id, demand_path=failed.demand_path)
            if failed_exc is not None:
                exc.__cause__ = failed_exc
            raise exc
        return result
    finally:
        if prepared is not None:
            _report_workflow_viz_finished(prepared)
            with contextlib.suppress(Exception):
                _replay_captured_workflow_observability(prepared)
            _cleanup_workflow_finally(prepared, resources_finalized=resources_finalized)


__all__ = (
    "ScalimWorkflowRunFailedError",
    "WorkflowResult",
    "WorkflowRunError",
    "WorkflowRunOutcome",
    "run_workflow_ir",
)
