"""Runtime orchestration helpers for pipeline state/event processing."""

from __future__ import annotations

import copy
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
import importlib
from importlib import metadata
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
import json
import logging
import os
import shutil
import socket
import subprocess
import time
import uuid
from urllib.parse import urlparse
import duckdb
from rdflib import Dataset

from .namespaces import CF_BASE
from .ontology_manager import OntologyManager
from .pipeline_states_store import PendingRunEvent, PipelineStateStore
from .rdf_document import canonical_json, load_rdf_document

logger = logging.getLogger(__name__)
_STRUCTURED_FORMAT = "json" + "-ld"
_LOCAL_TIMEZONE = datetime.now().astimezone().tzinfo or timezone.utc


OPCUA_ENDPOINT_UNREACHABLE_EXIT_CODE = 20
DEFAULT_EVENT_DEFER_SECONDS = 1.0


_PIPELINE_TYPES = {
    "cf:ProcessingPipeline",
    f"{CF_BASE}ProcessingPipeline",
}
_PARAMETER_BINDING_TYPES = {
    "cf:ParameterBinding",
    f"{CF_BASE}ParameterBinding",
}
_INVOCATION_TYPES = {
    "cf:PipelineInvocation",
    f"{CF_BASE}PipelineInvocation",
}


@dataclass(frozen=True)
class PipelineRuntimeInfo:
    pipeline_uri: Optional[str]
    event_type: Optional[str]
    min_interval_seconds: float
    engine_interval_seconds: float
    engine_iterations: int
    engine_duration_seconds: float
    auto_sleep_seconds: float
    steps_paths: List[str]
    plugin_paths: List[str]
    opcua_endpoint: Optional[str]


@dataclass
class PipelineStateRow:
    pipeline_id: str
    current_rev: int
    desired_state: str
    concurrency_limit: int
    priority: int
    last_run_id: Optional[str]
    last_run_at: Optional[datetime]
    next_run_at: Optional[datetime]


def _as_text(value: Any) -> Optional[str]:
    if isinstance(value, str):
        return value
    if isinstance(value, dict):
        if "@value" in value:
            return str(value.get("@value"))
        if "@id" in value:
            return str(value.get("@id"))
    return None


def _as_float(value: Any, default: float) -> float:
    if value is None:
        return default
    if isinstance(value, (int, float)):
        return float(value)
    if isinstance(value, dict):
        raw = value.get("@value")
        if isinstance(raw, (int, float)):
            return float(raw)
        if isinstance(raw, str):
            try:
                return float(raw)
            except Exception:
                return default
    if isinstance(value, str):
        try:
            return float(value)
        except Exception:
            return default
    return default


def _as_int(value: Any, default: int) -> int:
    if value is None:
        return default
    if isinstance(value, bool):
        return int(value)
    if isinstance(value, int):
        return int(value)
    if isinstance(value, float):
        return int(value)
    if isinstance(value, dict):
        raw = value.get("@value")
        if isinstance(raw, (int, float)):
            return int(raw)
        if isinstance(raw, str):
            try:
                return int(float(raw))
            except Exception:
                return default
    if isinstance(value, str):
        try:
            return int(float(value))
        except Exception:
            return default
    return default


def _normalize_timestamp(value: Optional[datetime]) -> Optional[datetime]:
    if value is None:
        return None
    if value.tzinfo is None:
        return value.replace(tzinfo=_LOCAL_TIMEZONE).astimezone(timezone.utc)
    return value.astimezone(timezone.utc)


def _iter_nodes(doc: Dict[str, Any]) -> Iterable[Dict[str, Any]]:
    graph = doc.get("@graph")
    if isinstance(graph, list):
        for node in graph:
            if isinstance(node, dict):
                yield node
    elif isinstance(doc, dict):
        yield doc


def _has_type(node: Dict[str, Any], candidates: set[str]) -> bool:
    raw = node.get("@type")
    types: List[str] = []
    if isinstance(raw, str):
        types.append(raw)
    elif isinstance(raw, list):
        for entry in raw:
            if isinstance(entry, str):
                types.append(entry)
            elif isinstance(entry, dict):
                entry_id = entry.get("@id")
                if isinstance(entry_id, str):
                    types.append(entry_id)
    elif isinstance(raw, dict):
        entry_id = raw.get("@id")
        if isinstance(entry_id, str):
            types.append(entry_id)
    return any(t in candidates for t in types)


def _find_pipeline_node(doc: Dict[str, Any]) -> Optional[Dict[str, Any]]:
    for node in _iter_nodes(doc):
        if _has_type(node, _PIPELINE_TYPES):
            return node
    return None


def _extract_id(value: Any) -> Optional[str]:
    if isinstance(value, dict):
        entry_id = value.get("@id")
        if isinstance(entry_id, str):
            return entry_id
    if isinstance(value, str):
        return value
    return None


def _extract_id_list(value: Any) -> List[str]:
    ids: List[str] = []
    if isinstance(value, list):
        for item in value:
            item_id = _extract_id(item)
            if item_id:
                ids.append(item_id)
    else:
        item_id = _extract_id(value)
        if item_id:
            ids.append(item_id)
    return ids


def _extract_list_values(value: Any) -> List[str]:
    values: List[str] = []
    if isinstance(value, list):
        for item in value:
            text = _as_text(item)
            if text:
                values.append(text)
    else:
        text = _as_text(value)
        if text:
            values.append(text)
    return values


def _extract_runtime_info(doc: Dict[str, Any]) -> PipelineRuntimeInfo:
    pipeline_node = _find_pipeline_node(doc)
    pipeline_uri = None
    event_type = None
    min_interval = 0.0
    engine_interval = 0.0
    engine_iterations = 0
    engine_duration = 0.0
    auto_sleep = 0.0
    steps_paths: List[str] = []
    plugin_paths: List[str] = []
    opcua_endpoint = None

    if pipeline_node:
        pipeline_uri = _as_text(pipeline_node.get("@id"))
        min_interval = _as_float(
            pipeline_node.get("cfrun:minIntervalSeconds")
            or pipeline_node.get("cf:minIntervalSeconds"),
            0.0,
        )
        engine_interval = _as_float(
            pipeline_node.get("cfrun:engineIntervalSeconds")
            or pipeline_node.get("cf:engineIntervalSeconds"),
            0.0,
        )
        engine_iterations = _as_int(
            pipeline_node.get("cfrun:engineIterations")
            or pipeline_node.get("cf:engineIterations"),
            0,
        )
        engine_duration = _as_float(
            pipeline_node.get("cfrun:engineDurationSeconds")
            or pipeline_node.get("cf:engineDurationSeconds"),
            0.0,
        )
        auto_sleep = _as_float(
            pipeline_node.get("cfrun:autoSleepSeconds")
            or pipeline_node.get("cf:autoSleepSeconds"),
            0.0,
        )
        trigger_ids = _extract_id_list(
            pipeline_node.get("cfrun:hasTrigger")
            or pipeline_node.get("cf:hasTrigger")
        )
        if trigger_ids:
            trigger_id = trigger_ids[0]
            for node in _iter_nodes(doc):
                if node.get("@id") == trigger_id:
                    event_type = _as_text(
                        node.get("cfrun:eventType")
                        or node.get("cf:eventType")
                        or node.get("schema:name")
                    )
                    break

        steps_header_ids = _extract_id_list(pipeline_node.get("cf:hasStepsHeader"))
        plugins_header_ids = _extract_id_list(pipeline_node.get("cf:hasPluginsHeader"))

        if steps_header_ids:
            for node in _iter_nodes(doc):
                if node.get("@id") in steps_header_ids:
                    steps_paths = _extract_list_values(node.get("cf:stepsPath"))
                    break

        if plugins_header_ids:
            for node in _iter_nodes(doc):
                if node.get("@id") in plugins_header_ids:
                    plugin_paths = _extract_list_values(node.get("cf:pluginPath"))
                    break

    for node in _iter_nodes(doc):
        if not _has_type(node, _PARAMETER_BINDING_TYPES):
            continue
        value = _as_text(node.get("cf:value"))
        if value and value.startswith("opc.tcp://"):
            opcua_endpoint = value
            break

    return PipelineRuntimeInfo(
        pipeline_uri=pipeline_uri,
        event_type=event_type,
        min_interval_seconds=min_interval,
        engine_interval_seconds=engine_interval,
        engine_iterations=engine_iterations,
        engine_duration_seconds=engine_duration,
        auto_sleep_seconds=auto_sleep,
        steps_paths=steps_paths,
        plugin_paths=plugin_paths,
        opcua_endpoint=opcua_endpoint,
    )


def _to_list(value: Any) -> List[Any]:
    if value is None:
        return []
    if isinstance(value, list):
        return value
    return [value]


def _build_node_index(doc: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
    by_id: Dict[str, Dict[str, Any]] = {}
    for node in _iter_nodes(doc):
        node_id = node.get("@id")
        if isinstance(node_id, str):
            by_id[node_id] = node
    return by_id


def _deref_node(value: Any, by_id: Dict[str, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
    if isinstance(value, dict):
        entry_id = value.get("@id")
        if isinstance(entry_id, str):
            return by_id.get(entry_id, value)
        return value
    if isinstance(value, str):
        return by_id.get(value)
    return None


def _normalize_override_value(value: Any) -> Any:
    if isinstance(value, dict) and "@value" in value:
        return value.get("@value")
    return value


def _extract_invocation_settings(
    settings_node: Dict[str, Any],
    invocation_by_id: Dict[str, Dict[str, Any]],
) -> Dict[str, Any]:
    settings: Dict[str, Any] = {}
    for prop_ref in _to_list(settings_node.get("schema:additionalProperty")):
        prop_node = _deref_node(prop_ref, invocation_by_id)
        if not isinstance(prop_node, dict):
            continue
        key = _as_text(prop_node.get("schema:propertyID"))
        if not key or "schema:value" not in prop_node:
            continue
        settings[str(key)] = _normalize_override_value(prop_node.get("schema:value"))
    return settings


def _parameter_key_candidates(parameter_id: str) -> set[str]:
    candidates: set[str] = set()
    if not parameter_id:
        return candidates
    candidates.add(parameter_id)
    if ":" in parameter_id:
        candidates.add(parameter_id.split(":", 1)[1])
    local = parameter_id.rsplit("/", 1)[-1].rsplit("#", 1)[-1]
    candidates.add(local)
    if "_param_" in local:
        candidates.add(local.split("_param_", 1)[1])
    return {entry for entry in candidates if entry}


def _id_candidates(identifier: str) -> set[str]:
    candidates: set[str] = set()
    if not identifier:
        return candidates

    candidates.add(identifier)
    local = identifier.rsplit("/", 1)[-1].rsplit("#", 1)[-1]
    candidates.add(local)

    if ":" in identifier and not identifier.startswith(("http://", "https://")):
        prefix, suffix = identifier.split(":", 1)
        if suffix:
            candidates.add(suffix)
            if prefix == "cf":
                candidates.add(f"{CF_BASE}{suffix}")
            elif prefix == "ex":
                candidates.add(f"{CF_BASE}examples/pipeline/{suffix}")

    return {entry for entry in candidates if entry}


def _ids_equivalent(left: Optional[str], right: Optional[str]) -> bool:
    if not left or not right:
        return False
    return not _id_candidates(left).isdisjoint(_id_candidates(right))


def _resolve_indexed_node(
    by_id: Dict[str, Dict[str, Any]],
    node_id: str,
) -> Optional[Dict[str, Any]]:
    direct = by_id.get(node_id)
    if direct is not None:
        return direct
    for candidate_id, candidate_node in by_id.items():
        if _ids_equivalent(candidate_id, node_id):
            return candidate_node
    return None


def _apply_override_settings_to_node(
    node: Dict[str, Any],
    pipeline_by_id: Dict[str, Dict[str, Any]],
    settings: Dict[str, Any],
) -> None:
    binding_refs = _to_list(node.get("cf:hasParameterBinding"))
    if not binding_refs:
        node_id = _as_text(node.get("@id")) or "<unknown>"
        raise ValueError(f"Invocation override target node has no parameter bindings: {node_id}")

    binding_lookup: Dict[str, List[Dict[str, Any]]] = {}
    binding_lookup_folded: Dict[str, List[Dict[str, Any]]] = {}
    for binding_ref in binding_refs:
        binding_node = _deref_node(binding_ref, pipeline_by_id)
        if not isinstance(binding_node, dict):
            continue
        parameter_id = _extract_id(binding_node.get("cf:bindsParameter"))
        if not parameter_id:
            continue
        for key in _parameter_key_candidates(parameter_id):
            binding_lookup.setdefault(key, []).append(binding_node)
            binding_lookup_folded.setdefault(key.casefold(), []).append(binding_node)

    for raw_key, value in settings.items():
        key = str(raw_key)
        matches = binding_lookup.get(key)
        if not matches:
            matches = binding_lookup_folded.get(key.casefold())
        if not matches:
            node_id = _as_text(node.get("@id")) or "<unknown>"
            raise ValueError(
                f"Invocation override key '{key}' does not match any parameter binding on node {node_id}"
            )

        seen: set[int] = set()
        for binding_node in matches:
            ident = id(binding_node)
            if ident in seen:
                continue
            seen.add(ident)
            binding_node["cf:value"] = value


def _apply_invocation_overrides(
    om: OntologyManager,
    pipeline_doc: Dict[str, Any],
    invocation_doc: Dict[str, Any],
) -> Dict[str, Any]:
    patched_doc = copy.deepcopy(pipeline_doc)
    pipeline_node = _find_pipeline_node(patched_doc)
    if pipeline_node is None:
        raise ValueError("Pipeline document does not contain a cf:ProcessingPipeline node")
    pipeline_uri = _as_text(pipeline_node.get("@id"))

    invocation_nodes = [
        node for node in _iter_nodes(invocation_doc) if _has_type(node, _INVOCATION_TYPES)
    ]
    if not invocation_nodes:
        raise ValueError("Invocation document does not contain a cf:PipelineInvocation node")
    if len(invocation_nodes) > 1:
        raise ValueError("Invocation document must contain exactly one cf:PipelineInvocation node")
    invocation_node = invocation_nodes[0]

    invoked_pipeline_ids = _extract_id_list(invocation_node.get("cf:invokesPipeline"))
    if pipeline_uri and invoked_pipeline_ids and not any(
        _ids_equivalent(pipeline_uri, invoked_pipeline_id)
        for invoked_pipeline_id in invoked_pipeline_ids
    ):
        raise ValueError(
            "Invocation targets pipeline(s) "
            f"{invoked_pipeline_ids}, but active pipeline is '{pipeline_uri}'"
        )

    merged_doc = OntologyManager._combine_documents([patched_doc, invocation_doc])
    validation = om.validate_pipeline_document(merged_doc)
    if not bool(validation.get("conforms")):
        report = str(validation.get("report_text") or "").strip()
        if isinstance(om, OntologyManager):
            if report:
                logger.warning("Invocation SHACL validation reported issues but runtime will continue:\n%s", report)
            else:
                logger.warning("Invocation SHACL validation reported issues but runtime will continue.")
        elif report:
            raise ValueError(f"Invocation SHACL validation failed:\n{report}")
        else:
            raise ValueError("Invocation SHACL validation failed")

    invocation_by_id = _build_node_index(invocation_doc)
    pipeline_by_id = _build_node_index(patched_doc)

    override_refs = _to_list(invocation_node.get("cf:overridesEntryPoint"))
    if not override_refs:
        raise ValueError("Invocation does not declare any cf:overridesEntryPoint entries")

    applied = 0
    for override_ref in override_refs:
        override_node = _deref_node(override_ref, invocation_by_id)
        if not isinstance(override_node, dict):
            raise ValueError("Invocation override reference could not be resolved")

        target_node_id = _extract_id(override_node.get("cf:targetsNode"))
        if not target_node_id:
            raise ValueError("Invocation override is missing cf:targetsNode")

        target_node = _resolve_indexed_node(pipeline_by_id, target_node_id)
        if not isinstance(target_node, dict):
            raise ValueError(
                f"Invocation override target node '{target_node_id}' does not exist in pipeline"
            )

        settings_node = _deref_node(override_node.get("cf:overrideSettings"), invocation_by_id)
        if not isinstance(settings_node, dict):
            raise ValueError(
                f"Invocation override for node '{target_node_id}' is missing cf:overrideSettings"
            )

        settings = _extract_invocation_settings(settings_node, invocation_by_id)
        if not settings:
            raise ValueError(
                f"Invocation override for node '{target_node_id}' has no schema:additionalProperty settings"
            )

        _apply_override_settings_to_node(target_node, pipeline_by_id, settings)
        applied += 1

    logger.info("Applied %d invocation entry-point override(s).", applied)
    return patched_doc


def _resolve_paths(values: List[str], base_path: Optional[Path]) -> List[str]:
    resolved: List[str] = []
    for raw in values:
        candidate = Path(raw)
        if not candidate.is_absolute() and base_path is not None:
            candidate = (base_path / candidate).resolve()
        resolved.append(str(candidate))
    return resolved


def _existing_paths(paths: List[str]) -> List[str]:
    existing: List[str] = []
    for raw in paths:
        candidate = Path(raw)
        if candidate.exists():
            existing.append(str(candidate))
    return existing


def _discover_installed_plugin_dirs() -> List[str]:
    plugin_dirs: List[str] = []
    seen_pkgs: set[str] = set()
    seen_dirs: set[str] = set()

    try:
        entry_points = metadata.entry_points(group="cogniflow.steps")
    except TypeError:  # pragma: no cover - legacy API compatibility
        entry_points = metadata.entry_points().get("cogniflow.steps", [])
    except Exception:  # pragma: no cover - optional metadata edge-cases
        return plugin_dirs

    for ep in entry_points:
        module_name = ep.value.split(":")[0]
        pkg_name = module_name.split(".")[0]
        if pkg_name in seen_pkgs:
            continue
        seen_pkgs.add(pkg_name)
        try:
            module = importlib.import_module(pkg_name)
        except Exception:
            continue
        module_file = getattr(module, "__file__", None)
        if not module_file:
            continue
        bin_dir = Path(module_file).resolve().parent / "bin"
        if not bin_dir.exists():
            continue
        resolved = str(bin_dir)
        if resolved in seen_dirs:
            continue
        seen_dirs.add(resolved)
        plugin_dirs.append(resolved)

    return plugin_dirs


def _opcua_server_reachable(endpoint: Optional[str], timeout: float = 1.0) -> bool:
    if not endpoint:
        return False
    parsed = urlparse(endpoint)
    host = parsed.hostname
    port = parsed.port or 4840
    if not host:
        return False
    try:
        with socket.create_connection((host, port), timeout=timeout):
            return True
    except OSError:
        return False


def _find_packaged_pipeline_exe() -> Optional[str]:
    try:
        import cf_pipeline_engine
    except Exception:  # pragma: no cover - optional dependency
        return None

    try:
        candidate = str(cf_pipeline_engine.cf_pipeline_v2_path())
    except Exception:  # pragma: no cover - optional dependency
        return None
    return candidate or None


def _find_repo_build_pipeline_exe() -> Optional[str]:
    engine_dirs: List[Path] = [Path("sandcastle/cf_pipeline/cf_pipeline_engine")]
    try:
        sandcastle_dir = Path(__file__).resolve().parents[3]
        engine_dirs.append(sandcastle_dir / "cf_pipeline" / "cf_pipeline_engine")
    except Exception:
        pass

    seen: set[str] = set()
    ordered_dirs: List[Path] = []
    for entry in engine_dirs:
        key = str(entry)
        if key in seen:
            continue
        seen.add(key)
        ordered_dirs.append(entry)

    config_dirs = ["Release", "RelWithDebInfo", "Debug", "MinSizeRel"]
    exe_names = ["cf_pipeline_v2.exe", "cf_pipeline_v2"]

    candidates: List[Path] = []
    for engine_dir in ordered_dirs:
        build_dir = engine_dir / "build"
        for exe_name in exe_names:
            candidates.append(build_dir / exe_name)
        for config_dir in config_dirs:
            for exe_name in exe_names:
                candidates.append(build_dir / config_dir / exe_name)
                candidates.append(build_dir / "bin" / config_dir / exe_name)

    for exe_name in exe_names:
        candidates.append(Path(exe_name))

    for candidate in candidates:
        if candidate.exists():
            return str(candidate)
    return None


def _find_pipeline_exe() -> str:
    env_override = os.environ.get("CF_PIPELINE_V2_EXE")
    if env_override:
        return env_override

    packaged = _find_packaged_pipeline_exe()
    if packaged:
        return packaged

    on_path = shutil.which("cf_pipeline_v2") or shutil.which("cf_pipeline_v2.exe")
    if on_path:
        return on_path

    repo_build = _find_repo_build_pipeline_exe()
    if repo_build:
        return repo_build

    return "cf_pipeline_v2"


def _run_engine(
    pipeline_path: Path,
    *,
    steps_paths: List[str],
    plugin_paths: List[str],
    engine_interval_seconds: float = 0.0,
    engine_iterations: int = 0,
    engine_duration_seconds: float = 0.0,
    exe: Optional[str] = None,
    env_overrides: Optional[Dict[str, str]] = None,
) -> int:
    def _build_engine_env() -> Optional[Dict[str, str]]:
        needs_env = bool(env_overrides) or bool(plugin_paths)
        if not needs_env:
            return None

        engine_env = os.environ.copy()

        if plugin_paths:
            prepend_entries: List[str] = []
            seen_keys: set[str] = set()
            for plugin_path in plugin_paths:
                entry = str(plugin_path).strip()
                if not entry:
                    continue
                key = entry.lower() if os.name == "nt" else entry
                if key in seen_keys:
                    continue
                seen_keys.add(key)
                prepend_entries.append(entry)
            if prepend_entries:
                current_path = engine_env.get("PATH", "")
                if current_path:
                    engine_env["PATH"] = os.pathsep.join(prepend_entries + [current_path])
                else:
                    engine_env["PATH"] = os.pathsep.join(prepend_entries)

        if env_overrides:
            for key, value in env_overrides.items():
                engine_env[str(key)] = str(value)

        return engine_env

    engine = exe or _find_pipeline_exe()
    cmd = [engine, "--pipeline", str(pipeline_path)]
    for step_path in steps_paths:
        cmd.extend(["--steps", step_path])
    for plugin_path in plugin_paths:
        cmd.extend(["--plugins", plugin_path])
    if engine_interval_seconds and engine_interval_seconds > 0:
        cmd.extend(["--interval", str(engine_interval_seconds)])
    if engine_duration_seconds and engine_duration_seconds > 0:
        cmd.extend(["--duration", str(engine_duration_seconds)])
    elif engine_iterations and engine_iterations > 0:
        cmd.extend(["--iterations", str(engine_iterations)])
    engine_env = _build_engine_env()
    if engine_env is None:
        proc = subprocess.run(cmd, check=False)
    else:
        proc = subprocess.run(cmd, check=False, env=engine_env)
    return int(proc.returncode or 0)


def _materialize_engine_document(
    value: Any,
    *,
    target_path: Path,
    label: str,
) -> Path:
    document = load_rdf_document(value, label=label)
    target_path.parent.mkdir(parents=True, exist_ok=True)
    target_path.write_text(canonical_json(document), encoding="utf-8")
    return target_path


def _load_pipeline_state(
    con: duckdb.DuckDBPyConnection,
    pipeline_id: str,
) -> Optional[PipelineStateRow]:
    row = con.execute(
        """
        SELECT
          pipeline_id,
          current_rev,
          desired_state,
          concurrency_limit,
          priority,
          last_run_id,
          last_run_at,
          next_run_at
        FROM pipeline_state
        WHERE pipeline_id = ?
        """,
        [pipeline_id],
    ).fetchone()
    if not row:
        return None
    return PipelineStateRow(
        pipeline_id=str(row[0]),
        current_rev=int(row[1]),
        desired_state=str(row[2]),
        concurrency_limit=int(row[3] or 0),
        priority=int(row[4] or 0),
        last_run_id=str(row[5]) if row[5] is not None else None,
        last_run_at=_normalize_timestamp(row[6]),
        next_run_at=_normalize_timestamp(row[7]),
    )


def _ensure_pipeline_state(
    con: duckdb.DuckDBPyConnection,
    pipeline_id: str,
    current_rev: int,
    *,
    desired_state: str = "enabled",
    concurrency_limit: int = 1,
    priority: int = 0,
) -> None:
    con.execute(
        """
        INSERT INTO pipeline_state (
          pipeline_id,
          current_rev,
          desired_state,
          concurrency_limit,
          priority,
          created_at,
          updated_at
        )
        VALUES (?, ?, ?, ?, ?, current_timestamp, current_timestamp)
        ON CONFLICT (pipeline_id) DO UPDATE SET
          current_rev = EXCLUDED.current_rev,
          updated_at = now()
        """,
        [
            pipeline_id,
            int(current_rev),
            str(desired_state),
            int(concurrency_limit),
            int(priority),
        ],
    )


def _update_pipeline_state(
    con: duckdb.DuckDBPyConnection,
    pipeline_id: str,
    *,
    desired_state: Optional[str] = None,
    concurrency_limit: Optional[int] = None,
    priority: Optional[int] = None,
    last_run_id: Optional[str] = None,
    last_run_at: Optional[datetime] = None,
    next_run_at: Optional[datetime] = None,
) -> None:
    updates: List[str] = []
    params: List[Any] = []
    if desired_state is not None:
        updates.append("desired_state = ?")
        params.append(str(desired_state))
    if concurrency_limit is not None:
        updates.append("concurrency_limit = ?")
        params.append(int(concurrency_limit))
    if priority is not None:
        updates.append("priority = ?")
        params.append(int(priority))
    if last_run_id is not None:
        updates.append("last_run_id = ?")
        params.append(last_run_id)
    if last_run_at is not None:
        updates.append("last_run_at = ?")
        params.append(last_run_at)
    if next_run_at is not None:
        updates.append("next_run_at = ?")
        params.append(next_run_at)

    if not updates:
        return

    updates.append("updated_at = current_timestamp")
    params.append(pipeline_id)
    con.execute(
        f"UPDATE pipeline_state SET {', '.join(updates)} WHERE pipeline_id = ?",
        params,
    )


def _count_running_runs(con: duckdb.DuckDBPyConnection, pipeline_id: str) -> int:
    row = con.execute(
        "SELECT COUNT(*) FROM pipeline_runs WHERE pipeline_id = ? AND status = ?",
        [pipeline_id, "running"],
    ).fetchone()
    return int(row[0] or 0) if row else 0


def _start_run(
    con: duckdb.DuckDBPyConnection,
    *,
    run_id: str,
    queue_id: str,
    worker_id: str,
    lease_seconds: int = 60,
) -> None:
    lock_expires = datetime.now(timezone.utc) + timedelta(seconds=lease_seconds)
    queue_row = con.execute(
        """
        SELECT
          pipeline_id,
          run_id,
          available_at,
          priority,
          attempts,
          last_error,
          created_at
        FROM run_queue
        WHERE queue_id = ?
        """,
        [queue_id],
    ).fetchone()
    if queue_row:
        con.execute("DELETE FROM run_queue WHERE queue_id = ?", [queue_id])
        con.execute(
            """
            INSERT INTO run_queue (
              queue_id,
              pipeline_id,
              run_id,
              available_at,
              priority,
              locked_by,
              lock_expires_at,
              attempts,
              last_error,
              created_at,
              updated_at
            )
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, current_timestamp)
            """,
            [
                queue_id,
                str(queue_row[0]),
                str(queue_row[1]),
                queue_row[2],
                int(queue_row[3] or 0),
                worker_id,
                lock_expires,
                int(queue_row[4] or 0) + 1,
                str(queue_row[5]) if queue_row[5] is not None else None,
                queue_row[6],
            ],
        )

    run_row = con.execute(
        """
        SELECT
          pipeline_id,
          session_id,
          session_run_index,
          pipeline_rev,
          triggered_by_event_id,
          created_at,
          attempt
        FROM pipeline_runs
        WHERE run_id = ?
        """,
        [run_id],
    ).fetchone()
    if not run_row:
        return
    con.execute("DELETE FROM pipeline_runs WHERE run_id = ?", [run_id])
    con.execute(
        """
        INSERT INTO pipeline_runs (
          run_id,
          pipeline_id,
          session_id,
          session_run_index,
          pipeline_rev,
          status,
          triggered_by_event_id,
          created_at,
          started_at,
          finished_at,
          termination_reason,
          worker_id,
          attempt
        )
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, current_timestamp, NULL, NULL, ?, ?)
        """,
        [
            run_id,
            str(run_row[0]),
            str(run_row[1]),
            int(run_row[2] or 0),
            int(run_row[3] or 0),
            "running",
            str(run_row[4]) if run_row[4] is not None else None,
            run_row[5],
            worker_id,
            int(run_row[6] or 0),
        ],
    )


def _finish_run(
    con: duckdb.DuckDBPyConnection,
    *,
    run_id: str,
    queue_id: str,
    status: str,
    termination_reason: Optional[str] = None,
) -> None:
    run_row = con.execute(
        """
        SELECT
          pipeline_id,
          session_id,
          session_run_index,
          pipeline_rev,
          triggered_by_event_id,
          created_at,
          started_at,
          worker_id,
          attempt
        FROM pipeline_runs
        WHERE run_id = ?
        """,
        [run_id],
    ).fetchone()
    if run_row:
        con.execute("DELETE FROM pipeline_runs WHERE run_id = ?", [run_id])
        con.execute(
            """
            INSERT INTO pipeline_runs (
              run_id,
              pipeline_id,
              session_id,
              session_run_index,
              pipeline_rev,
              status,
              triggered_by_event_id,
              created_at,
              started_at,
              finished_at,
              termination_reason,
              worker_id,
              attempt
            )
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, current_timestamp, ?, ?, ?)
            """,
            [
                run_id,
                str(run_row[0]),
                str(run_row[1]),
                int(run_row[2] or 0),
                int(run_row[3] or 0),
                status,
                str(run_row[4]) if run_row[4] is not None else None,
                run_row[5],
                run_row[6],
                termination_reason,
                str(run_row[7]) if run_row[7] is not None else None,
                int(run_row[8] or 0),
            ],
        )
    con.execute("DELETE FROM run_queue WHERE queue_id = ?", [queue_id])


def _get_current_rev(pipelines_db_path: Path, pipeline_id: str) -> int:
    con = duckdb.connect(str(pipelines_db_path), read_only=True)
    try:
        row = con.execute(
            "SELECT current_rev FROM pipelines WHERE pipeline_id = ?",
            [pipeline_id],
        ).fetchone()
    finally:
        con.close()
    if not row:
        raise KeyError(f"Pipeline not found: {pipeline_id}")
    return int(row[0] or 0)


def _get_graph_source_path(
    pipelines_db_path: Path,
    pipeline_id: str,
    rev: int,
) -> Optional[Path]:
    con = duckdb.connect(str(pipelines_db_path), read_only=True)
    try:
        row = con.execute(
            """
            SELECT pr.graph_id
            FROM pipeline_revisions pr
            WHERE pr.pipeline_id = ? AND pr.rev = ?
            """,
            [pipeline_id, int(rev)],
        ).fetchone()
        if not row:
            return None
        graph_id = str(row[0])
        path_row = con.execute(
            "SELECT path FROM graphs WHERE graph_id = ?",
            [graph_id],
        ).fetchone()
    finally:
        con.close()
    if not path_row or path_row[0] is None:
        return None
    try:
        return Path(str(path_row[0])).resolve()
    except Exception:
        return None


def _defer_pending_event(
    state_store: PipelineStateStore,
    con: duckdb.DuckDBPyConnection,
    event: PendingRunEvent,
    *,
    reason: str,
    now: datetime,
    delay_seconds: float,
    last_error: Optional[str] = None,
) -> None:
    delay = max(float(delay_seconds or 0.0), 0.2)
    next_attempt = now + timedelta(seconds=delay)
    state_store.defer_event(
        event_id=event.event_id,
        reason=reason,
        next_attempt_at=next_attempt,
        last_error=last_error,
        con=con,
    )


def activate_pipeline_state(
    pipeline_id: str,
    *,
    desired_state: str = "enabled",
    concurrency_limit: int = 1,
    priority: int = 0,
) -> PipelineStateRow:
    om = OntologyManager()
    state_store = PipelineStateStore(
        db_path=om.pipeline_states_db_path,
        semantics_dir=om._semantics_dir,
    )
    state_store.init_schema()

    current_rev = _get_current_rev(om.pipelines_db_path, pipeline_id)
    con = state_store._connect(read_only=False)
    try:
        _ensure_pipeline_state(
            con,
            pipeline_id,
            current_rev,
            desired_state=desired_state,
            concurrency_limit=concurrency_limit,
            priority=priority,
        )
        _update_pipeline_state(
            con,
            pipeline_id,
            desired_state=desired_state,
            concurrency_limit=concurrency_limit,
            priority=priority,
        )
        row = _load_pipeline_state(con, pipeline_id)
    finally:
        con.close()
    if row is None:
        raise RuntimeError("Failed to activate pipeline state")
    return row


def set_pipeline_state(pipeline_id: str, desired_state: str) -> None:
    om = OntologyManager()
    state_store = PipelineStateStore(
        db_path=om.pipeline_states_db_path,
        semantics_dir=om._semantics_dir,
    )
    state_store.init_schema()
    con = state_store._connect(read_only=False)
    try:
        _update_pipeline_state(con, pipeline_id, desired_state=desired_state)
    finally:
        con.close()


def run_event_loop(
    pipeline_id: str,
    *,
    worker_id: Optional[str] = None,
    poll_interval: float = 1.0,
    once: bool = False,
    invocation_document: Optional[str] = None,
) -> int:
    om = OntologyManager()
    state_store = PipelineStateStore(
        db_path=om.pipeline_states_db_path,
        semantics_dir=om._semantics_dir,
    )
    state_store.init_schema()

    doc = om.get_pipeline_revision(pipeline_id, flatten=True)
    if invocation_document:
        invocation_path = Path(invocation_document).expanduser().resolve()
        invocation_doc = load_rdf_document(invocation_path, label="invocation document")
        doc = _apply_invocation_overrides(om, doc, invocation_doc)
        logger.info("Invocation overrides loaded from %s", invocation_path)
    runtime_info = _extract_runtime_info(doc)

    current_rev = _get_current_rev(om.pipelines_db_path, pipeline_id)
    source_path = _get_graph_source_path(om.pipelines_db_path, pipeline_id, current_rev)
    base_path = source_path.parent if source_path else None

    resolved_steps = _resolve_paths(runtime_info.steps_paths, base_path)
    resolved_plugins = _resolve_paths(runtime_info.plugin_paths, base_path)
    missing_plugins = [path for path in resolved_plugins if not Path(path).exists()]
    if missing_plugins:
        logger.warning(
            "Ignoring %d missing plugin path(s) from pipeline header: %s",
            len(missing_plugins),
            ", ".join(missing_plugins),
        )
    resolved_plugins = _existing_paths(resolved_plugins)
    if not resolved_plugins:
        discovered_plugins = _discover_installed_plugin_dirs()
        if discovered_plugins:
            logger.info(
                "No valid plugin paths in pipeline header. Falling back to installed plugin dirs: %s",
                ", ".join(discovered_plugins),
            )
            resolved_plugins = discovered_plugins

    worker = worker_id or f"worker-{uuid.uuid4().hex[:8]}"

    con = state_store._connect(read_only=False)
    try:
        _ensure_pipeline_state(con, pipeline_id, current_rev)
    finally:
        con.close()

    def _now_for_state(state: PipelineStateRow) -> datetime:
        if state.last_run_at is not None and state.last_run_at.tzinfo is None:
            return datetime.now()
        return datetime.now(timezone.utc)

    loop_started_at: Optional[datetime] = None
    auto_sleep_triggered = False

    while True:
        con = state_store._connect(read_only=False)
        run_id: Optional[str] = None
        queue_id: Optional[str] = None
        now = datetime.now(timezone.utc)
        pending_event: Optional[PendingRunEvent] = None
        try:
            state = _load_pipeline_state(con, pipeline_id)
            if state is None:
                raise RuntimeError("Pipeline state missing")

            now = _now_for_state(state)
            if loop_started_at is None:
                loop_started_at = now

            pending_event = state_store.fetch_next_pending_event(
                pipeline_id=pipeline_id,
                now=now,
                con=con,
            )
            if pending_event is None:
                if once:
                    return 0
                time.sleep(poll_interval)
                continue

            desired = state.desired_state.lower().strip()
            if desired in {"disabled", "paused", "sleep"}:
                _defer_pending_event(
                    state_store,
                    con,
                    pending_event,
                    reason=f"state_{desired}",
                    now=now,
                    delay_seconds=max(poll_interval, DEFAULT_EVENT_DEFER_SECONDS),
                )
                if once:
                    return 0
                time.sleep(poll_interval)
                continue

            if (
                runtime_info.auto_sleep_seconds > 0
                and not auto_sleep_triggered
                and (now - loop_started_at).total_seconds() >= runtime_info.auto_sleep_seconds
            ):
                _update_pipeline_state(con, pipeline_id, desired_state="sleep")
                auto_sleep_triggered = True
                _defer_pending_event(
                    state_store,
                    con,
                    pending_event,
                    reason="auto_sleep",
                    now=now,
                    delay_seconds=max(poll_interval, DEFAULT_EVENT_DEFER_SECONDS),
                )
                if once:
                    return 0
                time.sleep(poll_interval)
                continue

            if runtime_info.event_type and pending_event.event_type != runtime_info.event_type:
                _defer_pending_event(
                    state_store,
                    con,
                    pending_event,
                    reason="event_type_mismatch",
                    now=now,
                    delay_seconds=max(poll_interval, 2.0),
                    last_error=(
                        f"expected event_type '{runtime_info.event_type}', "
                        f"got '{pending_event.event_type}'"
                    ),
                )
                if once:
                    return 0
                time.sleep(poll_interval)
                continue

            if runtime_info.event_type and "opcua" in runtime_info.event_type:
                if not _opcua_server_reachable(runtime_info.opcua_endpoint):
                    endpoint = runtime_info.opcua_endpoint or "<missing>"
                    message = f"Could not connect to OPC UA endpoint: {endpoint}"
                    _defer_pending_event(
                        state_store,
                        con,
                        pending_event,
                        reason="opcua_unreachable",
                        now=now,
                        delay_seconds=max(poll_interval, DEFAULT_EVENT_DEFER_SECONDS),
                        last_error=message,
                    )
                    if once:
                        logger.error(message)
                        return OPCUA_ENDPOINT_UNREACHABLE_EXIT_CODE
                    logger.warning(message)
                    time.sleep(poll_interval)
                    continue

            running = _count_running_runs(con, pipeline_id)
            if running >= max(state.concurrency_limit, 1):
                _defer_pending_event(
                    state_store,
                    con,
                    pending_event,
                    reason="concurrency_limit",
                    now=now,
                    delay_seconds=max(poll_interval, DEFAULT_EVENT_DEFER_SECONDS),
                )
                if once:
                    return 0
                time.sleep(poll_interval)
                continue

            if state.last_run_at is not None:
                delta = (now - state.last_run_at).total_seconds()
                min_interval = max(runtime_info.min_interval_seconds, 0.0)
                if delta < min_interval:
                    wait_seconds = max(min_interval - delta, poll_interval, 0.2)
                    _defer_pending_event(
                        state_store,
                        con,
                        pending_event,
                        reason="min_interval",
                        now=now,
                        delay_seconds=wait_seconds,
                    )
                    if once:
                        return 0
                    time.sleep(poll_interval)
                    continue

            consumed = state_store.consume_event_to_run(
                event_id=pending_event.event_id,
                pipeline_rev=state.current_rev,
                priority=state.priority,
                now=now,
                con=con,
            )
            if consumed is None:
                if once:
                    return 0
                time.sleep(poll_interval)
                continue

            run_id, queue_id, _ = consumed
            next_run_at = None
            if runtime_info.min_interval_seconds > 0:
                next_run_at = now + timedelta(seconds=runtime_info.min_interval_seconds)
            _update_pipeline_state(
                con,
                pipeline_id,
                last_run_id=run_id,
                last_run_at=now,
                next_run_at=next_run_at,
            )
            _start_run(con, run_id=run_id, queue_id=queue_id, worker_id=worker)
        finally:
            con.close()

        if run_id is None or queue_id is None:
            if once:
                return 0
            time.sleep(poll_interval)
            continue

        pipeline_payload = doc
        # Persist run-local engine inputs under semantics/runs instead of touching
        # the original repo documents.
        temp_dir = om._semantics_dir / "runs"
        temp_dir.mkdir(parents=True, exist_ok=True)
        run_pipeline_path = temp_dir / f"{pipeline_id}_{run_id}.engine.json"
        try:
            _materialize_engine_document(
                pipeline_payload,
                target_path=run_pipeline_path,
                label="run pipeline document",
            )
        except Exception:
            logger.warning("Failed to write structured engine pipeline document to %s", run_pipeline_path)

        run_steps_paths: List[str] = []
        for step_index, step_path in enumerate(resolved_steps):
            target_path = temp_dir / f"{pipeline_id}_{run_id}_step_{step_index}.engine.json"
            try:
                materialized = _materialize_engine_document(
                    Path(step_path),
                    target_path=target_path,
                    label="run step document",
                )
                run_steps_paths.append(str(materialized))
            except Exception:
                logger.warning("Failed to materialize step document for engine input: %s", step_path)
                run_steps_paths.append(step_path)

        exit_code = _run_engine(
            run_pipeline_path,
            steps_paths=run_steps_paths,
            plugin_paths=resolved_plugins,
            engine_interval_seconds=runtime_info.engine_interval_seconds,
            engine_iterations=runtime_info.engine_iterations,
            engine_duration_seconds=runtime_info.engine_duration_seconds,
            env_overrides={"CF_ALLOW_DIRECT_ENGINE": "1"},
        )

        con = state_store._connect(read_only=False)
        try:
            if exit_code == 0:
                _finish_run(con, run_id=run_id, queue_id=queue_id, status="succeeded")
            else:
                _finish_run(
                    con,
                    run_id=run_id,
                    queue_id=queue_id,
                    status="failed",
                    termination_reason=f"engine_exit_{exit_code}",
                )
        finally:
            con.close()

        if once:
            return exit_code

        time.sleep(poll_interval)

    return 0
