"""
NoETL V2 Worker with NATS Integration

Pure event-sourced worker that:
1. Subscribes to NATS JetStream for command notifications (event_id references)
2. Fetches command details from GET /api/commands/{event_id} (reads command.issued event)
3. Executes based on tool.kind
4. Emits events back to server (POST /api/events)

Single source of truth: event table. No queue table.
"""

import asyncio
import json
import logging
import httpx
import os
import random
import time
import uuid
from collections import OrderedDict
from typing import Optional, Any, Literal
from datetime import datetime, timezone


from noetl.core.messaging import NATSCommandSubscriber
from noetl.worker.adaptive_concurrency import AdaptiveConcurrencyController
from noetl.core.logging_context import LoggingContext
from noetl.core.logger import setup_logger
from noetl.core.urls import (
    build_api_url as _api_url,
    normalize_server_base_url as _normalize_server_base_url,
)
logger = setup_logger(__name__, include_location=True)


def _safe_keys(value: Any, max_items: int = 10) -> list[str]:
    if isinstance(value, dict):
        return list(value.keys())[:max_items]
    if isinstance(value, (list, tuple)):
        return [str(v) for v in list(value)[:max_items]]
    return []


# Pre-import tool executors at module level to avoid 5s cold-start delay
# These were previously imported inside _execute_tool() causing slow first execution
from noetl.tools import http, postgres, duckdb, python
from noetl.tools.http import execute_http_task
from noetl.tools.postgres import execute_postgres_task, execute_postgres_task_async
from noetl.tools.duckdb import execute_duckdb_task
from noetl.tools.snowflake import execute_snowflake_task
from noetl.tools.gcs import execute_gcs_task
from noetl.tools.transfer import execute_transfer_action
from noetl.tools.transfer.snowflake_transfer import execute_snowflake_transfer_action
from noetl.tools.script import execute_script_task
from noetl.tools.agent import execute_agent_task
from noetl.core.secrets import execute_secrets_task
from noetl.core.workflow.workbook import execute_workbook_task
from noetl.core.workflow.playbook import execute_playbook_task
from noetl.tools.python import execute_python_task_async
from jinja2 import Environment, BaseLoader
from noetl.core.storage import Scope, default_store, estimate_size
from noetl.worker.keychain_resolver import populate_keychain_context
from noetl.worker.case_evaluator import CaseEvaluator, build_eval_context
from noetl.worker.result_handler import ResultHandler, is_result_ref


# Module-level template cache for worker - avoids compiling same templates repeatedly
class _WorkerTemplateCache:
    """LRU cache for compiled Jinja2 templates in worker. Memory bounded."""

    def __init__(self, max_size: int = 500):
        self._cache: OrderedDict[str, Any] = OrderedDict()
        self._max_size = max_size
        self._env = None  # Lazy initialization
        self._hits = 0
        self._misses = 0
        self._evictions = 0

    def get_env(self):
        """Get or create Jinja2 environment."""
        if self._env is None:
            from jinja2 import Environment
            self._env = Environment()
        return self._env

    def get_or_compile(self, template_str: str) -> Any:
        """Get compiled template from cache or compile and cache it."""
        if template_str in self._cache:
            self._cache.move_to_end(template_str)
            self._hits += 1
            return self._cache[template_str]

        self._misses += 1
        compiled = self.get_env().from_string(template_str)

        if len(self._cache) >= self._max_size:
            self._cache.popitem(last=False)
            self._evictions += 1

        self._cache[template_str] = compiled

        # Log stats periodically
        if self._misses % 100 == 0:
            logger.debug(
                f"[TEMPLATE-CACHE] Worker stats: size={len(self._cache)}/{self._max_size}, "
                f"hits={self._hits}, misses={self._misses}, hit_rate={self._hits / (self._hits + self._misses) * 100:.1f}%"
            )

        return compiled

    def stats(self) -> dict:
        """Return cache statistics."""
        total = self._hits + self._misses
        return {
            "size": len(self._cache),
            "max_size": self._max_size,
            "hits": self._hits,
            "misses": self._misses,
            "evictions": self._evictions,
            "hit_rate": (self._hits / total * 100) if total > 0 else 0.0
        }


_template_cache = _WorkerTemplateCache(max_size=500)


class ClaimTerminalError(RuntimeError):
    """Terminal claim error after retries are exhausted or command is invalid."""

    def __init__(self, status_code: int, message: str, retryable: bool):
        super().__init__(message)
        self.status_code = status_code
        self.retryable = retryable


class V2Worker:
    """
    V2 Worker that receives command notifications from NATS and executes them.
    
    Architecture (Pure Event Sourcing):
    - Subscribes to NATS JetStream for command notifications
    - Receives lightweight message with {execution_id, event_id, command_id, step, server_url}
    - Fetches full command from GET /api/commands/{event_id} (reads command.issued event)
    - Executes tool based on tool.kind
    - Emits events to POST /api/events (command.completed or command.failed)
    - Single source of truth: event table. No queue table.
    """
    
    def __init__(
        self,
        worker_id: str,
        nats_url: Optional[str] = None,
        server_url: Optional[str] = None
    ):
        from noetl.core.config import get_worker_settings
        worker_settings = get_worker_settings()
        self.worker_id = worker_id
        self.nats_url = nats_url or worker_settings.nats_url
        self.server_url = server_url  # Fallback, usually comes from notification
        self._running = False
        self._http_client: Optional[httpx.AsyncClient] = None
        self._nats_subscriber: Optional[NATSCommandSubscriber] = None
        self._heartbeat_task: Optional[asyncio.Task] = None
        self._registered = False
        self._max_inflight_commands = max(1, int(worker_settings.max_inflight_commands))
        self._max_inflight_db_commands = max(1, int(worker_settings.max_inflight_db_commands))
        self._throttle_poll_interval = float(worker_settings.throttle_poll_interval)
        self._postgres_pool_waiting_threshold = max(0, int(worker_settings.postgres_pool_waiting_threshold))
        self._db_command_semaphore = asyncio.Semaphore(self._max_inflight_db_commands)
        self._last_db_throttle_log_at = 0.0
        self._active_claim_retry_floor_seconds = max(
            1.0,
            min(30.0, float(worker_settings.command_timeout_seconds) / 4.0),
        )
        self._recent_command_activity_ttl_seconds = max(
            self._active_claim_retry_floor_seconds,
            min(120.0, float(worker_settings.command_timeout_seconds)),
        )
        self._recent_command_activity: dict[str, float] = {}
        self._recent_command_activity_last_prune_monotonic = 0.0
        # Adaptive AIMD concurrency controller: limits simultaneous claim/event
        # requests from this worker process, backing off when the server pool
        # is saturated (503) and recovering as it clears.
        self._concurrency = AdaptiveConcurrencyController(
            initial_limit=max(1.0, self._max_inflight_commands / 2.0),
            min_limit=1.0,
            max_limit=float(self._max_inflight_commands),
            probe_interval=worker_settings.concurrency_probe_interval,
        )

    @staticmethod
    def _is_db_heavy_tool(tool_kind: Optional[str]) -> bool:
        return str(tool_kind or "").strip().lower() in {
            "postgres",
            "transfer",
            "snowflake",
            "snowflake_transfer",
            "task_sequence",
        }

    def _is_postgres_pool_saturated(self) -> bool:
        """
        Check plugin pool pressure.

        Returns True when any pool shows active queueing beyond threshold.
        """
        try:
            from noetl.tools.postgres.pool import get_plugin_pool_stats

            pool_stats = get_plugin_pool_stats()
        except Exception:
            return False

        if not pool_stats:
            return False

        for stats in pool_stats.values():
            if not isinstance(stats, dict) or "error" in stats:
                continue
            waiting = int(stats.get("waiting", 0) or 0)
            available = int(stats.get("available", 0) or 0)
            if waiting > self._postgres_pool_waiting_threshold:
                return True
            if waiting > 0 and available <= 0:
                return True

        return False

    def _prune_recent_command_activity(self, now: Optional[float] = None, *, force: bool = False) -> None:
        now = time.monotonic() if now is None else now
        if not force and (now - self._recent_command_activity_last_prune_monotonic) < 5.0:
            return
        self._recent_command_activity_last_prune_monotonic = now
        expired = [
            command_id
            for command_id, expires_at in self._recent_command_activity.items()
            if expires_at <= now
        ]
        for command_id in expired:
            self._recent_command_activity.pop(command_id, None)

    def _remember_recent_command_activity(self, command_id: str, *, ttl_seconds: Optional[float] = None) -> None:
        now = time.monotonic()
        self._prune_recent_command_activity(now, force=len(self._recent_command_activity) >= 2048)
        ttl = max(1.0, float(ttl_seconds or self._recent_command_activity_ttl_seconds))
        self._recent_command_activity[str(command_id)] = now + ttl

    def _is_recent_command_activity(self, command_id: str) -> bool:
        now = time.monotonic()
        self._prune_recent_command_activity(now)
        expires_at = self._recent_command_activity.get(str(command_id))
        if expires_at is None:
            return False
        if expires_at <= now:
            self._recent_command_activity.pop(str(command_id), None)
            return False
        return True

    async def _wait_for_postgres_capacity(self, step: str, command_id: str) -> None:
        """Pause db-heavy command execution while plugin pools are saturated."""
        while self._running and self._is_postgres_pool_saturated():
            now = time.monotonic()
            if now - self._last_db_throttle_log_at >= 5.0:
                logger.info(
                    "[THROTTLE] Delaying db-heavy command %s (%s): postgres pools saturated",
                    command_id,
                    step,
                )
                self._last_db_throttle_log_at = now
            await asyncio.sleep(self._throttle_poll_interval)

    async def _resolve_command_context_if_needed(self, context: Any) -> dict[str, Any]:
        """Resolve a top-level ref-wrapped command context when the server externalized it."""
        if isinstance(context, dict) and context.get("kind") in {"temp_ref", "result_ref"} and context.get("ref"):
            resolved = await default_store.resolve(context)
            if isinstance(resolved, dict):
                return resolved
            raise RuntimeError(
                f"Resolved command context must be an object, got {type(resolved).__name__}"
            )
        return context if isinstance(context, dict) else {}

    def _normalize_command_context_mapping(
        self,
        value: Any,
        *,
        field_name: str,
        step: str,
        execution_id: Any,
    ) -> dict[str, Any]:
        """Accept inline dicts and JSON-stringified dicts for worker command context fields."""
        if value is None:
            return {}
        if isinstance(value, dict):
            return value
        if isinstance(value, str):
            stripped = value.strip()
            if not stripped:
                return {}
            try:
                parsed = json.loads(stripped)
            except json.JSONDecodeError:
                logger.warning(
                    "[COMMAND-CONTEXT] Expected object for %s but got non-JSON string | step=%s execution=%s",
                    field_name,
                    step,
                    execution_id,
                )
                return {}
            if isinstance(parsed, dict):
                return parsed
            logger.warning(
                "[COMMAND-CONTEXT] Expected object for %s but parsed %s | step=%s execution=%s",
                field_name,
                type(parsed).__name__,
                step,
                execution_id,
            )
            return {}
        logger.warning(
            "[COMMAND-CONTEXT] Expected object for %s but got %s | step=%s execution=%s",
            field_name,
            type(value).__name__,
            step,
            execution_id,
        )
        return {}

    def _normalize_command_context_list(
        self,
        value: Any,
        *,
        field_name: str,
        step: str,
        execution_id: Any,
    ) -> list[Any]:
        """Accept inline lists and JSON-stringified lists for worker command context fields."""
        if value is None:
            return []
        if isinstance(value, list):
            return value
        if isinstance(value, str):
            stripped = value.strip()
            if not stripped:
                return []
            try:
                parsed = json.loads(stripped)
            except json.JSONDecodeError:
                logger.warning(
                    "[COMMAND-CONTEXT] Expected list for %s but got non-JSON string | step=%s execution=%s",
                    field_name,
                    step,
                    execution_id,
                )
                return []
            if isinstance(parsed, list):
                return parsed
            logger.warning(
                "[COMMAND-CONTEXT] Expected list for %s but parsed %s | step=%s execution=%s",
                field_name,
                type(parsed).__name__,
                step,
                execution_id,
            )
            return []
        logger.warning(
            "[COMMAND-CONTEXT] Expected list for %s but got %s | step=%s execution=%s",
            field_name,
            type(value).__name__,
            step,
            execution_id,
        )
        return []

    async def _externalize_event_value_if_needed(
        self,
        *,
        execution_id: int,
        step: str,
        event_name: str,
        field_name: str,
        value: Any,
    ) -> Any:
        """Bound large event fields by storing them externally and sending a ref wrapper."""
        if value is None:
            return value
        if is_result_ref(value):
            return value
        if isinstance(value, dict) and value.get("kind") in {"temp_ref", "result_ref"} and value.get("ref"):
            return value

        try:
            size_bytes = estimate_size(value)
        except Exception:
            try:
                size_bytes = len(str(value).encode("utf-8"))
            except Exception:
                return value

        event_inline_max_bytes = max(
            4096,
            int(
                os.getenv(
                    "NOETL_EVENT_INLINE_MAX_BYTES",
                    os.getenv("NOETL_INLINE_MAX_BYTES", "65536"),
                )
            ),
        )
        if size_bytes <= event_inline_max_bytes:
            return value

        try:
            ref = await default_store.put(
                execution_id=str(execution_id),
                name=f"{step}_{event_name}_{field_name}",
                data=value,
                scope=Scope.EXECUTION,
                source_step=step,
                correlation={
                    "event_name": event_name,
                    "field_name": field_name,
                    "worker_id": self.worker_id,
                },
            )
            return {
                "_ref": ref.model_dump(mode="json"),
                "_size_bytes": size_bytes,
                "_store": ref.store.value,
            }
        except Exception as exc:
            logger.warning(
                "[EVENT] Failed to externalize %s.%s for execution %s step=%s: %s",
                event_name,
                field_name,
                execution_id,
                step,
                exc,
            )
            return value

    async def _prepare_event_payload_for_transport(
        self,
        *,
        execution_id: int,
        step: str,
        event_name: str,
        payload: dict[str, Any],
    ) -> dict[str, Any]:
        if not isinstance(payload, dict):
            return payload

        prepared = dict(payload)
        bulky_fields = ("response", "result", "context", "inputs")
        for field_name in bulky_fields:
            if field_name not in prepared:
                continue
            prepared[field_name] = await self._externalize_event_value_if_needed(
                execution_id=execution_id,
                step=step,
                event_name=event_name,
                field_name=field_name,
                value=prepared.get(field_name),
            )
        return prepared
    
    async def _register_worker(self, server_url: str) -> bool:
        """Register this worker in the runtime table via API."""
        try:
            hostname = os.environ.get("HOSTNAME", os.environ.get("POD_NAME", "unknown"))
            register_url = _api_url(server_url, "worker/pool/register")
            
            payload = {
                "name": self.worker_id,
                "component_type": "worker_pool",
                "runtime": "python",
                "status": "ready",
                "capacity": 1,  # Single worker instance
                "pid": os.getpid(),
                "hostname": hostname,
                "labels": {
                    "nats_consumer": os.environ.get("NOETL_WORKER_NATS_CONSUMER", "noetl_worker_pool"),
                },
                "meta": {
                    "nats_url": self.nats_url,
                    "started_at": datetime.now(timezone.utc).isoformat(),
                }
            }
            
            response = await self._http_client.post(register_url, json=payload, timeout=10.0)
            if response.status_code == 200:
                self._registered = True
                logger.debug(f"Worker {self.worker_id} registered in runtime table")
                return True
            else:
                logger.warning(f"Worker registration failed: {response.status_code} - {response.text}")
                return False
        except Exception as e:
            logger.warning(f"Worker registration error: {e}")
            return False
    
    async def _deregister_worker(self, server_url: str) -> bool:
        """Deregister this worker from the runtime table via API."""
        try:
            deregister_url = _api_url(server_url, "worker/pool/deregister")
            
            payload = {
                "name": self.worker_id,
                "component_type": "worker_pool",
            }
            
            response = await self._http_client.post(deregister_url, json=payload, timeout=10.0)
            if response.status_code == 200:
                self._registered = False
                logger.debug(f"Worker {self.worker_id} deregistered from runtime table")
                return True
            else:
                logger.warning(f"Worker deregistration failed: {response.status_code}")
                return False
        except Exception as e:
            logger.warning(f"Worker deregistration error: {e}")
            return False
    
    async def _heartbeat_loop(self, server_url: str):
        """Background task to send heartbeat updates to runtime table."""
        logger.info(f"Worker {self.worker_id} heartbeat loop started (interval: 15s)")
        _cycles = 0
        while self._running:
            try:
                await asyncio.sleep(15)  # Heartbeat every 15 seconds

                if not self._running:
                    break

                _cycles += 1
                # Log a liveness ping every 4 cycles (~60s) to detect silent workers
                if _cycles % 4 == 0:
                    logger.info(
                        "[WORKER] Alive: %s | concurrency=%s",
                        self.worker_id,
                        self._concurrency.get_status(),
                    )

                # Re-register acts as heartbeat (upsert updates heartbeat timestamp)
                await self._register_worker(server_url)

            except asyncio.CancelledError:
                break
            except Exception as e:
                logger.warning(f"Heartbeat error: {e}")
                await asyncio.sleep(5)  # Back off on error

        logger.info(f"Worker {self.worker_id} heartbeat loop stopped")
    
    async def start(self):
        """Start the worker NATS subscription."""
        from noetl.core.config import get_worker_settings
        worker_settings = get_worker_settings()
        self._running = True
        self._http_client = httpx.AsyncClient(timeout=worker_settings.http_client_timeout)
        self._nats_subscriber = NATSCommandSubscriber(
            nats_url=self.nats_url,
            subject=worker_settings.nats_subject,
            consumer_name=worker_settings.nats_consumer,
            stream_name=worker_settings.nats_stream,
            max_inflight=self._max_inflight_commands,
            max_ack_pending=worker_settings.nats_max_ack_pending,
            fetch_timeout=worker_settings.nats_fetch_timeout_seconds,
            fetch_heartbeat=worker_settings.nats_fetch_heartbeat_seconds,
        )

        logger.info(
            "Worker %s starting (NATS: %s, inflight=%s, db_inflight=%s, max_ack_pending=%s)",
            self.worker_id,
            self.nats_url,
            self._max_inflight_commands,
            self._max_inflight_db_commands,
            worker_settings.nats_max_ack_pending,
        )
        
        # Connect to NATS
        await self._nats_subscriber.connect()
        logger.info("Connected to NATS and subscribing to command notifications")
        
        # Register worker in runtime table
        server_url = _normalize_server_base_url(self.server_url or worker_settings.server_url)
        if server_url:
            await self._register_worker(server_url)
            # Start heartbeat background task
            self._heartbeat_task = asyncio.create_task(self._heartbeat_loop(server_url))
        else:
            logger.warning("No server_url configured - worker registration skipped")

        # Start concurrency controller probe (monitors server DB pool proactively)
        if server_url and self._http_client:
            await self._concurrency.start(self._http_client, server_url)

        # Subscribe to command notifications (this should never return)
        await self._nats_subscriber.subscribe(self._handle_command_notification)
    
    async def cleanup(self):
        """Cleanup resources."""
        self._running = False

        # Stop adaptive concurrency controller probe
        await self._concurrency.stop()

        # Cancel heartbeat task
        if self._heartbeat_task:
            self._heartbeat_task.cancel()
            try:
                await self._heartbeat_task
            except asyncio.CancelledError:
                pass
        
        # Deregister from runtime table
        from noetl.core.config import get_worker_settings
        worker_settings = get_worker_settings()
        server_url = _normalize_server_base_url(self.server_url or worker_settings.server_url)
        if server_url and self._http_client and self._registered:
            await self._deregister_worker(server_url)
        
        if self._nats_subscriber:
            await self._nats_subscriber.close()
        if self._http_client:
            await self._http_client.aclose()
        
        # Close all connection pools
        try:
            from noetl.tools.postgres.pool import close_all_plugin_pools
            await close_all_plugin_pools()
            logger.info("Closed all Postgres connection pools")
        except Exception as e:
            logger.warning(f"Error closing connection pools: {e}")
        
        logger.info(f"Worker {self.worker_id} stopped")
    
    def stop(self):
        """Stop the worker."""
        self._running = False
    
    async def _monitor_pool_health(self):
        """
        Background task to monitor connection pool health.
        
        Runs every 5 minutes to:
        - Check pool statistics
        - Log warnings for unhealthy pools
        - Report pool metrics
        """
        logger.info("Started connection pool health monitor")
        
        while self._running:
            try:
                await asyncio.sleep(300)  # Check every 5 minutes
                
                if not self._running:
                    break
                
                from noetl.tools.postgres.pool import get_plugin_pool_stats
                stats = get_plugin_pool_stats()
                
                if not stats:
                    continue
                
                # Log summary
                logger.info(f"Connection pool health check: {len(stats)} active pools")
                
                for pool_key, pool_stats in stats.items():
                    if "error" in pool_stats:
                        logger.warning(f"Pool {pool_key}: {pool_stats['error']}")
                        continue
                    
                    # Check for warning conditions
                    waiting = pool_stats.get('waiting', 0)
                    available = pool_stats.get('available', 0)
                    age = pool_stats.get('age_seconds', 0)
                    
                    if waiting > 5:
                        logger.warning(
                            f"Pool {pool_stats['name']}: {waiting} requests waiting, "
                            f"{available} connections available"
                        )
                    
                    if age > 3600:  # 1 hour
                        logger.info(
                            f"Pool {pool_stats['name']}: Active for {age}s, "
                            f"will be refreshed on next use"
                        )
                    
                    # Log healthy pool stats at debug level
                    logger.debug(
                        f"Pool {pool_stats['name']}: "
                        f"size={pool_stats.get('size')}, "
                        f"available={available}, "
                        f"waiting={waiting}, "
                        f"age={age}s"
                    )
                
            except asyncio.CancelledError:
                break
            except Exception as e:
                logger.error(f"Error in pool health monitor: {e}", exc_info=True)
                await asyncio.sleep(60)  # Back off on error
        
        logger.info("Stopped connection pool health monitor")
    
    async def _check_execution_cancelled(self, server_url: str, execution_id: int) -> bool:
        """
        Check if an execution has been cancelled.
        
        Queries the server for execution.cancelled events.
        
        Returns True if execution is cancelled and should not proceed.
        """
        try:
            # Query execution cancellation status via API
            response = await self._http_client.get(
                _api_url(server_url, f"executions/{execution_id}/cancellation-check"),
                timeout=5.0
            )
            if response.status_code == 200:
                data = response.json()
                if data.get("cancelled", False):
                    logger.info(f"[CANCEL] Execution {execution_id} has been cancelled - skipping command")
                    return True
            return False
        except Exception as e:
            # If we can't check, continue with execution (fail-open)
            logger.warning(f"[CANCEL] Could not check cancellation status for {execution_id}: {e}")
            return False
    
    async def _handle_command_notification(self, notification: dict) -> str:
        """
        Handle command notification from NATS (Event-Driven).

        Notification contains: {execution_id, event_id, command_id, step, server_url}

        Optimized flow using single /api/commands/{event_id}/claim endpoint:
        1. Call claim endpoint (atomically claims + fetches + checks cancellation)
        2. If claim succeeds, execute command
        3. If claim fails (409), another worker got it - silently skip
        """
        t_total_start = time.perf_counter()
        db_slot_acquired = False

        with LoggingContext(logger, notification=notification, execution_id=notification.get("execution_id")):
            try:
                execution_id = notification["execution_id"]
                event_id = notification["event_id"]
                command_id = notification["command_id"]
                step = notification["step"]
                server_url = _normalize_server_base_url(notification["server_url"])

                logger.info(f"[EVENT] Worker {self.worker_id} received notification: exec={execution_id}, command={command_id}, step={step}")

                if self._is_recent_command_activity(command_id):
                    logger.info(
                        "[CLAIM] Command %s was recently claimed/settled on this worker; "
                        "acking duplicate notification before re-claim",
                        command_id,
                    )
                    return "ack"

                # Single atomic call: claim + cancel check + fetch command details
                t_claim_start = time.perf_counter()
                command, claim_decision, retry_after_seconds = await self._claim_and_fetch_command(server_url, event_id)
                t_claim_end = time.perf_counter()
                logger.info(f"[PERF] claim_and_fetch took {(t_claim_end - t_claim_start)*1000:.1f}ms")

                if claim_decision == "retry_later":
                    retry_after_seconds = max(0.0, float(retry_after_seconds))
                    nak_action = "nak"
                    if retry_after_seconds > 0:
                        # Contract: NATSCommandSubscriber interprets "nak:<seconds>"
                        # and calls JetStream `msg.nak(delay=<seconds>)`.
                        nak_action = f"nak:{retry_after_seconds:.3f}"
                    logger.info(
                        "[CLAIM] Deferring command %s (event_id=%s) back to queue for later retry (action=%s)",
                        command_id,
                        event_id,
                        nak_action,
                    )
                    return nak_action

                if command is None:
                    # "skip_ack" means the NATS message should be ACKed with no further
                    # command execution. This covers terminal no-ops and duplicate
                    # notifications for commands already claimed elsewhere.
                    self._remember_recent_command_activity(command_id, ttl_seconds=retry_after_seconds or None)
                    return "ack"

                self._remember_recent_command_activity(command_id)
                logger.info(f"[EVENT] Worker {self.worker_id} claimed command {command_id}")

                command_tool = command.get("action")
                if self._is_db_heavy_tool(command_tool):
                    await self._db_command_semaphore.acquire()
                    db_slot_acquired = True
                    await self._wait_for_postgres_capacity(step=step, command_id=command_id)

                # Execute the command
                t_command_start = time.perf_counter()
                await self._execute_command(command, server_url, command_id)
                t_command_end = time.perf_counter()
                logger.info(f"[PERF] _execute_command for {step} took {(t_command_end - t_command_start)*1000:.1f}ms")

                # Total time
                t_total_end = time.perf_counter()
                logger.info(f"[PERF] TOTAL command handling for {step} took {(t_total_end - t_total_start)*1000:.1f}ms")
                return "ack"

            except ClaimTerminalError as e:
                logger.error(
                    "[CLAIM] Terminal failure for event_id=%s command=%s step=%s status=%s retryable=%s: %s",
                    notification.get("event_id"),
                    notification.get("command_id"),
                    notification.get("step"),
                    e.status_code,
                    e.retryable,
                    e,
                )
                # Do not silently drop failed claims - mark command as failed for diagnostics.
                await self._emit_command_failed(
                    notification["server_url"],
                    int(notification["execution_id"]),
                    notification["command_id"],
                    notification["step"],
                    f"claim_failed status={e.status_code} retryable={str(e.retryable).lower()} message={e}",
                )
                return "ack"
            except Exception as e:
                logger.exception(f"Error handling command notification: {e}")
                return "nak"
            finally:
                if db_slot_acquired:
                    self._db_command_semaphore.release()
    
    async def _claim_and_fetch_command(
        self, server_url: str, event_id: int
    ) -> tuple[Optional[dict], Literal["claimed", "skip_ack", "retry_later"], float]:
        """
        Atomically claim a command and fetch its details in a single call.

        Uses POST /api/commands/{event_id}/claim which:
        1. Checks if execution is cancelled
        2. Acquires advisory lock
        3. Checks if already claimed
        4. Inserts command.claimed event
        5. Returns command details

        Returns:
            - (command, "claimed", 0.0) on successful claim
            - (None, "skip_ack", 0.0) for ACKed no-op outcomes
              (already completed/cancelled or duplicate active-claim notifications)
            - (None, "retry_later", seconds) for transient overload/claim contention
        """
        max_attempts = 5
        delay_seconds = 0.25
        max_delay_seconds = 2.0
        retryable_statuses = {408, 425, 429, 500, 502, 503, 504}

        def _claim_detail(response: httpx.Response) -> tuple[str, str]:
            try:
                payload = response.json()
            except Exception:
                return "", response.text.strip().lower()
            detail = payload.get("detail")
            if isinstance(detail, dict):
                code = detail.get("code")
                message = detail.get("message")
                return (
                    str(code).strip().lower() if code else "",
                    str(message).strip().lower() if message else "",
                )
            if isinstance(detail, str):
                return "", detail.strip().lower()
            return "", ""

        def _parse_retry_after_seconds(raw_value: Optional[str], default: float = 1.0) -> float:
            if raw_value is None:
                return default
            try:
                return max(0.0, float(raw_value))
            except (TypeError, ValueError):
                return default

        for attempt in range(1, max_attempts + 1):
            # Wait for a concurrency slot; respects any global backoff set by
            # previous 503 responses across all claim calls in this process.
            await self._concurrency.acquire()
            _released = False
            try:
                response = await self._http_client.post(
                    _api_url(server_url, f"commands/{event_id}/claim"),
                    json={"worker_id": self.worker_id},
                )

                if response.status_code == 200:
                    await self._concurrency.release_success()
                    _released = True
                    data = response.json()
                    resolved_context = await self._resolve_command_context_if_needed(
                        data.get("context")
                    )
                    logger.info(f"[CLAIM] Successfully claimed command for event_id={event_id}")
                    return ({
                        "execution_id": data["execution_id"],
                        "node_id": data["node_id"],
                        "node_name": data["node_name"],
                        "action": data["action"],
                        "context": resolved_context,
                        "meta": data["meta"]
                    }, "claimed", 0.0)
                if response.status_code == 409:
                    # 409 is not a pool overload — release as success (doesn't penalise limit)
                    await self._concurrency.release_success()
                    _released = True
                    code, message = _claim_detail(response)
                    if code == "active_claim" or "being claimed" in message or "another worker" in message:
                        logger.info(
                            "[CLAIM] Command for event_id=%s is already actively claimed elsewhere; "
                            "acking duplicate notification (code=%s)",
                            event_id,
                            code or "unknown",
                        )
                        # Treat this as an ACKed duplicate notification. The existing
                        # claim remains authoritative, and the command reaper handles
                        # recovery if that worker later dies.
                        return None, "skip_ack", 0.0
                    if code in {"already_terminal", "execution_cancelled"}:
                        logger.info(
                            "[CLAIM] Command for event_id=%s is terminal/cancelled (code=%s), skipping",
                            event_id,
                            code,
                        )
                        return None, "skip_ack", 0.0
                    # Safety: unknown 409 conflicts are treated as transient to prevent
                    # ACK-dropping commands that never reached terminal lifecycle.
                    logger.warning(
                        "[CLAIM] Ambiguous 409 for event_id=%s (code=%s, message=%s); treating as retry_later",
                        event_id,
                        code or "unknown",
                        message[:160] if message else "",
                    )
                    return None, "retry_later", 1.0
                if response.status_code == 404:
                    await self._concurrency.release_error()
                    _released = True
                    raise ClaimTerminalError(
                        status_code=404,
                        message=f"Command not found for event_id={event_id}",
                        retryable=False,
                    )
                if response.status_code in retryable_statuses and attempt < max_attempts:
                    retry_after_seconds = _parse_retry_after_seconds(
                        response.headers.get("Retry-After"),
                        default=1.0,
                    )
                    # Notify controller: server pool saturated.
                    # This applies a global backoff and reduces concurrency limit.
                    # The next acquire() in the next iteration will wait it out.
                    await self._concurrency.release_overload(retry_after_seconds)
                    _released = True
                    jitter_multiplier = 1.0 + random.uniform(-0.2, 0.2)
                    computed_delay = min(delay_seconds * jitter_multiplier, max_delay_seconds)
                    sleep_seconds = min(max(computed_delay, retry_after_seconds), max_delay_seconds)
                    logger.warning(
                        "[CLAIM] Transient claim failure for event_id=%s status=%s attempt=%s/%s; retrying in %.2fs %s",
                        event_id,
                        response.status_code,
                        attempt,
                        max_attempts,
                        sleep_seconds,
                        self._concurrency.get_status(),
                    )
                    await asyncio.sleep(sleep_seconds)
                    delay_seconds = min(delay_seconds * 2, max_delay_seconds)
                    continue
                if response.status_code in retryable_statuses:
                    await self._concurrency.release_overload(1.0)
                    _released = True
                    logger.warning(
                        "[CLAIM] Retryable claim status persisted for event_id=%s (status=%s attempts=%s); deferring to queue redelivery",
                        event_id,
                        response.status_code,
                        attempt,
                    )
                    return None, "retry_later", 1.0

                await self._concurrency.release_error()
                _released = True
                logger.warning(
                    "[CLAIM] Failed to claim command event_id=%s status=%s attempt=%s/%s body=%s",
                    event_id,
                    response.status_code,
                    attempt,
                    max_attempts,
                    response.text[:500],
                )
                raise ClaimTerminalError(
                    status_code=response.status_code,
                    message=f"claim endpoint failed with status={response.status_code}",
                    retryable=response.status_code in retryable_statuses,
                )

            except ClaimTerminalError:
                if not _released:
                    await self._concurrency.release_error()
                raise
            except asyncio.CancelledError:
                # Task cancelled — release slot before propagating
                if not _released:
                    await self._concurrency.release_error()
                raise
            except Exception as e:
                if not _released:
                    await self._concurrency.release_error()
                    _released = True
                if attempt < max_attempts:
                    jitter_multiplier = 1.0 + random.uniform(-0.2, 0.2)
                    sleep_seconds = min(delay_seconds * jitter_multiplier, max_delay_seconds)
                    logger.warning(
                        "[CLAIM] Error claiming command event_id=%s attempt=%s/%s: %s; retrying in %.2fs",
                        event_id,
                        attempt,
                        max_attempts,
                        e,
                        sleep_seconds,
                    )
                    await asyncio.sleep(sleep_seconds)
                    delay_seconds = min(delay_seconds * 2, max_delay_seconds)
                    continue
                logger.warning(
                    "[CLAIM] Network/transport failures persisted for event_id=%s after %s attempts; deferring to queue redelivery: %s",
                    event_id,
                    attempt,
                    e,
                )
                return None, "retry_later", 1.0

        return None, "retry_later", 1.0

    async def _claim_command(self, server_url: str, execution_id: int, command_id: str) -> bool:
        """
        DEPRECATED: Use _claim_and_fetch_command instead.

        Atomically claim a command by emitting command.claimed event.

        Server checks if command is already claimed - returns 409 Conflict if so.

        Returns True if this worker successfully claimed the command.
        """
        try:
            response = await self._http_client.post(
                    _api_url(server_url, "events"),
                json={
                    "execution_id": str(execution_id),
                    "step": command_id.split(":")[1] if ":" in command_id else "unknown",  # Extract step from command_id
                    "name": "command.claimed",
                    "payload": {
                        "command_id": command_id,
                        "worker_id": self.worker_id
                    },
                    "meta": {
                        "worker_id": self.worker_id,
                        "command_id": command_id
                    },
                    "worker_id": self.worker_id
                }
            )

            if response.status_code == 200:
                logger.info(f"[EVENT] Successfully claimed command {command_id}")
                return True
            elif response.status_code == 409:
                # Command already claimed by another worker - this is expected in multi-worker setup
                logger.info(f"[EVENT] Command {command_id} already claimed by another worker, skipping")
                return False
            else:
                logger.warning(f"[EVENT] Failed to claim command {command_id}: {response.status_code}")
                return False

        except Exception as e:
            logger.error(f"[EVENT] Error claiming command {command_id}: {e}")
            return False
    
    async def _fetch_command_details(self, server_url: str, event_id: int) -> Optional[dict]:
        """
        Fetch command details from server API.
        
        Uses the /api/commands/{event_id} endpoint to get command configuration
        from the command.issued event.
        """
        try:
            response = await self._http_client.get(
                _api_url(server_url, f"commands/{event_id}")
            )
            
            if response.status_code == 404:
                logger.error(f"[EVENT] No command.issued event found for event_id={event_id}")
                return None
            
            if response.status_code != 200:
                logger.error(f"[EVENT] Failed to fetch command details: {response.status_code} - {response.text}")
                return None
            
            command = response.json()
            logger.info(f"[EVENT] Fetched command details: step={command.get('node_name')}, tool={command.get('action')}")
            return command
            
        except Exception as e:
            logger.error(f"[EVENT] Error fetching command details: {e}", exc_info=True)
            return None
    
    async def _emit_command_failed(self, server_url: str, execution_id: int, command_id: str, step: str, error_msg: str):
        """Emit command.failed event."""
        try:
            await self._http_client.post(
                _api_url(server_url, "events"),
                json={
                    "execution_id": str(execution_id),
                    "step": step,
                    "name": "command.failed",
                    "payload": {
                        "command_id": command_id,
                        "error": error_msg
                    },
                    "meta": {
                        "worker_id": self.worker_id,
                        "command_id": command_id
                    },
                    "worker_id": self.worker_id
                }
            )
        except Exception as e:
            logger.error(f"[EVENT] Failed to emit command.failed: {e}")
    
    async def _fetch_execution_variables(
        self,
        server_url: str,
        execution_id: int
    ) -> dict:
        """
        Fetch all execution variables from server API.
        
        Returns dict with variable names as keys and their values.
        Used for case condition evaluation when variables are referenced.
        """
        import httpx
        
        try:
            async with httpx.AsyncClient(timeout=10.0) as client:
                response = await client.get(
                    _api_url(server_url, f"vars/{execution_id}"),
                    headers={"Content-Type": "application/json"}
                )
                
                if response.status_code == 200:
                    data = response.json()
                    # Extract just the values from metadata structure
                    # API returns: {variables: {name: {value, type, source_step, ...}}}
                    variables = data.get('variables', {})
                    return {name: meta.get('value') for name, meta in variables.items()}
                else:
                    logger.warning(f"[VARS] Failed to fetch variables: {response.status_code}")
                    return {}
        except Exception as e:
            logger.error(f"[VARS] Error fetching variables: {e}")
            return {}
    
    async def _evaluate_case_blocks_with_event(
        self,
        case_blocks: list,
        response: dict,
        render_context: dict,
        server_url: str,
        execution_id: int,
        step: str,
        event_name: str = "call.done",
        error: str = None
    ) -> dict | None:
        """
        Enhanced case evaluation with explicit event context.

        Evaluates case conditions against:
        - Event context (event.name == 'call.done' or 'call.error')
        - Data context (response.status_code, result.field, error, etc.)

        This allows case blocks to handle both success and error scenarios:

        case:
          - when: "{{ event.name == 'call.error' and response.status_code == 429 }}"
            then:
              retry:
                delay: "{{ response.headers.get('Retry-After', 60) }}"
          - when: "{{ event.name == 'call.done' }}"
            then:
              next:
                - step: process_data

        Args:
            case_blocks: List of case conditions
            response: Tool response/result
            render_context: Full render context
            server_url: Server API URL
            execution_id: Execution ID
            step: Step name
            event_name: Event name ('call.done' or 'call.error')
            error: Optional error message for call.error events

        Returns:
            Action dict {type: 'next'|'retry', details: {...}} or None
        """
        from jinja2 import Environment
        
        if not case_blocks or not isinstance(case_blocks, list):
            return None
        
        logger.info(f"[CASE-EVAL] Evaluating {len(case_blocks)} case blocks | event={event_name} | has_error={error is not None}")
        
        # Create Jinja environment for condition evaluation
        jinja_env = Environment()
        
        # Build hybrid evaluation context with both event and data
        eval_context = {
            **render_context,
            'response': response,
            'result': response,
            'this': response,
            'event': {
                'name': event_name,  # 'call.done' or 'call.error'
                'type': 'tool.completed' if event_name == 'call.done' else 'tool.error',
                'step': step
            },
            'error': error
        }
        
        # Add HTTP-specific context if available
        if isinstance(response, dict):
            # Add status_code to top level for convenience
            if 'status_code' in response:
                eval_context['status_code'] = response['status_code']
            # Add data.status_code for nested HTTP responses
            if isinstance(response.get('data'), dict) and 'status_code' in response['data']:
                eval_context['status_code'] = response['data']['status_code']
        
        # Track if we need to fetch variables from server
        variables_fetched = False
        
        # Check each case block
        for idx, case in enumerate(case_blocks):
            if not isinstance(case, dict):
                continue
            
            when_condition = case.get('when')
            then_block = case.get('then')
            
            if not when_condition or not then_block:
                continue
            
            # Evaluate condition with hybrid context (with retry on missing variables)
            max_retries = 2  # Allow one retry with server variable lookup
            for attempt in range(max_retries):
                try:
                    # Use cached template for performance
                    template = _template_cache.get_or_compile(when_condition)
                    condition_result = template.render(eval_context)
                    
                    # Parse boolean result (Jinja2 returns string)
                    # Jinja2's `and` operator returns the actual value, not "True/False"
                    # e.g., {{ cond1 and some_string }} returns the string if truthy
                    # So we evaluate truthiness: non-empty strings that aren't falsy values
                    result_stripped = condition_result.strip()
                    result_lower = result_stripped.lower()
                    matches = bool(result_stripped) and result_lower not in ['false', '0', 'no', 'none', '']
                    
                    logger.info(f"[CASE-EVAL] Case {idx} condition: {when_condition[:100]}... = {matches}")
                    
                    if not matches:
                        # Condition not met - break retry loop and try next case
                        break
                    
                    # Case matched - extract action
                    logger.info(f"[CASE-EVAL] Case {idx} matched (event={event_name})! Extracting action")
                    
                    # Normalize then_block to list of action dicts
                    # then_block can be: dict, list of dicts, or list with 'next' key
                    if isinstance(then_block, dict):
                        # Single dict - could be {next: [...]} or named task etc
                        if 'next' in then_block:
                            # Special case: {next: [...]} format
                            then_action_list = [then_block]
                        else:
                            # Regular action dict - wrap in list
                            then_action_list = [then_block]
                    elif isinstance(then_block, list):
                        then_action_list = then_block
                    else:
                        then_action_list = []

                    # Process actions: routing (retry/next/set)
                    # Note: named inline tasks are now handled by server
                    has_retry = False
                    has_next = False
                    retry_config = None
                    next_steps = None
                    set_config = None

                    for action in then_action_list:
                        if not isinstance(action, dict):
                            continue

                        if 'retry' in action:
                            has_retry = True
                            # Get raw retry config (may contain Jinja2 templates)
                            raw_retry_config = action['retry']
                            
                            # Render retry args templates with current context
                            # This ensures page numbers and other dynamic values are computed NOW
                            retry_config = {}
                            for key, value in raw_retry_config.items():
                                if key == 'args' and isinstance(value, dict):
                                    # Recursively render args
                                    rendered_args = {}
                                    for arg_key, arg_value in value.items():
                                        if isinstance(arg_value, dict):
                                            # Recursively render nested dicts (e.g., params)
                                            rendered_nested = {}
                                            for nested_key, nested_value in arg_value.items():
                                                if isinstance(nested_value, str) and '{{' in nested_value:
                                                    template = _template_cache.get_or_compile(nested_value)
                                                    rendered_nested[nested_key] = template.render(eval_context)
                                                else:
                                                    rendered_nested[nested_key] = nested_value
                                            rendered_args[arg_key] = rendered_nested
                                        elif isinstance(arg_value, str) and '{{' in arg_value:
                                            template = _template_cache.get_or_compile(arg_value)
                                            rendered_args[arg_key] = template.render(eval_context)
                                        else:
                                            rendered_args[arg_key] = arg_value
                                    retry_config[key] = rendered_args
                                else:
                                    retry_config[key] = value
                            
                            logger.debug(
                                "[CASE-EVAL] Rendered retry config keys=%s",
                                _safe_keys(retry_config),
                            )
                        if 'next' in action:
                            has_next = True
                            next_steps = action['next']
                        if 'set' in action:
                            set_config = action['set']

                    # Handle retry locally in the worker (don't send to server)
                    if has_retry:
                        logger.info(f"[CASE-EVAL] Case {idx} triggered retry - returning retry action for local handling")
                        return {
                            'type': 'retry',
                            'config': retry_config,
                            'case_index': idx,
                            'triggered_by': event_name
                        }
                    
                    if has_next:
                        if isinstance(next_steps, list) and len(next_steps) > 0:
                            return {
                                'type': 'next',
                                'steps': next_steps,
                                'case_index': idx,
                                'triggered_by': event_name
                            }
                    
                    if set_config:
                        logger.info(f"[CASE-EVAL] Case {idx} has set action - updating variables")
                        return {
                            'type': 'set',
                            'config': set_config,
                            'case_index': idx,
                            'triggered_by': event_name
                        }

                    # Successfully evaluated - break retry loop
                    break
                    
                except Exception as e:
                    # Check if error is due to missing variable reference
                    error_msg = str(e)
                    if ('undefined' in error_msg.lower() or 'not defined' in error_msg.lower()) and attempt == 0 and not variables_fetched:
                        # First attempt failed due to missing variable - fetch from server
                        logger.warning(f"[CASE-EVAL] Missing variable in condition, fetching from server: {error_msg}")
                        
                        # Fetch variables from server API
                        server_vars = await self._fetch_execution_variables(server_url, execution_id)
                        
                        if server_vars:
                            logger.debug(
                                "[CASE-EVAL] Fetched %s variables from server (keys=%s)",
                                len(server_vars),
                                _safe_keys(server_vars),
                            )
                            # Merge into eval_context under 'vars' namespace
                            eval_context['vars'] = server_vars
                            variables_fetched = True
                            # Continue to retry with enriched context
                            continue
                        else:
                            logger.error(f"[CASE-EVAL] Failed to fetch variables from server")
                            break
                    else:
                        # Other error or retry exhausted
                        logger.error(f"[CASE-EVAL] Error evaluating case {idx} (attempt {attempt + 1}): {e}")
                        break
        
        # No matching case with routing action
        logger.info(f"[CASE-EVAL] No matching case with routing action found for event={event_name}")
        return None
    
    async def _evaluate_case_blocks(
        self,
        case_blocks: list,
        response: dict,
        render_context: dict,
        server_url: str,
        execution_id: int,
        step: str
    ) -> dict | None:
        """
        Hybrid case evaluation: Worker-side evaluation with both event and data context.

        Evaluates case conditions against:
        - Event context (event.name, event.type, etc.)
        - Data context (response.status_code, result.field, error, etc.)

        Returns action to take: {type: 'next'|'retry', details: {...}}
        If no case matches, returns None.

        NOTE: This is a backward-compatible wrapper. New code should use
        _evaluate_case_blocks_with_event for explicit event context.
        """
        # Delegate to enhanced method with default call.done context
        return await self._evaluate_case_blocks_with_event(
            case_blocks=case_blocks,
            response=response,
            render_context=render_context,
            server_url=server_url,
            execution_id=execution_id,
            step=step,
            event_name="call.done",
            error=response.get('error') if isinstance(response, dict) else None
        )
    
    async def _execute_command(self, command: dict, server_url: str, command_id: str = None):
        """Execute a command and emit events."""
        execution_id = command["execution_id"]
        step = command.get("node_name") or command.get("step")
        tool_kind = command.get("action") or command.get("tool_kind")
        context = command["context"]
        
        # Store execution_id for sub-playbook calls
        self._current_execution_id = execution_id

        tool_config = self._normalize_command_context_mapping(
            context.get("tool_config"),
            field_name="tool_config",
            step=step,
            execution_id=execution_id,
        )
        args = self._normalize_command_context_mapping(
            context.get("args"),
            field_name="args",
            step=step,
            execution_id=execution_id,
        )
        render_context = self._normalize_command_context_mapping(
            context.get("render_context"),
            field_name="render_context",
            step=step,
            execution_id=execution_id,
        )  # Full render context from engine
        case_blocks = self._normalize_command_context_list(
            context.get("case"),
            field_name="case",
            step=step,
            execution_id=execution_id,
        )  # Case blocks from server for immediate execution
        spec = self._normalize_command_context_mapping(
            context.get("spec"),
            field_name="spec",
            step=step,
            execution_id=execution_id,
        )  # Step behavior spec (case_mode, eval_mode)

        # Extract case evaluation settings from spec
        case_mode = "exclusive"  # Default: first match wins (XOR-split)
        eval_mode = "on_entry"   # Default: evaluate once
        if spec:
            case_mode = spec.get("case_mode", "exclusive")
            eval_mode = spec.get("eval_mode", "on_entry")
            logger.debug(f"[SPEC] Step '{step}' spec: case_mode={case_mode}, eval_mode={eval_mode}")
        
        # CRITICAL: Merge tool_config.args with top-level args
        # In V2 DSL, step args are often defined within the tool block
        # e.g., tool: {kind: python, args: {name: "value"}, script: {...}}
        # The engine puts these in tool_config.args, but the worker needs them in args
        if "args" in tool_config:
            # Merge with top-level args taking precedence
            merged_args = {**tool_config["args"], **args}
            args = merged_args
            logger.debug("Args config: merged_from_tool_config | keys=%s", _safe_keys(args))
        else:
            logger.debug("Args config: using_top_level | keys=%s", _safe_keys(args))
        
        logger.info(f"[EVENT] Executing {step} (tool={tool_kind}) for execution {execution_id}" + (f" command={command_id}" if command_id else ""))

        # Extract catalog_id from meta (where server stores it)
        meta = command.get("meta", {})
        catalog_id = meta.get("catalog_id")
        loop_event_id = meta.get("loop_event_id")

        # Ensure execution_id and catalog_id are in render_context for keychain resolution
        if "execution_id" not in render_context:
            render_context["execution_id"] = execution_id
        if catalog_id and "catalog_id" not in render_context:
            render_context["catalog_id"] = catalog_id
            logger.debug(f"[KEYCHAIN] Added catalog_id to render_context for step {step}")
        
        try:
            import time

            # Batch-emit command.started + step.enter in a single HTTP call.
            t_events_start = time.perf_counter()
            initial_event_timeout = 5.0
            initial_events = []
            if command_id:
                initial_events.append({
                    "step": step,
                    "name": "command.started",
                    "payload": {"command_id": command_id, "worker_id": self.worker_id},
                    "actionable": False,
                    "informative": True,
                })
            initial_events.append({
                "step": step,
                "name": "step.enter",
                "payload": {"status": "started"},
                "actionable": False,
                "informative": True,
            })
            await self._emit_batch_events(
                server_url,
                execution_id,
                initial_events,
                timeout_seconds=initial_event_timeout,
                max_retries=2,
                raise_on_failure=False,
            )
            t_events_end = time.perf_counter()
            logger.info(f"[PERF] emit_initial_events (batch) took {(t_events_end - t_events_start)*1000:.1f}ms")

            # Execute tool
            t_tool_start = time.perf_counter()
            response = await self._execute_tool(tool_kind, tool_config, args, step, render_context, case_blocks=case_blocks)
            t_tool_end = time.perf_counter()
            logger.info(f"[PERF] Tool execution for {step} took {(t_tool_end - t_tool_start)*1000:.1f}ms")

            # Process result through ResultHandler for large result externalization
            # This implements output_select pattern: large results stored externally,
            # small extracted fields available in render_context for templating
            # Skip for artifact.get since it's meant to provide full data inline for next step
            t_result_start = time.perf_counter()
            skip_result_processing = tool_kind == "artifact" and tool_config.get("action") == "get"
            if skip_result_processing:
                logger.debug(f"[RESULT] Step {step}: skipping result processing (artifact.get)")
                response_for_events = response
            else:
                try:
                    result_handler = ResultHandler(execution_id=execution_id)
                    # Get output config from tool_config if present
                    output_config = tool_config.get("result", {})
                    processed_response = await result_handler.process_result(
                        step_name=step,
                        result=response,
                        output_config=output_config
                    )
                    # If result was externalized, use processed version
                    if is_result_ref(processed_response):
                        logger.info(
                            f"[RESULT] Step {step}: externalized large result | "
                            f"store={processed_response.get('_store', 'unknown')} | "
                            f"size={processed_response.get('_size_bytes', 0)}b"
                        )
                        # Keep original response for case evaluation, but mark for external storage
                        response_for_events = processed_response
                    else:
                        response_for_events = response
                except Exception as result_err:
                    logger.warning(f"[RESULT] Failed to process result for {step}: {result_err}")
                    response_for_events = response
            t_result_end = time.perf_counter()
            logger.debug(f"[PERF] Result processing for {step} took {(t_result_end - t_result_start)*1000:.1f}ms")

            # Note: _internal_data will be cleaned up before emitting final events
            
            logger.debug(
                "[CASE-CHECK] step=%s case_blocks_present=%s case_blocks_type=%s case_blocks_count=%s",
                step,
                case_blocks is not None,
                type(case_blocks).__name__,
                len(case_blocks) if isinstance(case_blocks, list) else 0,
            )
            logger.debug(f"[DEBUG] After tool execution for step: {step}")
            logger.debug(f"[DEBUG] case_blocks is None: {case_blocks is None}")
            logger.debug(f"[DEBUG] case_blocks bool: {bool(case_blocks)}")
            if case_blocks:
                logger.debug(f"[DEBUG] case_blocks length: {len(case_blocks)}")
                for idx, cb in enumerate(case_blocks):
                    logger.debug("[DEBUG] case_block[%s] keys=%s", idx, _safe_keys(cb))

            logger.debug(f"[DEBUG] context has_case={'case' in context} | case_blocks={case_blocks is not None} | case_count={len(case_blocks) if case_blocks else 0}")
            
            # Check if tool returned error status FIRST (before case evaluation)
            # This allows case blocks to handle errors via call.error event
            # NOTE: task_sequence returns status="failed", other tools use status="error"
            tool_error = None
            error_response = None
            if isinstance(response, dict):
                if response.get('status') in ('error', 'failed'):
                    # Extract error from either 'error' dict (task_sequence) or 'error' string (other tools)
                    error_val = response.get('error')
                    if isinstance(error_val, dict):
                        tool_error = error_val.get('message', str(error_val))
                    else:
                        tool_error = error_val or 'Tool returned error status'
                    error_response = response
                # Also check nested data errors (for tools that return {data: {...}})
                elif isinstance(response.get('data'), dict):
                    for key, value in response['data'].items():
                        if isinstance(value, dict) and value.get('status') == 'error':
                            tool_error = f"{key}: {value.get('message', 'Unknown error')}"
                            error_response = response
                            break
            
            # HYBRID CASE EVALUATION: Worker-side evaluation with both event and data context
            # Evaluate case blocks for BOTH success (call.done) and error (call.error) scenarios
            # Uses CaseEvaluator with proper exclusive/inclusive mode support
            case_action = None
            if case_blocks:
                logger.info(f"[CASE-CHECK] Evaluating case blocks for {step} | mode={case_mode} | has_error={tool_error is not None}")

                # Build evaluation context based on success or error
                eval_event_name = "call.error" if tool_error else "call.done"
                eval_response = error_response if tool_error else response

                # Build evaluation context with all necessary data
                eval_context = build_eval_context(
                    render_context=render_context,
                    response=eval_response,
                    step=step,
                    event_name=eval_event_name,
                    error=tool_error
                )

                # Create evaluator with step's case_mode (default: exclusive)
                evaluator = CaseEvaluator(case_mode=case_mode, eval_mode=eval_mode)
                eval_result = evaluator.evaluate(case_blocks, eval_context, eval_event_name)

                # Convert routing action to legacy format for compatibility
                if eval_result.routing_action:
                    ra = eval_result.routing_action
                    if ra.type == "next":
                        case_action = {
                            'type': 'next',
                            'steps': ra.config.get('steps', []),
                            'case_index': ra.case_index,
                            'triggered_by': ra.triggered_by
                        }
                    elif ra.type == "retry":
                        case_action = {
                            'type': 'retry',
                            'config': ra.config,
                            'case_index': ra.case_index,
                            'triggered_by': ra.triggered_by
                        }
                    elif ra.type == "set":
                        case_action = {
                            'type': 'set',
                            'config': ra.config,
                            'case_index': ra.case_index,
                            'triggered_by': ra.triggered_by
                        }
                
                # If case action resulted in routing (next/retry), report and handle
                if case_action and case_action.get('type') in ['next', 'retry']:
                    logger.info(f"[CASE-ACTION] Case evaluation triggered {case_action['type']} action for {step}")
                    
                    # ARCHITECTURE PRINCIPLE #3:
                    # Report case action to server via ACTIONABLE event
                    # Server will issue new command with rendered args from case_action.config
                    # This follows server-worker-server control loop pattern
                    await self._emit_event(
                        server_url,
                        execution_id,
                        step,
                        "case.evaluated",
                        {
                            "action": case_action,
                            "result": eval_response,  # Data reference only (no full payload)
                            "triggered_by": eval_event_name
                        },
                        actionable=True,  # Server should process this for routing
                        informative=True
                    )
                    
                    # Emit appropriate event based on success or error
                    # IMPORTANT: Use response_for_events for errors too so large failed
                    # payloads are externalized and do not bloat event storage.
                    error_event_response = response_for_events if tool_error else error_response
                    if tool_error:
                        await self._emit_event(
                            server_url,
                            execution_id,
                            step,
                            "call.error",
                            {
                                "error": tool_error,
                                "response": error_event_response,
                                "case_handled": True,
                                "command_id": command_id,
                                "loop_event_id": loop_event_id,
                            },
                            actionable=True,  # Case evaluated - server may route/retry
                            informative=True
                        )
                    else:
                        await self._emit_event(
                            server_url,
                            execution_id,
                            step,
                            "call.done",
                            {
                                "response": response_for_events,
                                "case_handled": True,
                                "command_id": command_id,
                                "loop_event_id": loop_event_id,
                            },
                            actionable=True,
                            informative=True
                        )
                    
                    await self._emit_event(
                        server_url,
                        execution_id,
                        step,
                        "step.exit",
                        {
                            "status": "COMPLETED" if not tool_error else "CASE_HANDLED",
                            "result": eval_response,
                            "case_action": case_action,
                            "command_id": command_id,
                        },
                        actionable=True,  # Server should handle routing
                        informative=True
                    )
                    
                    # Emit command.completed
                    if command_id:
                        await self._emit_event(
                            server_url,
                            execution_id,
                            step,
                            "command.completed",
                            {
                                "command_id": command_id,
                                "worker_id": self.worker_id,
                                "result": eval_response,
                                "case_action": case_action
                            },
                            actionable=False,  # Informational - case.evaluated has action
                            informative=True
                        )
                    
                    logger.info(f"[EVENT] Completed {step} with case action for execution {execution_id}")
                    return  # Exit - server will handle routing based on case_action
            
            # No case action handled it - check for unhandled tool errors
            if tool_error:
                # Tool returned error - treat as failure (no case handled it)
                logger.error(f"Tool execution failed for {step}: {tool_error}")
                # Use processed response (externalized when large) to avoid persisting
                # raw oversized payloads in failure events.
                error_event_response = response_for_events if error_response is not None else error_response
                
                # Emit call.error with error payload - ACTIONABLE for server to handle
                await self._emit_event(
                    server_url,
                    execution_id,
                    step,
                    "call.error",
                    {"error": tool_error, "response": error_event_response},
                    actionable=True,  # Server should evaluate case blocks or fail
                    informative=True
                )
                
                # Emit step.exit with failed status
                await self._emit_event(
                    server_url,
                    execution_id,
                    step,
                    "step.exit",
                    {
                        "status": "FAILED",  # Uppercase to match database status values
                        "error": tool_error,
                        "result": error_event_response,
                        "command_id": command_id,
                    },
                    actionable=True,  # Server may want to handle failure
                    informative=True
                )
                
                # Emit command.failed event
                if command_id:
                    await self._emit_event(
                        server_url,
                        execution_id,
                        step,
                        "command.failed",
                        {
                            "command_id": command_id,
                            "worker_id": self.worker_id,
                            "error": tool_error,
                            "result": error_event_response
                        },
                        actionable=False,  # Informational
                        informative=True
                    )
                
                logger.error(f"[EVENT] Failed {step} for execution {execution_id}" + (f" command={command_id}" if command_id else ""))
                return  # Exit without emitting completed events
            
            # Tool succeeded - emit success events with error recovery
            try:
                # Clean up _internal_data and data before emitting events
                # This prevents large payloads from being stored in events/NATS
                if isinstance(response, dict):
                    response_data = response.get('data', {})
                    if isinstance(response_data, dict):
                        # Remove _internal_data (full response data)
                        internal_data = response_data.pop('_internal_data', None)
                        # Also remove 'data' field which is a duplicate for backwards compat
                        response_data.pop('data', None)

                        if internal_data:
                            logger.info(
                                f"[CLEANUP] Removed _internal_data and data before event emission | "
                                f"payload_size_reduction: ~{len(str(internal_data))} bytes"
                            )
                            logger.info(f"[CLEANUP] Response now contains only reference: {response_data.get('data_reference', {})}")
                
                # Emit call.done event - ACTIONABLE so server evaluates routing
                # Use response_for_events which may have externalized large results
                await self._emit_event(
                    server_url,
                    execution_id,
                    step,
                    "call.done",
                    {
                        "response": response_for_events,
                        "command_id": command_id,
                        "loop_event_id": loop_event_id,
                    },
                    actionable=True,  # Server should evaluate next/case routing
                    informative=True
                )

                # Batch step.exit + command.completed in a single HTTP call
                # (both are informational - call.done already handled routing)
                terminal_events = [
                    {
                        "step": step,
                        "name": "step.exit",
                        "payload": {
                            "result": response_for_events,
                            "status": "completed",
                            "command_id": command_id,
                        },
                        "actionable": False,
                        "informative": True,
                    },
                ]
                if command_id:
                    terminal_events.append({
                        "step": step,
                        "name": "command.completed",
                        "payload": {
                            "command_id": command_id,
                            "worker_id": self.worker_id,
                            "result": response_for_events,
                        },
                        "actionable": False,
                        "informative": True,
                    })
                await self._emit_batch_events(
                    server_url,
                    execution_id,
                    terminal_events,
                    raise_on_failure=False,
                )
                
                logger.info(f"[EVENT] Completed {step} for execution {execution_id}" + (f" command={command_id}" if command_id else ""))
                
            except Exception as emit_error:
                # Event emission failed - try to report failure
                logger.exception(f"Failed to emit success events for {step}: {emit_error}")
                try:
                    # Attempt to emit command.failed to mark execution as failed
                    if command_id:
                        await self._emit_event(
                            server_url,
                            execution_id,
                            step,
                            "command.failed",
                            {
                                "command_id": command_id,
                                "worker_id": self.worker_id,
                                "error": f"Event emission failed: {str(emit_error)}"
                            }
                        )
                except Exception as recovery_error:
                    logger.exception(f"Failed to emit recovery failure event: {recovery_error}")
                # Re-raise so the command handler knows it failed
                raise emit_error
            
        except Exception as e:
            logger.error(f"Execution error for {step}: {e}", exc_info=True)
            
            # Emit error event
            await self._emit_event(
                server_url,
                execution_id,
                step,
                "call.error",
                {"error": str(e)}
            )
            
            # Emit step.exit with error status
            await self._emit_event(
                server_url,
                execution_id,
                step,
                "step.exit",
                {
                    "status": "failed",
                    "error": str(e),
                    "error_type": type(e).__name__,
                    "command_id": command_id,
                }
            )
            
            # Emit command.failed event (for event-driven tracking)
            if command_id:
                await self._emit_event(
                    server_url,
                    execution_id,
                    step,
                    "command.failed",
                    {
                        "command_id": command_id,
                        "worker_id": self.worker_id,
                        "error": str(e),
                        "error_type": type(e).__name__
                    }
                )
            
            # Pure event-driven: No queue table operations needed
    
    async def _execute_tool(
        self,
        tool_kind: str,
        config: dict,
        args: dict,
        step: str,
        render_context: dict,
        case_blocks: Optional[list] = None
    ) -> Any:
        """
        Execute tool using noetl/tools/* implementations.
        
        This delegates to the mature tool system which includes:
        - Script loading (GCS, S3, HTTP, file)
        - Authentication resolution and caching
        - Pagination and retry logic
        - Event callbacks for progress tracking
        - Template rendering
        - Error handling
        
        Args:
            tool_kind: Tool type (http, postgres, python, etc.)
            config: Tool configuration
            args: Tool arguments
            step: Step name
            render_context: Full render context with workload, step results, etc.
            case_blocks: Optional case blocks for result references
        """
        # Tool executors are pre-imported at module level for fast execution
        
        # Use render_context from engine (includes workload, step results, execution_id, etc.)
        # This allows plugins to render Jinja2 templates with full state
        context = render_context if render_context else {"args": args, "step": step}
        
        logger.debug(
            "WORKER: initial context keys=%s execution_id=%s has_catalog_id=%s",
            _safe_keys(context),
            context.get("execution_id"),
            bool(context.get("catalog_id")),
        )
        
        # Note: catalog_id should come from server in render_context
        # Worker does NOT query noetl database - only executes tool steps
        
        # Add job metadata to context for {{ job.uuid }} templates
        if "job" not in context:
            context["job"] = {
                "uuid": context.get("execution_id", ""),
                "execution_id": context.get("execution_id", "")
            }
        
        # Resolve keychain references before tool execution
        # This scans the config for {{ keychain.* }} references and populates context['keychain']
        catalog_id = context.get('catalog_id')
        execution_id = context.get('execution_id')
        logger.debug(
            "[KEYCHAIN] Step=%s has_catalog_id=%s execution_id=%s tool_kind=%s",
            step,
            bool(catalog_id),
            execution_id,
            tool_kind,
        )
        if catalog_id:
            # Combine config and args into single dict for scanning
            task_config_combined = {**config, **args}
            server_url = context.get('server_url', 'http://noetl.noetl.svc.cluster.local:8082')
            
            # Get refresh threshold from settings
            from noetl.core.config import get_worker_settings
            worker_settings = get_worker_settings()
            refresh_threshold = worker_settings.keychain_refresh_threshold
            
            import time
            k_start = time.perf_counter()
            context = await populate_keychain_context(
                task_config=task_config_combined,
                context=context,
                catalog_id=catalog_id,
                execution_id=execution_id,
                api_base_url=server_url,
                refresh_threshold_seconds=refresh_threshold
            )
            k_end = time.perf_counter()
            logger.debug(f"[PERF] populate_keychain_context took {k_end - k_start:.4f}s")
        
        import time
        t_jinja_start = time.perf_counter()
        from jinja2 import Environment, BaseLoader
        from noetl.core.dsl.render import add_b64encode_filter
        from noetl.core.auth.token_resolver import register_token_functions
        
        jinja_env = Environment(loader=BaseLoader())
        jinja_env = add_b64encode_filter(jinja_env)  # Add custom filters including tojson
        register_token_functions(jinja_env, context)
        t_jinja_end = time.perf_counter()
        logger.debug(f"[PERF] Jinja2 setup took {t_jinja_end - t_jinja_start:.4f}s")
        
        # Map V2 config format to plugin task_config format
        # Plugins use different field names than V2 DSL
        # For workbook tool, preserve 'name' field from config (it's the workbook action name)
        # For other tools, add 'name' as step name for logging
        task_config = {**config}
        # Merge args: config["args"] (from pipeline tool spec) + args parameter
        # This preserves tool args from pipeline while allowing parameter override
        config_args = config.get("args", {})
        task_config["args"] = {**config_args, **args} if config_args or args else {}
        if "name" not in config:
            task_config["name"] = step
        
        if tool_kind == "python":
            # Use plugin's execute_python_task_async (not the sync wrapper!)
            # This includes:
            # - Script loading (GCS/S3/HTTP/file)
            # - Base64 code support
            # - Kwargs unpacking
            # - Error handling
            result = await execute_python_task_async(task_config, context, jinja_env, args)
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                # Keep error response intact (worker needs status field to detect error)
                return result
            # Normalize result - plugin returns dict, may need to extract 'data' or 'result'
            if isinstance(result, dict):
                return result.get('data', result.get('result', result))
            return result
            
        elif tool_kind == "http":
            # Check if retry config has pagination/success-driven repeats (next_call or collect)
            # If so, use execute_with_retry which handles per-iteration retries
            retry_config = config.get('retry') if isinstance(config, dict) else None
            has_pagination_retry = False
            if isinstance(retry_config, list):
                for idx, policy in enumerate(retry_config):
                    if isinstance(policy, dict) and 'then' in policy:
                        then_block = policy['then']
                        has_next_call = 'next_call' in then_block if isinstance(then_block, dict) else False
                        has_collect = 'collect' in then_block if isinstance(then_block, dict) else False
                        if isinstance(then_block, dict) and (has_next_call or has_collect):
                            has_pagination_retry = True
                            break

            logger.debug(f"HTTP TOOL: config_keys={list(config.keys()) if isinstance(config, dict) else 'not dict'} | retry={retry_config is not None} | policies={len(retry_config) if isinstance(retry_config, list) else 0} | has_pagination={has_pagination_retry}")

            if has_pagination_retry:
                logger.info("HTTP tool using execute_with_retry for pagination support")
                # Use retry-aware executor which handles per-iteration retries
                from noetl.core.runtime.retry import execute_with_retry
                
                # Create executor function that wraps async execute_http_task
                def http_executor(cfg, ctx, env, task_w):
                    import asyncio
                    try:
                        loop = asyncio.get_running_loop()
                    except RuntimeError:
                        # No running loop - create one
                        return asyncio.run(execute_http_task(cfg, ctx, env, task_w or {}))
                    else:
                        # Running in existing loop - run in executor to avoid blocking
                        import concurrent.futures
                        with concurrent.futures.ThreadPoolExecutor() as pool:
                            future = pool.submit(
                                lambda: asyncio.run(execute_http_task(cfg, ctx, env, task_w or {}))
                            )
                            return future.result()
                
                # Execute with retry handler (supports pagination)
                result = execute_with_retry(
                    http_executor,
                    task_config,
                    step,
                    context,
                    jinja_env,
                    args
                )
                
                # Check if retry handler returned error status
                if isinstance(result, dict) and result.get('status') == 'error':
                    return result
                # Normalize result
                if isinstance(result, dict):
                    return result.get('data', result)
                return result
            else:
                # No pagination retry - use direct execution (original path)
                # Use plugin's execute_http_task which includes:
                # - Auth resolution and credential caching
                # - Template rendering
                # - Response processing
                task_with = args  # Plugin uses 'task_with' for rendered params
                result = await execute_http_task(task_config, context, jinja_env, task_with)
                # Check if plugin returned error status
                if isinstance(result, dict) and result.get('status') == 'error':
                    # Keep error response intact (worker needs status field to detect error)
                    return result
                # Normalize result
                if isinstance(result, dict):
                    return result.get('data', result)
                return result
            
        elif tool_kind == "postgres":
            # Call async postgres executor directly on the worker's event loop.
            # This lets the plugin pool reuse connections (same loop_id → same pool key).
            # The old pattern (run_in_executor + asyncio.run) created a new event loop
            # per call, causing a fresh pool each time and leaking connections.
            task_with = {**config, **args}  # Merge config (has auth) with args
            result = await execute_postgres_task_async(task_config, context, jinja_env, task_with)
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                # Keep error response intact (worker needs status field to detect error)
                return result
            return result.get('data', result) if isinstance(result, dict) else result
            
        elif tool_kind == "duckdb":
            # Use plugin's execute_duckdb_task (sync function - run in executor)
            # Pass full tool config as task_with to ensure auth is available
            task_with = {**config, **args}  # Merge config (has auth) with args
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_duckdb_task(task_config, context, jinja_env, task_with)
            )
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                # Keep error response intact (worker needs status field to detect error)
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "ducklake":
            # Use plugin's execute_ducklake_task (sync function - run in executor)
            from noetl.tools.ducklake import execute_ducklake_task
            # Pass full tool config as task_with to ensure auth is available
            task_with = {**config, **args}  # Merge config (has auth) with args
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_ducklake_task(task_config, context, jinja_env, task_with)
            )
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                # Keep error response intact (worker needs status field to detect error)
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "snowflake":
            # Use plugin's execute_snowflake_task (sync wrapper)
            # Pass full tool config as task_with to ensure auth is available
            task_with = {**config, **args}  # Merge config (has auth) with args
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_snowflake_task(task_config, context, jinja_env, task_with)
            )
            if isinstance(result, dict) and result.get('status') == 'error':
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "transfer":
            # Generic transfer action (may delegate to Snowflake/Postgres executors)
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_transfer_action(task_config, context, jinja_env, args)
            )
            if isinstance(result, dict) and result.get('status') == 'error':
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "snowflake_transfer":
            # Explicit snowflake_transfer action (legacy specialized executor)
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_snowflake_transfer_action(task_config, context, jinja_env, args)
            )
            if isinstance(result, dict) and result.get('status') == 'error':
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "script":
            # Execute script as Kubernetes job (async plugin)
            result = await execute_script_task(task_config, context, jinja_env, args)
            if isinstance(result, dict) and result.get('status') == 'error':
                return result
            return result
            
        elif tool_kind == "secrets":
            # Use plugin's execute_secrets_task (sync function - run in executor)
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_secrets_task(task_config, context, jinja_env)
            )
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                # Keep error response intact (worker needs status field to detect error)
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "workbook":
            # Call async execute_workbook_task directly (don't use executor for async functions)
            result = await execute_workbook_task(task_config, context, jinja_env, args)
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                # Keep error response intact (worker needs status field to detect error)
                return result
            return result.get('data', result) if isinstance(result, dict) else result
            
        elif tool_kind == "playbook":
            # Playbook tasks need merged task args from config["args"] (task_sequence path passes args={}).
            # If return_step is configured, use native async executor with polling support.
            playbook_args = task_config.get("args") if isinstance(task_config.get("args"), dict) else args
            if task_config.get("return_step"):
                return await self._execute_playbook(task_config, playbook_args or {})

            # Fallback to plugin executor for fire-and-forget behavior.
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_playbook_task(task_config, context, jinja_env, playbook_args or {})
            )
            return result
            
        elif tool_kind == "gcs":
            # Use plugin's execute_gcs_task (sync function - run in executor)
            task_with = {**config, **args}  # Merge config (has source, destination, credential) with args
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                None,
                lambda: execute_gcs_task(task_config, context, jinja_env, task_with)
            )
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                # Keep error response intact (worker needs status field to detect error)
                return result
            return result.get('data', result) if isinstance(result, dict) else result
            
        elif tool_kind == "container":
            # Container execution - keep inline for now as it's V2-specific
            return await self._execute_container(config, args)
            
        elif tool_kind == "script":
            # Script loading is now handled by python plugin via 'script' field
            # This is a legacy fallback
            return await self._execute_script(config, args)

        elif tool_kind == "nats":
            # NATS tool for K/V Store, Object Store, and JetStream operations
            from noetl.tools.nats import execute_nats_task
            task_with = {**config, **args}
            result = await execute_nats_task(task_config, context, jinja_env, task_with)
            # Check if plugin returned error status
            if isinstance(result, dict) and result.get('status') == 'error':
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "agent":
            # Agent runtime bridge (ADK/LangChain/custom callable adapters)
            task_with = {**config, **args}
            result = await execute_agent_task(task_config, context, jinja_env, task_with)
            if isinstance(result, dict) and result.get("status") == "error":
                return result
            return result.get("data", result) if isinstance(result, dict) else result

        elif tool_kind == "artifact":
            # Artifact tool for loading/storing externalized results
            from noetl.tools.artifact.executor import execute_artifact_get, execute_artifact_put
            task_with = {**config, **args}
            action = config.get('action', 'get')
            loop = asyncio.get_running_loop()
            if action == 'get':
                result = await loop.run_in_executor(
                    None, execute_artifact_get, task_config, context, jinja_env, task_with, None
                )
            elif action == 'put':
                result = await loop.run_in_executor(
                    None, execute_artifact_put, task_config, context, jinja_env, task_with, None
                )
            else:
                raise ValueError(f"Unknown artifact action: {action}")
            if isinstance(result, dict) and result.get('status') == 'error':
                return result
            return result.get('data', result) if isinstance(result, dict) else result

        elif tool_kind == "task_sequence":
            # Task sequence execution - run labeled tasks with tool.eval flow control
            from noetl.worker.task_sequence_executor import TaskSequenceExecutor

            # Create task sequence executor with tool execution callback
            async def execute_task_tool(kind: str, config: dict, ctx: dict) -> Any:
                """Execute a single tool within the task sequence."""
                return await self._execute_tool(kind, config, {}, step, ctx)

            def render_template_str(template: str, ctx: dict) -> Any:
                """Render a single Jinja2 template string, parsing JSON results back to objects."""
                import json
                import ast
                try:
                    rendered = jinja_env.from_string(template).render(**ctx)
                    # If result looks like JSON (dict or list), parse it back to object
                    # This allows {{ outcome.result }} to return the actual dict, not a string
                    if isinstance(rendered, str):
                        stripped = rendered.strip()
                        if (stripped.startswith('{') and stripped.endswith('}')) or \
                           (stripped.startswith('[') and stripped.endswith(']')):
                            try:
                                return json.loads(stripped)
                            except json.JSONDecodeError:
                                # Fallback to ast.literal_eval for Python repr format (single quotes)
                                # This handles {{ outcome.result }} which renders as "{'key': 'value'}"
                                try:
                                    return ast.literal_eval(stripped)
                                except (ValueError, SyntaxError):
                                    pass
                    return rendered
                except Exception as e:
                    logger.warning(f"[TASK_SEQ] Template render error: {e}")
                    return template

            def render_dict_templates(data: dict, ctx: dict) -> dict:
                """Recursively render templates in a dict."""
                from noetl.core.dsl.render import render_template as recursive_render
                return recursive_render(jinja_env, data, ctx)

            executor = TaskSequenceExecutor(
                tool_executor=execute_task_tool,
                render_template=render_template_str,
                render_dict=render_dict_templates,
            )

            # Extract tasks and remaining actions from config
            tasks = task_config.get("tasks", [])
            remaining_actions = task_config.get("remaining_actions", [])

            # Execute the task sequence
            seq_result = await executor.execute(
                tasks=tasks,
                base_context=context,
            )

            # Include remaining actions in result for engine to process
            if remaining_actions and seq_result.get("status") != "failed":
                seq_result["remaining_actions"] = remaining_actions

            return seq_result

        elif tool_kind == "noop":
            # No-operation tool - used for case-driven steps that don't need tool execution
            # Returns empty result, step logic is driven by case conditions
            logger.debug(f"[NOOP] Step '{step}' - no-op execution")
            return {"status": "noop", "step": step}

        else:
            raise NotImplementedError(f"Tool kind '{tool_kind}' not implemented")
    
    async def _execute_python(self, config: dict, args: dict) -> Any:
        """Execute Python code."""
        import inspect
        
        # Priority: script > code_b64 > code
        code = config.get("code", "")
        
        # Handle script attribute (external code loading)
        if "script" in config:
            from noetl.worker.script_loader import load_script_content
            script_config = config["script"]
            code = await load_script_content(script_config)
        elif "code_b64" in config:
            import base64
            code = base64.b64decode(config["code_b64"]).decode("utf-8")
        
        if not code:
            raise ValueError("Python tool requires 'code', 'code_b64', or 'script' in config")
        
        # Execute code
        namespace = {"args": args}
        exec(code, namespace)
        
        # Call main() if it exists
        if "main" in namespace and callable(namespace["main"]):
            main_func = namespace["main"]
            sig = inspect.signature(main_func)
            
            # Determine how to call the function based on its signature
            if len(sig.parameters) == 0:
                # No parameters: call without args
                result = main_func()
            elif any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()):
                # Has **kwargs: unpack args as keyword arguments
                result = main_func(**args)
            elif all(isinstance(args, dict) and (p.name in args or p.default != inspect.Parameter.empty) 
                     for p in sig.parameters.values() if p.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)):
                # All parameters can be satisfied by args dict keys or have defaults: unpack kwargs
                result = main_func(**args)
            else:
                # Pass args dict as single parameter (backward compatibility)
                result = main_func(args)
            
            # Await if it's a coroutine
            if inspect.iscoroutine(result):
                result = await result
            
            return result
        
        return {"status": "ok"}
    
    async def _execute_http(self, config: dict, args: dict) -> Any:
        """Execute HTTP request."""
        method = config.get("method", "GET").upper()
        url = config.get("url") or config.get("endpoint")
        headers = config.get("headers", {})
        params = config.get("params", {})
        json_body = config.get("json")
        
        if not url:
            raise ValueError("HTTP tool requires 'url' or 'endpoint' in config")
        
        logger.debug(
            "HTTP %s request url=%s headers_keys=%s params_keys=%s has_json_body=%s",
            method,
            url,
            _safe_keys(headers),
            _safe_keys(params),
            json_body is not None,
        )
        
        # Make request
        response = await self._http_client.request(
            method=method,
            url=url,
            headers=headers,
            params=params,
            json=json_body
        )
        response.raise_for_status()
        
        return {
            "status_code": response.status_code,
            "body": response.json() if response.headers.get("content-type", "").startswith("application/json") else response.text
        }
    
    async def _execute_postgres(self, config: dict, args: dict) -> Any:
        """Execute Postgres query with auth resolution."""
        query = config.get("query") or config.get("sql") or config.get("command")
        auth_spec = config.get("auth")
        
        if not query:
            raise ValueError("Postgres tool requires 'query', 'sql', or 'command' in config")
        
        # If no auth specified, use default connection
        if not auth_spec:
            from noetl.core.common import get_pgdb_connection
            import psycopg
            
            conn_params = get_pgdb_connection()
            async with await psycopg.AsyncConnection.connect(**conn_params) as conn:
                async with conn.cursor() as cur:
                    await cur.execute(query, args if args else None)
                    
                    # Check if query returns results
                    if cur.description:
                        results = await cur.fetchall()
                        return [dict(row) for row in results]
                    else:
                        await conn.commit()
                        return {"status": "ok", "rowcount": cur.rowcount}
        
        # Resolve credentials via server API
        from noetl.worker.secrets import fetch_credential_by_key
        credential = fetch_credential_by_key(auth_spec)
        
        if not credential or not credential.get("data"):
            raise ValueError(f"Failed to resolve credential: {auth_spec}")
        
        # Extract connection parameters
        cred_data = credential.get("data", {})
        host = cred_data.get("db_host") or cred_data.get("host")
        port = cred_data.get("db_port") or cred_data.get("port") or 5432
        user = cred_data.get("db_user") or cred_data.get("user") or cred_data.get("username")
        password = cred_data.get("db_password") or cred_data.get("password")
        database = cred_data.get("db_name") or cred_data.get("database") or cred_data.get("dbname")
        
        if not all([host, user, password, database]):
            raise ValueError(f"Incomplete Postgres credentials for: {auth_spec}")
        
        # Create connection string
        import psycopg
        conn_string = f"host={host} port={port} user={user} password={password} dbname={database}"
        
        # Execute with credential-specific connection
        async with await psycopg.AsyncConnection.connect(conn_string) as conn:
            async with conn.cursor() as cur:
                await cur.execute(query, args if args else None)
                
                # Check if query returns results
                if cur.description:
                    results = await cur.fetchall()
                    return [dict(row) for row in results]
                else:
                    await conn.commit()
                    return {"status": "ok", "rowcount": cur.rowcount}
    
    async def _execute_duckdb(self, config: dict, args: dict) -> Any:
        """Execute DuckDB query."""
        import duckdb
        
        query = config.get("query") or config.get("sql")
        database = config.get("database", ":memory:")
        
        if not query:
            raise ValueError("DuckDB tool requires 'query' or 'sql' in config")
        
        # Connect to database (can be :memory: or file path)
        conn = duckdb.connect(database)
        
        try:
            # Execute query
            result = conn.execute(query, args if args else None)
            
            # Check if query returns results
            if result.description:
                # Fetch all rows and convert to list of dicts
                columns = [desc[0] for desc in result.description]
                rows = result.fetchall()
                return [dict(zip(columns, row)) for row in rows]
            else:
                # For DML operations (INSERT, UPDATE, DELETE)
                return {"status": "ok", "rowcount": len(result.fetchall())}
        finally:
            conn.close()
    
    async def _execute_workbook(self, config: dict, args: dict) -> Any:
        """Execute a named task from the workbook section."""
        task_name = config.get("name")
        if not task_name:
            raise ValueError("Workbook execution requires 'name' in config")
        
        # Get workbook task definition from catalog
        # In V2, workbook tasks are stored in playbook.workbook section
        # For now, return error - requires catalog access
        raise NotImplementedError("Workbook tool execution requires catalog integration")
    
    async def _execute_playbook(self, config: dict, args: dict) -> Any:
        """Execute a sub-playbook."""
        path = config.get("path")
        return_step = config.get("return_step")
        
        if not path:
            raise ValueError("Playbook execution requires 'path' in config")
        
        # Call server to start sub-playbook execution
        if not self._http_client:
            raise RuntimeError("HTTP client not initialized")
        
        # Get server URL from config or environment
        server_url = os.getenv("NOETL_SERVER_URL", "http://noetl.noetl.svc.cluster.local:8082")
        
        # Get current execution_id to pass as parent
        parent_execution_id = getattr(self, '_current_execution_id', None)
        
        payload = {"path": path, "payload": args}
        if parent_execution_id:
            payload["parent_execution_id"] = parent_execution_id

        transient_errors = (httpx.ReadError, httpx.RemoteProtocolError, httpx.ConnectError)
        retry_count = 3
        base_delay = 0.05

        async def _request_with_transient_retry(request_coro_factory, operation: str):
            for attempt in range(retry_count):
                try:
                    return await request_coro_factory()
                except transient_errors as e:
                    is_last_attempt = attempt == retry_count - 1
                    if is_last_attempt:
                        logger.error(
                            "Sub-playbook %s failed after %s transient retry attempts: %s",
                            operation,
                            retry_count,
                            e,
                        )
                        raise
                    delay = base_delay * (2 ** attempt)
                    logger.warning(
                        "Sub-playbook %s transient error (attempt %s/%s): %s. Retrying in %sms",
                        operation,
                        attempt + 1,
                        retry_count,
                        e,
                        delay * 1000,
                    )
                    await asyncio.sleep(delay)

        response = await _request_with_transient_retry(
            lambda: self._http_client.post(
                _api_url(server_url, "execute"),
                json=payload,
                timeout=30.0,
            ),
            "spawn request",
        )
        response.raise_for_status()
        result = response.json()
        
        execution_id = result.get("execution_id")
        
        # Poll for sub-playbook completion if return_step is specified
        if return_step:
            max_wait = config.get("timeout", 300)  # Default 5 minutes
            poll_interval = 2  # seconds
            elapsed = 0
            failed_statuses = {"FAILED", "ERROR", "CANCELLED", "CANCELED"}
            
            while elapsed < max_wait:
                await asyncio.sleep(poll_interval)
                elapsed += poll_interval
                
                # Check execution status using engine state endpoint.
                # /api/executions/{id} can show transient COMPLETED for intermediate
                # command.completed events, which is not a terminal playbook state.
                status_response = await _request_with_transient_retry(
                    lambda: self._http_client.get(
                        _api_url(server_url, f"executions/{execution_id}/status"),
                        timeout=10.0,
                    ),
                    f"status polling for execution {execution_id}",
                )
                
                if status_response.status_code == 200:
                    status_data = status_response.json()
                    state_completed = bool(status_data.get("completed"))
                    state_failed = bool(status_data.get("failed"))
                    
                    if state_completed or state_failed:
                        return status_data
                else:
                    # Backward-compatible fallback for older servers without /status.
                    fallback_response = await _request_with_transient_retry(
                        lambda: self._http_client.get(
                            _api_url(server_url, f"executions/{execution_id}"),
                            timeout=10.0,
                        ),
                        f"fallback status polling for execution {execution_id}",
                    )
                    if fallback_response.status_code == 200:
                        fallback_data = fallback_response.json()
                        state_completed = bool(fallback_data.get("completed"))
                        state_failed = bool(fallback_data.get("failed"))

                        # Only trust status string for explicit failure states in fallback mode.
                        status_value = str(fallback_data.get("status") or fallback_data.get("state") or "").upper()
                        if status_value in failed_statuses:
                            state_failed = True
                            state_completed = True

                        if state_completed or state_failed:
                            return fallback_data
            
            # Timeout - return what we have
            logger.warning(f"Sub-playbook {execution_id} timed out after {max_wait}s")
        
        # Async execution - return execution info immediately
        return {
            "status": "started",
            "execution_id": execution_id,
            "path": path,
            "async": return_step is None
        }
    
    async def _execute_secrets(self, config: dict, args: dict) -> Any:
        """Fetch secrets/credentials from secret manager."""
        secret_name = config.get("name")
        provider = config.get("provider", "env")  # env, aws, gcp, azure
        
        if not secret_name:
            raise ValueError("Secrets execution requires 'name' in config")
        
        if provider == "env":
            # Read from environment variable
            import os
            value = os.getenv(secret_name)
            if value is None:
                raise ValueError(f"Secret not found in environment: {secret_name}")
            return {"value": value}
        else:
            raise NotImplementedError(f"Secret provider '{provider}' not implemented")

    async def _execute_container(self, config: dict, args: dict) -> Any:
        """Execute code in a container (Kubernetes Job)."""
        raise NotImplementedError(
            "Container execution is not yet implemented in V2 worker. "
            "This feature requires Kubernetes Job creation and monitoring. "
            "Please use Python/HTTP tools or submit a feature request."
        )
    
    async def _execute_script(self, config: dict, args: dict) -> Any:
        """Execute external script (deprecated - use Python with script attribute)."""
        raise NotImplementedError(
            "Script tool kind is deprecated. Use 'python' tool kind with 'script' attribute instead. "
            "Example: tool: {kind: python, script: {uri: 'gs://bucket/script.py', source: {type: gcs, auth: credential}}}"
        )
    
    async def _emit_batch_events(
        self,
        server_url: str,
        execution_id: int,
        events: list[dict],
        timeout_seconds: Optional[float] = None,
        max_retries: Optional[int] = None,
        raise_on_failure: bool = True,
    ):
        """
        Emit multiple events in a single HTTP call via /api/events/batch.

        Falls back to individual _emit_event calls if the batch endpoint is
        unavailable (404) or if events list has only one item.
        """
        if not events:
            return True

        # Single event: just use the regular endpoint
        if len(events) == 1:
            evt = events[0]
            return await self._emit_event(
                server_url,
                execution_id,
                evt["step"],
                evt["name"],
                evt.get("payload", {}),
                actionable=evt.get("actionable", False),
                informative=evt.get("informative", True),
                timeout_seconds=timeout_seconds,
                max_retries=max_retries,
                raise_on_failure=raise_on_failure,
            )

        if not self._http_client:
            raise RuntimeError("HTTP client not initialized")

        batch_url = _api_url(server_url, "events/batch")
        prepared_events = []
        for evt in events:
            prepared_payload = await self._prepare_event_payload_for_transport(
                execution_id=execution_id,
                step=evt["step"],
                event_name=evt["name"],
                payload=evt.get("payload", {}),
            )
            prepared_evt = dict(evt)
            prepared_evt["payload"] = prepared_payload
            prepared_events.append(prepared_evt)
        batch_data = {
            "execution_id": str(execution_id),
            "worker_id": self.worker_id,
            "events": prepared_events,
        }

        import time
        from noetl.core.config import get_worker_settings
        worker_settings = get_worker_settings()

        retry_count = max_retries if max_retries is not None else 3
        request_timeout = (
            float(timeout_seconds)
            if timeout_seconds is not None
            else float(worker_settings.http_event_timeout)
        )
        base_delay = 0.05
        batch_idempotency_key = f"{self.worker_id}:{execution_id}:{uuid.uuid4().hex}"

        for attempt in range(retry_count):
            await self._concurrency.acquire()
            _released = False
            try:
                response = await self._http_client.post(
                    batch_url,
                    json=batch_data,
                    headers={"Idempotency-Key": batch_idempotency_key},
                    timeout=request_timeout,
                )
                if response.status_code == 404:
                    # Batch endpoint not available - fall back to individual calls
                    await self._concurrency.release_success()
                    _released = True
                    logger.debug("[BATCH] /api/events/batch not found, falling back to individual calls")
                    for evt in prepared_events:
                        await self._emit_event(
                            server_url,
                            execution_id,
                            evt["step"],
                            evt["name"],
                            evt.get("payload", {}),
                            actionable=evt.get("actionable", False),
                            informative=evt.get("informative", True),
                            timeout_seconds=timeout_seconds,
                            max_retries=1,
                            raise_on_failure=raise_on_failure,
                        )
                    return True

                if response.status_code == 503:
                    detail_code = "server_busy"
                    detail_message = "server returned 503"
                    try:
                        detail = response.json().get("detail")
                        if isinstance(detail, dict):
                            detail_code = str(detail.get("code") or detail_code)
                            detail_message = str(detail.get("message") or detail_message)
                    except Exception:
                        pass
                    retry_after = 1.0
                    try:
                        retry_after = max(0.0, float(response.headers.get("Retry-After", "1")))
                    except (TypeError, ValueError):
                        pass
                    await self._concurrency.release_overload(retry_after)
                    _released = True
                    if attempt == retry_count - 1:
                        logger.error(
                            "[BATCH] 503 after %s attempts for execution %s code=%s message=%s %s",
                            retry_count, execution_id, detail_code, detail_message, self._concurrency.get_status(),
                        )
                        if raise_on_failure:
                            raise RuntimeError(
                                f"Batch event emission failed: code={detail_code} message={detail_message}"
                            )
                        return False
                    logger.warning(
                        "[BATCH] 503 attempt %s/%s code=%s message=%s; backoff applied %s",
                        attempt + 1, retry_count, detail_code, detail_message, self._concurrency.get_status(),
                    )
                    continue  # next acquire() waits out the controller backoff

                response.raise_for_status()
                await self._concurrency.release_success()
                _released = True
                batch_request_id = None
                try:
                    body = response.json()
                    if isinstance(body, dict):
                        batch_request_id = body.get("request_id")
                except Exception:
                    batch_request_id = None
                logger.debug(
                    "[BATCH] Emitted %s events for execution %s in single call request_id=%s",
                    len(events),
                    execution_id,
                    batch_request_id,
                )
                return True

            except asyncio.CancelledError:
                if not _released:
                    await self._concurrency.release_error()
                raise
            except Exception as e:
                if not _released:
                    await self._concurrency.release_error()
                    _released = True
                is_last_attempt = attempt == retry_count - 1
                if is_last_attempt:
                    logger.error(
                        "[BATCH] Failed to emit batch events after %s attempts: %s",
                        retry_count,
                        e,
                    )
                    if raise_on_failure:
                        raise RuntimeError(f"Batch event emission failed: {e}") from e
                    return False
                delay = base_delay * (2 ** attempt)
                logger.warning(
                    "[BATCH] Batch event emission failed (attempt %s/%s): %s. Retrying in %sms",
                    attempt + 1,
                    retry_count,
                    e,
                    delay * 1000,
                )
                await asyncio.sleep(delay)

    async def _emit_event(
        self,
        server_url: str,
        execution_id: int,
        step: str,
        name: str,
        payload: dict,
        actionable: bool = False,
        informative: bool = True,
        correlation: dict = None,
        inputs: dict = None,
        timeout_seconds: Optional[float] = None,
        max_retries: Optional[int] = None,
        raise_on_failure: bool = True,
    ):
        """
        Emit an event to the server using the v2 API schema with retry logic.
        
        Implements ResultRef pattern for efficient result storage:
        - Small results (< inline_max_bytes) stored directly in output_inline
        - Large results stored as artifacts with output_ref pointer
        - Correlation keys (iteration, page, attempt) for tracking
        - Actionable/informative flags for control flow
        
        Args:
            server_url: Server API URL
            execution_id: Execution identifier
            step: Step name
            name: Event name (e.g., call.done, step.exit)
            payload: Event payload data
            actionable: If True, server should take action (evaluate case, route)
            informative: If True, event is for logging/observability
            correlation: Optional correlation keys dict (iteration, page, attempt)
            inputs: Optional rendered input snapshot
        """
        if not self._http_client:
            raise RuntimeError("HTTP client not initialized")
        
        event_url = _api_url(server_url, "events")
        prepared_payload = await self._prepare_event_payload_for_transport(
            execution_id=execution_id,
            step=step,
            event_name=name,
            payload=payload,
        )
        
        # Build event data - server persists reference-only result records.
        event_data = {
            "execution_id": str(execution_id),
            "step": step,
            "name": name,
            "payload": prepared_payload,
            "worker_id": self.worker_id,
            "actionable": actionable,
            "informative": informative,
        }
            
        # Add correlation keys if provided
        if correlation:
            event_data["correlation"] = correlation
        
        import time
        t_emit_start = time.perf_counter()

        from noetl.core.config import get_worker_settings
        worker_settings = get_worker_settings()

        retry_count = max_retries if max_retries is not None else 3
        request_timeout = (
            float(timeout_seconds)
            if timeout_seconds is not None
            else float(worker_settings.http_event_timeout)
        )
        base_delay = 0.05  # Fast retry: 50ms instead of 500ms

        for attempt in range(retry_count):
            await self._concurrency.acquire()
            _released = False
            try:
                t_http_start = time.perf_counter()
                logger.debug(
                    "[HTTP] POST %s event=%s step=%s execution=%s attempt=%s/%s",
                    event_url,
                    name,
                    step,
                    execution_id,
                    attempt + 1,
                    retry_count,
                )

                response = await self._http_client.post(
                    event_url,
                    json=event_data,
                    timeout=request_timeout,
                )

                if response.status_code == 503:
                    retry_after = 1.0
                    try:
                        retry_after = max(0.0, float(response.headers.get("Retry-After", "1")))
                    except (TypeError, ValueError):
                        pass
                    await self._concurrency.release_overload(retry_after)
                    _released = True
                    if attempt == retry_count - 1:
                        logger.error(
                            "[HTTP] Event %s: 503 pool saturated after %s attempts %s",
                            name, retry_count, self._concurrency.get_status(),
                        )
                        if raise_on_failure:
                            raise RuntimeError(
                                f"Event emission failed: server pool saturated after {retry_count} retries"
                            )
                        return False
                    logger.warning(
                        "[HTTP] Event %s: 503 attempt %s/%s; backoff applied %s",
                        name, attempt + 1, retry_count, self._concurrency.get_status(),
                    )
                    continue  # next acquire() waits out the controller backoff

                response.raise_for_status()
                await self._concurrency.release_success()
                _released = True

                t_http_end = time.perf_counter()
                logger.debug(f"[PERF] emit_event({name}) HTTP took {(t_http_end - t_http_start)*1000:.1f}ms - status={response.status_code}")
                return True  # Success - exit retry loop

            except asyncio.CancelledError:
                if not _released:
                    await self._concurrency.release_error()
                raise
            except Exception as e:
                if not _released:
                    await self._concurrency.release_error()
                    _released = True
                is_last_attempt = (attempt == retry_count - 1)

                if is_last_attempt:
                    t_emit_end = time.perf_counter()
                    logger.error(
                        f"[HTTP] Failed to emit event {name} after {retry_count} attempts "
                        f"({(t_emit_end - t_emit_start)*1000:.1f}ms total): {e}",
                        exc_info=True,
                    )
                    if raise_on_failure:
                        raise RuntimeError(f"Event emission failed after {retry_count} retries: {e}") from e
                    return False
                else:
                    delay = base_delay * (2 ** attempt)  # Fast exponential backoff: 50ms, 100ms, 200ms
                    logger.warning(
                        f"[HTTP] Event {name} emission failed "
                        f"(attempt {attempt + 1}/{retry_count}): {e}. Retrying in {delay*1000:.0f}ms..."
                    )
                    await asyncio.sleep(delay)


async def run_v2_worker(
    worker_id: str,
    nats_url: str = "nats://noetl:noetl@nats.nats.svc.cluster.local:4222",
    server_url: Optional[str] = None
):
    """Run a V2 worker instance."""
    # Set environment variable to indicate worker context (for TransientVars API routing)
    os.environ["NOETL_WORKER_MODE"] = "true"
    
    # No database pool initialization - worker uses server API for all noetl schema access
    # Only tool steps (postgres, duckdb) access external databases with their own credentials
    logger.info("V2 Worker uses server API for variables, events, and context (no direct DB access)")
    
    worker = V2Worker(
        worker_id=worker_id,
        nats_url=nats_url,
        server_url=server_url
    )
    
    try:
        await worker.start()  # Should run forever
    except KeyboardInterrupt:
        logger.info("Worker interrupted by user")
    except Exception as e:
        logger.error(f"Worker error: {e}", exc_info=True)
        raise
    finally:
        await worker.cleanup()
        logger.info("Worker cleanup complete")


def run_worker_v2_sync(
    nats_url: str = "nats://noetl:noetl@nats.nats.svc.cluster.local:4222",
    server_url: Optional[str] = None
):
    """
    Synchronous entry point for CLI.
    
    Generates worker ID and runs async worker in event loop.
    """
    import uuid
    import sys
    
    try:
        
        # Get from environment or use defaults
        nats_url = os.getenv("NATS_URL", nats_url)
        server_url = server_url or os.getenv("NOETL_SERVER_URL", "http://noetl.noetl.svc.cluster.local:8082")
        
        worker_id = f"worker-{uuid.uuid4().hex[:8]}"
        
        with open("/tmp/worker_config.txt", "w") as f:
            f.write(f"Worker ID: {worker_id}\n")
            f.write(f"NATS URL: {nats_url}\n")
            f.write(f"Server URL: {server_url}\n")
            f.flush()
        
        logger.info(f"Starting V2 worker {worker_id} | NATS={nats_url} | Server={server_url}")
        
        with open("/tmp/worker_before_run.txt", "w") as f:
            f.write(f"About to call asyncio.run at {datetime.now()}\n")
            f.flush()
        
        asyncio.run(run_v2_worker(worker_id, nats_url, server_url))
        
        with open("/tmp/worker_after_run.txt", "w") as f:
            f.write(f"asyncio.run returned at {datetime.now()}\n")
            f.flush()
        
    except KeyboardInterrupt:
        with open("/tmp/worker_interrupt.txt", "w") as f:
            f.write(f"Interrupted at {datetime.now()}\n")
            f.flush()
        logger.info("Worker interrupted by user")
        sys.exit(0)
    except Exception as e:
        with open("/tmp/worker_error.txt", "w") as f:
            f.write(f"Error at {datetime.now()}: {e}\n")
            import traceback
            f.write(traceback.format_exc())
            f.flush()
        
        logger.error(f"Worker failed to start: {e}", exc_info=True)
        sys.exit(1)
