"""
Action Executor - Lifecycle-aware execution of EMU actions.

Handles tool_call, route, decision_prompt, and context_directive actions
with support for shadow and canary execution modes.
"""
import asyncio
import logging
import time
from typing import Any, Optional, Union

from memrail.tools.models import (
    AckResult,
    CanaryConfig,
    Decision,
    ExecutionResult,
    ExecutionMetricsSample,
    ExecutorConfig,
    LifecycleMetrics,
    LifecycleState,
    MissingToolBehavior,
    RetryPolicy,
    ShadowComparisonResult,
    ShadowConfig,
    ShadowMode,
    ToolDefinition,
    make_sticky_decider,
)
from memrail.tools.registry import ToolRegistry
from memrail.tools.idempotency import (
    resolve_idempotency_key,
    IdempotencyKeyError,
    resolve_action_args,
    atoms_to_state_dict,
)
from memrail.tools.rate_limiter import (
    RateLimiter,
    RateLimiterBackend,
    RateLimitExceeded,
    RateLimitError,
)
from memrail.tools.schema_validation import validate_args, SchemaValidationError
from memrail.tools.circuit_breaker import (
    CircuitBreaker,
    CircuitBreakerConfig,
    CircuitBreakerRegistry,
    CircuitOpenError,
    CircuitState,
)


logger = logging.getLogger(__name__)


# =============================================================================
# Metrics Collector (FR3.9)
# =============================================================================

class MetricsCollector:
    """
    Collects and aggregates execution metrics for lifecycle comparison.

    Tracks active, shadow, and canary execution metrics to enable
    performance comparison and output drift detection.

    Example:
        ```python
        collector = MetricsCollector(max_samples=1000)

        # Metrics are auto-collected by ActionExecutor when enabled
        executor = ActionExecutor(registry)
        executor.enable_metrics(collector)

        # Get aggregated metrics
        metrics = collector.get_metrics("my-emu")
        print(f"Active success rate: {metrics.active_success_rate:.2%}")
        print(f"Canary success rate: {metrics.canary_success_rate:.2%}")
        ```
    """

    def __init__(
        self,
        max_samples_per_emu: int = 1000,
        max_latency_samples: int = 100,
    ):
        """
        Initialize the metrics collector.

        Args:
            max_samples_per_emu: Maximum samples to retain per EMU
            max_latency_samples: Maximum latency samples for percentile calculations
        """
        self._samples: dict[str, list[ExecutionMetricsSample]] = {}
        self._max_samples = max_samples_per_emu
        self._max_latency_samples = max_latency_samples
        self._comparison_results: dict[str, list[ShadowComparisonResult]] = {}

    def record(self, result: ExecutionResult) -> None:
        """
        Record an execution result.

        Args:
            result: The execution result to record
        """
        emu_key = result.decision.emu_key

        # Create output hash for comparison
        output_hash = None
        if result.output is not None:
            try:
                import hashlib
                import json
                output_str = json.dumps(result.output, sort_keys=True, default=str)
                output_hash = hashlib.md5(output_str.encode()).hexdigest()[:16]
            except Exception:
                pass

        sample = ExecutionMetricsSample(
            emu_key=emu_key,
            execution_mode=result.execution_mode,
            success=result.success,
            duration_ms=result.duration_ms,
            error=result.error,
            output_hash=output_hash,
        )

        if emu_key not in self._samples:
            self._samples[emu_key] = []

        self._samples[emu_key].append(sample)

        # Trim old samples
        if len(self._samples[emu_key]) > self._max_samples:
            self._samples[emu_key] = self._samples[emu_key][-self._max_samples:]

    def record_comparison(self, comparison: ShadowComparisonResult) -> None:
        """
        Record a shadow comparison result.

        Args:
            comparison: The comparison result to record
        """
        emu_key = comparison.emu_key
        if emu_key not in self._comparison_results:
            self._comparison_results[emu_key] = []

        self._comparison_results[emu_key].append(comparison)

        # Trim old comparisons
        if len(self._comparison_results[emu_key]) > self._max_samples:
            self._comparison_results[emu_key] = self._comparison_results[emu_key][-self._max_samples:]

    def get_metrics(
        self,
        emu_key: str,
        window_seconds: Optional[int] = None,
    ) -> LifecycleMetrics:
        """
        Get aggregated metrics for an EMU.

        Args:
            emu_key: The EMU key to get metrics for
            window_seconds: Optional time window (None = all samples)

        Returns:
            LifecycleMetrics with aggregated statistics
        """
        from datetime import timedelta

        now = datetime.utcnow()
        window_start = now - timedelta(seconds=window_seconds) if window_seconds else datetime.min

        samples = self._samples.get(emu_key, [])
        filtered_samples = [s for s in samples if s.timestamp >= window_start]

        metrics = LifecycleMetrics(
            emu_key=emu_key,
            window_start=window_start if window_seconds else (
                min(s.timestamp for s in filtered_samples) if filtered_samples else now
            ),
            window_end=now,
        )

        for sample in filtered_samples:
            if sample.execution_mode == "live":
                metrics.active_count += 1
                metrics.active_latency_sum_ms += sample.duration_ms
                if len(metrics.active_latency_samples) < self._max_latency_samples:
                    metrics.active_latency_samples.append(sample.duration_ms)
                if sample.success:
                    metrics.active_success_count += 1
                else:
                    metrics.active_error_count += 1

            elif sample.execution_mode == "canary":
                metrics.canary_count += 1
                metrics.canary_latency_sum_ms += sample.duration_ms
                if len(metrics.canary_latency_samples) < self._max_latency_samples:
                    metrics.canary_latency_samples.append(sample.duration_ms)
                if sample.success:
                    metrics.canary_success_count += 1
                else:
                    metrics.canary_error_count += 1

            elif sample.execution_mode == "shadow":
                metrics.shadow_count += 1
                metrics.shadow_latency_sum_ms += sample.duration_ms
                if len(metrics.shadow_latency_samples) < self._max_latency_samples:
                    metrics.shadow_latency_samples.append(sample.duration_ms)
                if sample.success:
                    metrics.shadow_success_count += 1
                else:
                    metrics.shadow_error_count += 1

        # Add comparison metrics
        comparisons = self._comparison_results.get(emu_key, [])
        for comp in comparisons:
            if comp.outputs_match:
                metrics.output_match_count += 1
            else:
                metrics.output_mismatch_count += 1

        return metrics

    def get_all_emu_keys(self) -> list[str]:
        """Get all EMU keys with recorded metrics."""
        return list(self._samples.keys())

    def clear(self, emu_key: Optional[str] = None) -> None:
        """
        Clear recorded metrics.

        Args:
            emu_key: Optional specific EMU to clear (None = clear all)
        """
        if emu_key:
            self._samples.pop(emu_key, None)
            self._comparison_results.pop(emu_key, None)
        else:
            self._samples.clear()
            self._comparison_results.clear()

    def export_all(self) -> dict[str, dict]:
        """Export all metrics as a dictionary."""
        return {
            emu_key: self.get_metrics(emu_key).to_dict()
            for emu_key in self._samples.keys()
        }


# Need datetime for MetricsCollector
from datetime import datetime


class ActionExecutor:
    """
    Lifecycle-aware executor for EMU actions.

    Handles:
    - Tool execution via registered handlers
    - Route action delegation
    - Decision prompt handling
    - Context directive pass-through
    - Shadow execution modes (skip, dry_run, execute_and_compare)
    - Canary execution with percentage-based routing

    Example:
        ```python
        executor = ActionExecutor(registry)
        executor.configure_shadow(ShadowMode.EXECUTE_AND_COMPARE, handler=compare_fn)
        executor.configure_canary(decider=sticky_session_fn)

        results = await executor.execute_decisions(decisions, executor_context={"user_id": "123"})
        ```
    """

    def __init__(
        self,
        registry: ToolRegistry,
        project: Optional[str] = None,
        config: Optional[ExecutorConfig] = None,
        rate_limiter: Optional[RateLimiter] = None,
    ):
        """
        Initialize the action executor.

        Args:
            registry: ToolRegistry with registered tools
            project: Project ID to scope tool access (required for project-scoped tools)
            config: Executor configuration (optional)
            rate_limiter: Rate limiter instance (optional, creates default if not provided)
        """
        self.registry = registry
        self.project = project
        self.config = config or ExecutorConfig()
        self._shadow_config = ShadowConfig()
        self._canary_config = CanaryConfig()
        self._rate_limiter = rate_limiter or RateLimiter()
        self._rate_limiting_enabled = True
        self._circuit_breakers = CircuitBreakerRegistry(enabled=False)  # Disabled by default
        self._metrics_collector: Optional[MetricsCollector] = None
        self._metrics_enabled = False
        # Cache of active results for shadow comparison (keyed by emu_key)
        self._active_results_cache: dict[str, ExecutionResult] = {}

        # Build project-scoped tool cache if project is specified
        self._project_tools: Optional[dict[str, ToolDefinition]] = None
        if project:
            self._project_tools = registry.get_tools_for_project(project)

    # =========================================================================
    # Configuration
    # =========================================================================

    def configure_shadow(
        self,
        mode: ShadowMode,
        handler: Optional[callable] = None,
    ) -> None:
        """
        Configure shadow execution behavior.

        Args:
            mode: Shadow execution mode (SKIP, DRY_RUN, EXECUTE_AND_COMPARE)
            handler: Optional comparison handler for EXECUTE_AND_COMPARE mode
                     Signature: async (decision, active_result, shadow_result) -> None
        """
        self._shadow_config = ShadowConfig(mode=mode, comparison_handler=handler)

    def configure_canary(
        self,
        decider: Optional[callable] = None,
        metrics_handler: Optional[callable] = None,
    ) -> None:
        """
        Configure canary execution behavior.

        Args:
            decider: Function to decide if canary should execute
                     Signature: (decision, context) -> bool
            metrics_handler: Optional handler for canary metrics
                     Signature: async (decision, result, is_canary) -> None
        """
        self._canary_config = CanaryConfig(
            decider=decider,
            metrics_handler=metrics_handler,
        )

    def configure(
        self,
        missing_tool_behavior: Optional[MissingToolBehavior] = None,
        execution_timeout_ms: Optional[int] = None,
        max_retries: Optional[int] = None,
        retry_backoff_ms: Optional[int] = None,
        rate_limiting_enabled: Optional[bool] = None,
    ) -> None:
        """
        Configure executor behavior.

        Args:
            missing_tool_behavior: Behavior when tool not found
            execution_timeout_ms: Default timeout per tool
            max_retries: Maximum retries for failed executions
            retry_backoff_ms: Backoff between retries
            rate_limiting_enabled: Enable/disable rate limiting (default: True)
        """
        if missing_tool_behavior is not None:
            self.config.missing_tool_behavior = missing_tool_behavior
        if execution_timeout_ms is not None:
            self.config.execution_timeout_ms = execution_timeout_ms
        if max_retries is not None:
            self.config.max_retries = max_retries
        if retry_backoff_ms is not None:
            self.config.retry_backoff_ms = retry_backoff_ms
        if rate_limiting_enabled is not None:
            self._rate_limiting_enabled = rate_limiting_enabled

    def configure_rate_limiter(
        self,
        backend: Optional[RateLimiterBackend] = None,
        enabled: bool = True,
    ) -> None:
        """
        Configure the rate limiter.

        Args:
            backend: Custom rate limiter backend (e.g., Redis-backed)
            enabled: Enable/disable rate limiting
        """
        if backend is not None:
            self._rate_limiter = RateLimiter(backend)
        self._rate_limiting_enabled = enabled

    @property
    def rate_limiter(self) -> RateLimiter:
        """Get the rate limiter instance."""
        return self._rate_limiter

    def configure_circuit_breaker(
        self,
        enabled: bool = True,
        default_config: Optional[CircuitBreakerConfig] = None,
    ) -> None:
        """
        Configure circuit breaker behavior.

        Args:
            enabled: Enable/disable circuit breakers
            default_config: Default configuration for all tools

        Example:
            ```python
            executor.configure_circuit_breaker(
                enabled=True,
                default_config=CircuitBreakerConfig(
                    failure_threshold=5,
                    recovery_timeout_ms=30000,
                ),
            )
            ```
        """
        self._circuit_breakers.enabled = enabled
        if default_config is not None:
            self._circuit_breakers = CircuitBreakerRegistry(
                default_config=default_config,
                enabled=enabled,
            )

    def configure_tool_circuit_breaker(
        self,
        tool_name: str,
        config: CircuitBreakerConfig,
    ) -> None:
        """
        Configure circuit breaker for a specific tool.

        Args:
            tool_name: Tool identifier
            config: Circuit breaker configuration for this tool

        Example:
            ```python
            executor.configure_tool_circuit_breaker(
                "critical_api",
                CircuitBreakerConfig(
                    failure_threshold=10,
                    recovery_timeout_ms=60000,
                ),
            )
            ```
        """
        self._circuit_breakers.configure(tool_name, config)

    @property
    def circuit_breakers(self) -> CircuitBreakerRegistry:
        """Get the circuit breaker registry."""
        return self._circuit_breakers

    # =========================================================================
    # Metrics Configuration (FR3.9)
    # =========================================================================

    def enable_metrics(
        self,
        collector: Optional[MetricsCollector] = None,
    ) -> MetricsCollector:
        """
        Enable metrics collection.

        Args:
            collector: Optional custom MetricsCollector (creates default if not provided)

        Returns:
            The MetricsCollector being used

        Example:
            ```python
            collector = executor.enable_metrics()

            # Execute some decisions...
            await executor.execute_decisions(decisions)

            # Get metrics
            metrics = collector.get_metrics("my-emu")
            print(f"Success rate: {metrics.active_success_rate:.2%}")
            ```
        """
        self._metrics_collector = collector or MetricsCollector()
        self._metrics_enabled = True
        return self._metrics_collector

    def disable_metrics(self) -> None:
        """Disable metrics collection."""
        self._metrics_enabled = False

    @property
    def metrics(self) -> Optional[MetricsCollector]:
        """Get the metrics collector (None if disabled)."""
        return self._metrics_collector if self._metrics_enabled else None

    def get_metrics(self, emu_key: str, window_seconds: Optional[int] = None) -> Optional[LifecycleMetrics]:
        """
        Get metrics for an EMU.

        Args:
            emu_key: The EMU key to get metrics for
            window_seconds: Optional time window in seconds

        Returns:
            LifecycleMetrics or None if metrics not enabled
        """
        if not self._metrics_enabled or not self._metrics_collector:
            return None
        return self._metrics_collector.get_metrics(emu_key, window_seconds)

    # =========================================================================
    # Execution Entry Points
    # =========================================================================

    async def execute_decisions(
        self,
        decisions: list[Decision],
        executor_context: Optional[dict] = None,
        atoms: Optional[list] = None,
        decision_point: Optional[str] = None,
    ) -> list[ExecutionResult]:
        """
        Execute a list of decisions with lifecycle awareness.

        When both active and shadow versions of the same EMU are present,
        the shadow comparison handler will receive both results for comparison.

        Args:
            decisions: List of Decision objects to execute
            executor_context: Additional context for canary decisions
            atoms: Optional list of context atoms for template resolution in action args.
                   When provided, {{state.key}} templates in tool args are resolved
                   using the atom values.

        Returns:
            List of ExecutionResult objects (one per decision)
        """
        executor_context = executor_context or {}

        # Build state dict from atoms for template resolution
        state_dict = atoms_to_state_dict(atoms) if atoms else {}
        results = []

        # Group decisions by emu_key to identify active/shadow pairs
        decisions_by_emu: dict[str, list[Decision]] = {}
        for decision in decisions:
            key = decision.emu_key
            if key not in decisions_by_emu:
                decisions_by_emu[key] = []
            decisions_by_emu[key].append(decision)

        # Execute active decisions first, then shadow (for proper comparison)
        active_decisions = [d for d in decisions if d.lifecycle_state == LifecycleState.ACTIVE]
        shadow_decisions = [d for d in decisions if d.lifecycle_state == LifecycleState.SHADOW]
        other_decisions = [d for d in decisions if d.lifecycle_state not in (LifecycleState.ACTIVE, LifecycleState.SHADOW)]

        # Clear the active results cache for this batch
        self._active_results_cache.clear()

        # Execute active decisions first and cache results
        for decision in active_decisions:
            result = await self._execute_decision(decision, executor_context, state_dict, decision_point=decision_point)
            results.append(result)
            # Cache for shadow comparison
            self._active_results_cache[decision.emu_key] = result

        # Execute shadow decisions with access to active results
        for decision in shadow_decisions:
            result = await self._execute_decision(decision, executor_context, state_dict, decision_point=decision_point)
            results.append(result)

        # Execute other decisions (canary, draft, etc.)
        for decision in other_decisions:
            result = await self._execute_decision(decision, executor_context, state_dict, decision_point=decision_point)
            results.append(result)

        return results

    async def execute_decision(
        self,
        decision: Decision,
        executor_context: Optional[dict] = None,
        atoms: Optional[list] = None,
        decision_point: Optional[str] = None,
    ) -> ExecutionResult:
        """
        Execute a single decision.

        Args:
            decision: Decision to execute
            executor_context: Additional context for canary decisions
            atoms: Optional list of context atoms for template resolution in action args
            decision_point: Named decision point for context_directive routing

        Returns:
            ExecutionResult
        """
        state_dict = atoms_to_state_dict(atoms) if atoms else {}
        return await self._execute_decision(decision, executor_context or {}, state_dict, decision_point=decision_point)

    # =========================================================================
    # Lifecycle Routing
    # =========================================================================

    async def _execute_decision(
        self,
        decision: Decision,
        executor_context: dict,
        state_dict: Optional[dict] = None,
        decision_point: Optional[str] = None,
    ) -> ExecutionResult:
        """Route decision based on lifecycle state."""
        state_dict = state_dict or {}
        state = decision.lifecycle_state

        if state == LifecycleState.DRAFT:
            # Draft EMUs should never be selected, but skip if somehow present
            return ExecutionResult(
                decision=decision,
                success=True,
                was_executed=False,
                execution_mode="skipped",
            )

        elif state == LifecycleState.SHADOW:
            return await self._execute_shadow(decision, executor_context, state_dict, decision_point=decision_point)

        elif state == LifecycleState.CANARY:
            return await self._execute_canary(decision, executor_context, state_dict, decision_point=decision_point)

        elif state == LifecycleState.ACTIVE:
            return await self._execute_live(decision, executor_context, execution_mode="live", state_dict=state_dict, decision_point=decision_point)

        elif state == LifecycleState.ARCHIVED:
            # Archived EMUs should never be selected
            return ExecutionResult(
                decision=decision,
                success=True,
                was_executed=False,
                execution_mode="skipped",
            )

        else:
            # Unknown state - treat as active for safety
            logger.warning(f"Unknown lifecycle state: {state}, treating as active")
            return await self._execute_live(decision, executor_context, execution_mode="live", state_dict=state_dict, decision_point=decision_point)

    async def _execute_shadow(
        self,
        decision: Decision,
        executor_context: dict,
        state_dict: Optional[dict] = None,
        decision_point: Optional[str] = None,
    ) -> ExecutionResult:
        """Execute a shadow decision based on shadow mode."""
        state_dict = state_dict or {}
        mode = self._shadow_config.mode

        if mode == ShadowMode.SKIP:
            return ExecutionResult(
                decision=decision,
                success=True,
                was_executed=False,
                execution_mode="skipped",
            )

        elif mode == ShadowMode.DRY_RUN:
            # Validate but don't execute
            action_type = decision.action.get("type", "unknown")
            if action_type == "tool_call":
                tool_name = decision.action.get("name") or decision.action.get("tool")

                # Check if tool exists
                if not tool_name:
                    return ExecutionResult(
                        decision=decision,
                        success=False,
                        error="tool_call action missing tool name",
                        was_executed=False,
                        execution_mode="dry_run",
                    )

                # Use project-scoped lookup if project is set
                tool: Optional[ToolDefinition] = None
                if self._project_tools is not None:
                    tool = self._project_tools.get(tool_name)
                    if tool is None:
                        # Check if tool exists but isn't scoped to this project
                        global_tool = self.registry.get_tool(tool_name)
                        if global_tool is not None:
                            return ExecutionResult(
                                decision=decision,
                                success=False,
                                error=f"Tool '{tool_name}' is not available for project '{self.project}'",
                                was_executed=False,
                                execution_mode="dry_run",
                            )
                else:
                    tool = self.registry.get_tool(tool_name)

                if tool is None:
                    return ExecutionResult(
                        decision=decision,
                        success=False,
                        error=f"Tool not found: {tool_name}",
                        was_executed=False,
                        execution_mode="dry_run",
                    )

                # Validate args against schema
                args = decision.action.get("args", {})
                validation_errors = validate_args(args, tool.input_schema)
                if validation_errors:
                    return ExecutionResult(
                        decision=decision,
                        success=False,
                        error=f"Schema validation failed: {'; '.join(validation_errors)}",
                        was_executed=False,
                        execution_mode="dry_run",
                    )

            return ExecutionResult(
                decision=decision,
                success=True,
                was_executed=False,
                execution_mode="dry_run",
            )

        elif mode == ShadowMode.EXECUTE_AND_COMPARE:
            # Execute shadow
            shadow_result = await self._execute_live(decision, executor_context, execution_mode="shadow", state_dict=state_dict, decision_point=decision_point)

            # Get corresponding active result from cache (if available)
            active_result = self._active_results_cache.get(decision.emu_key)

            # Create comparison result for metrics
            if self._metrics_enabled and self._metrics_collector:
                comparison = ShadowComparisonResult.compare(
                    emu_key=decision.emu_key,
                    active_result=active_result,
                    shadow_result=shadow_result,
                )
                self._metrics_collector.record_comparison(comparison)

            # Call comparison handler if provided (now with actual active_result!)
            if self._shadow_config.comparison_handler:
                try:
                    await self._shadow_config.comparison_handler(
                        decision,
                        active_result,  # Now properly populated from cache
                        shadow_result,
                    )
                except Exception as e:
                    logger.error(f"Shadow comparison handler error: {e}")

            return shadow_result

        else:
            # Unknown mode - skip
            return ExecutionResult(
                decision=decision,
                success=True,
                was_executed=False,
                execution_mode="skipped",
            )

    async def _execute_canary(
        self,
        decision: Decision,
        executor_context: dict,
        state_dict: Optional[dict] = None,
        decision_point: Optional[str] = None,
    ) -> ExecutionResult:
        """Execute a canary decision based on canary configuration."""
        state_dict = state_dict or {}
        should_execute = self._canary_config.should_execute_canary(decision, executor_context)

        if not should_execute:
            return ExecutionResult(
                decision=decision,
                success=True,
                was_executed=False,
                execution_mode="skipped",
            )

        result = await self._execute_live(decision, executor_context, execution_mode="canary", state_dict=state_dict, decision_point=decision_point)

        # Record metrics if enabled
        if self._metrics_enabled and self._metrics_collector:
            self._metrics_collector.record(result)

        # Call metrics handler if provided
        if self._canary_config.metrics_handler:
            try:
                await self._canary_config.metrics_handler(decision, result, True)
            except Exception as e:
                logger.error(f"Canary metrics handler error: {e}")

        return result

    # =========================================================================
    # Action Execution
    # =========================================================================

    async def _execute_live(
        self,
        decision: Decision,
        executor_context: dict,
        execution_mode: str,
        state_dict: Optional[dict] = None,
        decision_point: Optional[str] = None,
    ) -> ExecutionResult:
        """Execute a decision in live mode."""
        state_dict = state_dict or {}
        action = decision.action
        action_type = action.get("type", "unknown")

        start_time = time.perf_counter()

        try:
            if action_type == "tool_call":
                result = await self._execute_tool_call(decision, action, executor_context, state_dict)
            elif action_type == "context_directive":
                result = await self._execute_context_directive(decision, action, decision_point)
            elif action_type == "route":
                result = await self._execute_route(decision, action)
            elif action_type == "decision_prompt":
                result = await self._execute_decision_prompt(decision, action)
            else:
                result = ExecutionResult(
                    decision=decision,
                    success=False,
                    error=f"Unknown action type: {action_type}",
                    was_executed=False,
                )

            # Update execution mode
            result.execution_mode = execution_mode
            result.duration_ms = int((time.perf_counter() - start_time) * 1000)

            # Record metrics if enabled (for live and shadow modes)
            if self._metrics_enabled and self._metrics_collector and result.was_executed:
                self._metrics_collector.record(result)

            return result

        except asyncio.TimeoutError:
            duration_ms = int((time.perf_counter() - start_time) * 1000)
            result = ExecutionResult(
                decision=decision,
                success=False,
                error="Timeout",
                duration_ms=duration_ms,
                was_executed=True,
                execution_mode=execution_mode,
            )
            # Record timeout as failure in metrics
            if self._metrics_enabled and self._metrics_collector:
                self._metrics_collector.record(result)
            return result
        except Exception as e:
            duration_ms = int((time.perf_counter() - start_time) * 1000)
            logger.exception(f"Action execution error for {decision.emu_key}")
            result = ExecutionResult(
                decision=decision,
                success=False,
                error=str(e),
                duration_ms=duration_ms,
                was_executed=True,
                execution_mode=execution_mode,
            )
            # Record exception as failure in metrics
            if self._metrics_enabled and self._metrics_collector:
                self._metrics_collector.record(result)
            return result

    async def _execute_tool_call(
        self,
        decision: Decision,
        action: dict,
        executor_context: Optional[dict] = None,
        state_dict: Optional[dict] = None,
    ) -> ExecutionResult:
        """Execute a tool_call action."""
        executor_context = executor_context or {}
        state_dict = state_dict or {}

        # Get tool name and args - supports both flat SDK format and nested API format
        # Flat SDK format: {"type": "tool_call", "tool": "tool_name", "name": "tool_name", "args": {...}}
        # Nested API format: {"type": "tool_call", "intent": "...", "tool": {"tool_id": "tool_name", "args": {...}}}
        tool_spec = action.get("tool")

        # Extract EMU ToolSpec overrides (timeout, retry, side_effect)
        emu_timeout_ms: Optional[int] = None
        emu_retry_policy: Optional[str] = None
        emu_max_retries: Optional[int] = None
        emu_side_effect: Optional[bool] = None

        if isinstance(tool_spec, dict):
            # Nested API format (wire format from invoke response)
            tool_name = tool_spec.get("tool_id") or tool_spec.get("name")
            nested_args = tool_spec.get("args", {})
            emu_timeout_ms = tool_spec.get("timeout_ms")
            emu_side_effect = tool_spec.get("side_effect")
            retry_config = tool_spec.get("retry")
            if isinstance(retry_config, dict):
                emu_retry_policy = retry_config.get("policy")
                emu_max_retries = retry_config.get("max_retries")
        elif hasattr(tool_spec, "tool_id"):
            # Pydantic object edge case (e.g., ToolSpec model passed directly)
            tool_name = getattr(tool_spec, "tool_id", None) or getattr(tool_spec, "name", None)
            nested_args = getattr(tool_spec, "args", {})
            emu_timeout_ms = getattr(tool_spec, "timeout_ms", None)
            emu_side_effect = getattr(tool_spec, "side_effect", None)
            retry_obj = getattr(tool_spec, "retry", None)
            if retry_obj is not None:
                if hasattr(retry_obj, "policy"):
                    emu_retry_policy = retry_obj.policy if isinstance(retry_obj.policy, str) else retry_obj.policy.value
                if hasattr(retry_obj, "max_retries"):
                    emu_max_retries = retry_obj.max_retries
        else:
            # Flat SDK format - tool is a string
            tool_name = action.get("name") or tool_spec
            nested_args = None

        if not tool_name:
            return ExecutionResult(
                decision=decision,
                success=False,
                error="tool_call action missing tool name",
                was_executed=False,
            )

        # Get tool from project-scoped cache or registry
        tool: Optional[ToolDefinition] = None
        if self._project_tools is not None:
            # Project-scoped mode: only use tools registered to this project
            tool = self._project_tools.get(tool_name)
            if tool is None:
                # Check if tool exists but isn't scoped to this project
                global_tool = self.registry.get_tool(tool_name)
                if global_tool is not None:
                    return ExecutionResult(
                        decision=decision,
                        success=False,
                        error=f"Tool '{tool_name}' is not available for project '{self.project}'",
                        was_executed=False,
                    )
        else:
            # No project scope: use all tools (backward compatibility)
            tool = self.registry.get_tool(tool_name)

        if tool is None:
            # Handle missing tool based on config
            behavior = self.config.missing_tool_behavior

            if behavior == MissingToolBehavior.ERROR:
                return ExecutionResult(
                    decision=decision,
                    success=False,
                    error=f"Tool not found: {tool_name}",
                    was_executed=False,
                )
            elif behavior == MissingToolBehavior.SKIP:
                return ExecutionResult(
                    decision=decision,
                    success=True,
                    was_executed=False,
                )
            elif behavior == MissingToolBehavior.WARN:
                logger.warning(f"Tool not found: {tool_name}, skipping")
                return ExecutionResult(
                    decision=decision,
                    success=True,
                    was_executed=False,
                )

        if not tool.has_handler:
            return ExecutionResult(
                decision=decision,
                success=False,
                error=f"Tool '{tool_name}' has no handler bound",
                was_executed=False,
            )

        # Check rate limit if enabled and configured
        if self._rate_limiting_enabled and tool.rate_limit:
            try:
                # Use context user_id as key suffix for per-user rate limiting if available
                key_suffix = executor_context.get("user_id") if executor_context else None
                rate_result = self._rate_limiter.check(
                    tool_name=tool_name,
                    rate_limit=tool.rate_limit,
                    key_suffix=key_suffix,
                )
                if not rate_result.allowed:
                    return ExecutionResult(
                        decision=decision,
                        success=False,
                        error=f"Rate limit exceeded: {tool.rate_limit} (retry after {rate_result.reset_after:.1f}s)",
                        was_executed=False,
                    )
            except RateLimitError as e:
                logger.warning(f"Rate limit check failed for {tool_name}: {e}")
                # Continue execution if rate limit parsing fails

        # Check circuit breaker if enabled
        circuit_breaker: Optional[CircuitBreaker] = None
        if self._circuit_breakers.enabled:
            circuit_breaker = self._circuit_breakers.get(tool_name)
            if not circuit_breaker.allow_request():
                circuit_breaker.record_rejection()
                retry_after = circuit_breaker.time_until_retry()
                return ExecutionResult(
                    decision=decision,
                    success=False,
                    error=f"Circuit open for '{tool_name}': too many failures (retry in {retry_after:.1f}s)",
                    was_executed=False,
                )

        # Get args - use nested args from API format if available, else top-level args
        args = nested_args if nested_args is not None else action.get("args", {})

        # Resolve {{state.xxx}} templates in args using context atoms
        if state_dict and args:
            try:
                args = resolve_action_args(args, state_dict, strict=False)
                logger.debug(f"Resolved action args for {tool_name}: {args}")
            except Exception as e:
                logger.warning(f"Failed to resolve action args for {tool_name}: {e}")
                # Continue with original args if resolution fails

        # Resolve idempotency key if template is defined
        resolved_idempotency_key: Optional[str] = None
        if tool.idempotency_key:
            try:
                resolved_idempotency_key = resolve_idempotency_key(
                    template=tool.idempotency_key,
                    args=args,
                    context=executor_context,
                    strict=False,  # Don't fail on missing fields
                )
                logger.debug(
                    f"Resolved idempotency key for {tool_name}: {resolved_idempotency_key}"
                )
            except IdempotencyKeyError as e:
                logger.warning(f"Failed to resolve idempotency key for {tool_name}: {e}")

        # Execute with timeout and retry
        # 3-level fallback: EMU action ToolSpec > ToolDefinition (per-tool) > ExecutorConfig (global)
        timeout_ms = emu_timeout_ms or tool.timeout_ms or self.config.execution_timeout_ms
        timeout_s = timeout_ms / 1000

        retry_policy = emu_retry_policy or RetryPolicy.EXPONENTIAL.value
        max_retries = emu_max_retries if emu_max_retries is not None else self.config.max_retries
        retry_backoff_ms = self.config.retry_backoff_ms  # global only

        # RetryPolicy.NONE forces zero retries regardless of max_retries
        if retry_policy == RetryPolicy.NONE.value:
            max_retries = 0

        last_error: Optional[str] = None
        attempt = 0

        while attempt <= max_retries:
            try:
                output = await asyncio.wait_for(
                    tool.handler(args),
                    timeout=timeout_s,
                )
                # Record success for circuit breaker
                if circuit_breaker is not None:
                    circuit_breaker.record_success()
                return ExecutionResult(
                    decision=decision,
                    success=True,
                    output=output,
                    was_executed=True,
                    resolved_idempotency_key=resolved_idempotency_key,
                    side_effect=emu_side_effect,
                )
            except asyncio.TimeoutError:
                # Timeouts count as failures for circuit breaker
                if circuit_breaker is not None:
                    circuit_breaker.record_failure()
                return ExecutionResult(
                    decision=decision,
                    success=False,
                    error=f"Timeout after {timeout_ms}ms",
                    was_executed=True,
                    resolved_idempotency_key=resolved_idempotency_key,
                    side_effect=emu_side_effect,
                )
            except Exception as e:
                last_error = str(e)
                attempt += 1

                if attempt <= max_retries:
                    # Calculate backoff based on retry policy
                    if retry_policy == RetryPolicy.FIXED.value:
                        backoff_ms = retry_backoff_ms
                    else:
                        # Exponential (default)
                        backoff_ms = retry_backoff_ms * (2 ** (attempt - 1))
                    logger.warning(
                        f"Tool '{tool_name}' failed (attempt {attempt}/{max_retries + 1}): {e}. "
                        f"Retrying in {backoff_ms}ms..."
                    )
                    await asyncio.sleep(backoff_ms / 1000)
                else:
                    # All retries exhausted
                    logger.error(
                        f"Tool '{tool_name}' failed after {max_retries + 1} attempts: {e}"
                    )
                    # Record failure for circuit breaker
                    if circuit_breaker is not None:
                        circuit_breaker.record_failure()

        # If we get here, all retries failed
        return ExecutionResult(
            decision=decision,
            success=False,
            error=f"{last_error} (after {max_retries + 1} attempts)" if max_retries > 0 else last_error,
            was_executed=True,
            resolved_idempotency_key=resolved_idempotency_key,
            side_effect=emu_side_effect,
        )

    async def _execute_context_directive(
        self,
        decision: Decision,
        action: dict,
        decision_point: Optional[str] = None,
    ) -> ExecutionResult:
        """Execute a context_directive action by dispatching to registered handler."""
        handlers = self.registry.get_action_handlers()._context_directive_handlers

        handler = None
        if decision_point:
            # Try exact "dp:project" → exact "dp" → wildcard "*"
            project = action.get("project")
            if project:
                handler = handlers.get(f"{decision_point}:{project}")
            if not handler:
                handler = handlers.get(decision_point)
            if not handler:
                handler = handlers.get("*")
        else:
            # No decision_point: try wildcard, then fall through to sole handler
            handler = handlers.get("*")
            if not handler and len(handlers) == 1:
                handler = next(iter(handlers.values()))
            elif not handler and len(handlers) > 1:
                keys = ", ".join(k for k in handlers if k != "*")
                return ExecutionResult(
                    decision=decision,
                    success=False,
                    error=f"Ambiguous context_directive: no decision_point provided and multiple handlers registered ({keys}). Provide a decision_point to disambiguate.",
                    was_executed=False,
                )

        if handler is None:
            dp_label = decision_point or "unscoped"
            return ExecutionResult(
                decision=decision,
                success=False,
                error=f"No context_directive handler for decision_point '{dp_label}'",
                was_executed=False,
            )

        directive = action.get("directive")
        metadata = action.get("metadata", {})
        try:
            output = await handler(directive, metadata)
            return ExecutionResult(
                decision=decision,
                success=True,
                output=output,
                was_executed=True,
            )
        except Exception as e:
            return ExecutionResult(
                decision=decision,
                success=False,
                error=str(e),
                was_executed=False,
            )

    async def _execute_route(
        self,
        decision: Decision,
        action: dict,
    ) -> ExecutionResult:
        """Execute a route action."""
        handler = self.registry.get_action_handlers()._route_handler

        if handler is None:
            return ExecutionResult(
                decision=decision,
                success=False,
                error="No route handler registered",
                was_executed=False,
            )

        destination = action.get("route", "")
        metadata = action.get("metadata", {})

        try:
            output = await handler(destination, metadata)
            return ExecutionResult(
                decision=decision,
                success=True,
                output=output,
                was_executed=True,
            )
        except Exception as e:
            return ExecutionResult(
                decision=decision,
                success=False,
                error=str(e),
                was_executed=True,
            )

    async def _execute_decision_prompt(
        self,
        decision: Decision,
        action: dict,
    ) -> ExecutionResult:
        """Execute a decision_prompt action."""
        handler = self.registry.get_action_handlers()._decision_prompt_handler

        if handler is None:
            return ExecutionResult(
                decision=decision,
                success=False,
                error="No decision prompt handler registered",
                was_executed=False,
            )

        prompt = action.get("prompt", "")
        params = action.get("params", {})

        try:
            output = await handler(prompt, params)
            return ExecutionResult(
                decision=decision,
                success=True,
                output=output,
                was_executed=True,
            )
        except Exception as e:
            return ExecutionResult(
                decision=decision,
                success=False,
                error=str(e),
                was_executed=True,
            )
