"""
Workflows System for PraisonAI Agents.

Provides reusable multi-step workflows similar to Windsurf's Workflows,
allowing users to define and execute complex task sequences.

Features:
- YAML/Markdown workflow definitions
- Auto-discovery from .praisonai/workflows/
- Step-by-step execution with context passing
- Variable substitution
- Conditional steps
- Lazy loading for performance

Storage Structure:
    .praisonai/workflows/
    ├── deploy.md           # Deployment workflow
    ├── test.md             # Testing workflow
    └── review.md           # Code review workflow
"""

import os
import re
import json
import logging
from praisonaiagents._logging import get_logger
from pathlib import Path
from typing import Any, Dict, List, Optional, Callable, Tuple, Union
from dataclasses import dataclass, field

from .workflow_configs import (
    WorkflowPlanningConfig, WorkflowMemoryConfig,
    TaskContextConfig, TaskOutputConfig, TaskExecutionConfig, TaskRoutingConfig,
)
from ..task.task import Task

logger = get_logger(__name__)

def _parse_json_output(output: Any, step_name: str = "step") -> Any:
    """
    Parse JSON from LLM output if it's a string.
    
    Handles:
    - Direct JSON strings: '{"key": "value"}'
    - Markdown code blocks: ```json\n{"key": "value"}\n```
    - LLM echoing schema: {'type': 'array', 'items': [...]} -> extract items
    
    Returns:
        Parsed dict/list if successful, original output otherwise
    """
    if not isinstance(output, str) or not output:
        # Handle LLM echoing schema structure even for non-string outputs
        if isinstance(output, dict):
            return _extract_from_schema_echo(output)
        return output
    
    # Try direct JSON parse first
    try:
        parsed = json.loads(output)
        # Check if LLM echoed the schema structure
        if isinstance(parsed, dict):
            parsed = _extract_from_schema_echo(parsed)
        return parsed
    except json.JSONDecodeError:
        pass
    
    # Try extracting from markdown code block
    json_match = re.search(r'```(?:json)?\s*\n?([\s\S]*?)\n?```', output)
    if json_match:
        try:
            parsed = json.loads(json_match.group(1).strip())
            if isinstance(parsed, dict):
                parsed = _extract_from_schema_echo(parsed)
            return parsed
        except json.JSONDecodeError:
            pass
    
    # Try finding JSON object/array in text
    # Look for {...} or [...]
    for pattern in [r'(\{[^}]+\})', r'(\[[^\]]+\])']:
        match = re.search(pattern, output)
        if match:
            try:
                parsed = json.loads(match.group(1))
                if isinstance(parsed, dict):
                    parsed = _extract_from_schema_echo(parsed)
                return parsed
            except json.JSONDecodeError:
                continue
    
    # Return original if can't parse
    logger.debug(f"Could not parse JSON from step '{step_name}' output")
    return output

def _extract_from_schema_echo(data: dict) -> Any:
    """
    Extract actual data when LLM echoes the JSON schema structure.
    
    Common patterns:
    - {'type': 'array', 'items': [...]} -> return [...]
    - {'type': 'object', 'properties': {...}, 'data': {...}} -> return data
    - {'items': [...]} -> return [...] (native structured output wrapper)
    - {'keywords': [...]} -> return as-is (normal output)
    
    Args:
        data: Dict that might contain echoed schema
        
    Returns:
        Extracted data or original dict
    """
    # Check if this looks like a schema echo with 'type' and actual data
    if 'type' in data:
        data_type = data.get('type')
        
        # Array schema echo: {'type': 'array', 'items': [...]}
        if data_type == 'array' and 'items' in data:
            items = data['items']
            # If items is a list, that's our actual data
            if isinstance(items, list):
                logger.debug("Extracted items from schema-echoed array output")
                return items
        
        # Object schema echo: {'type': 'object', 'properties': {...}, 'data': {...}}
        if data_type == 'object':
            # Check for actual data field
            if 'data' in data and isinstance(data['data'], dict):
                logger.debug("Extracted data from schema-echoed object output")
                return data['data']
    
    # Handle native structured output wrapper: {'items': [...]}
    # This is created by _build_response_format when wrapping arrays
    if 'items' in data and len(data) == 1 and isinstance(data['items'], list):
        logger.debug("Extracted items from native structured output array wrapper")
        return data['items']
    
    # Not a schema echo, return as-is
    return data

@dataclass
class WorkflowContext:
    """Context passed to step handlers. Contains all information about the current workflow state."""
    input: str = ""  # Original workflow input
    previous_result: Optional[str] = None  # Output from previous step
    current_step: str = ""  # Current step name
    variables: Dict[str, Any] = field(default_factory=dict)  # All workflow variables
    
@dataclass
class StepResult:
    """Result returned from step handlers."""
    output: str = ""  # Step output content
    stop_workflow: bool = False  # If True, stop the entire workflow early
    variables: Dict[str, Any] = field(default_factory=dict)  # Variables to add/update

# Aliases for backward compatibility
StepInput = WorkflowContext
StepOutput = StepResult

# =============================================================================
# Workflow Pattern Helpers
# =============================================================================

@dataclass
class Route:
    """
    Decision-based branching. Routes to different steps based on previous output.
    
    Usage:
        workflow = Workflow(steps=[
            decision_maker,
            route({
                "approve": [approve_step],
                "reject": [reject_step],
                "default": [fallback_step]
            })
        ])
    """
    routes: Dict[str, List] = field(default_factory=dict)
    default: Optional[List] = None
    
    def __init__(self, routes: Dict[str, List], default: Optional[List] = None):
        self.routes = routes
        self.default = default or routes.get("default", [])

@dataclass  
class Parallel:
    """
    Execute multiple steps concurrently and combine results.
    
    Usage:
        workflow = Workflow(steps=[
            parallel([agent1, agent2, agent3]),
            aggregator
        ])
    """
    steps: List = field(default_factory=list)
    
    def __init__(self, steps: List):
        self.steps = steps

@dataclass
class Loop:
    """
    Iterate over a list or CSV file, executing step(s) for each item.
    
    Usage:
        # Loop over list variable (sequential) - single step
        workflow = Workflow(
            steps=[loop(processor, over="items")],
            variables={"items": ["a", "b", "c"]}
        )
        
        # Loop over CSV file
        workflow = Workflow(steps=[
            loop(processor, from_csv="data.csv")
        ])
        
        # Parallel loop over items (concurrent execution)
        workflow = Workflow(
            steps=[loop(processor, over="items", parallel=True)],
            variables={"items": ["a", "b", "c"]}
        )
        
        # Parallel with limited workers
        workflow = Workflow(
            steps=[loop(processor, over="items", parallel=True, max_workers=4)],
            variables={"items": ["a", "b", "c"]}
        )
        
        # Multi-step loop (NEW) - execute multiple steps per iteration
        workflow = Workflow(
            steps=[loop(
                steps=[researcher, writer, publisher],  # Multiple steps
                over="topics",
                parallel=True,
                max_workers=4
            )],
            variables={"topics": [...]}
        )
    """
    step: Any = None  # Single step (backward compat)
    steps: Optional[List[Any]] = None  # Multiple steps (NEW)
    over: Optional[str] = None  # Variable name containing list
    from_csv: Optional[str] = None  # CSV file path
    from_file: Optional[str] = None  # Text file path (one item per line)
    var_name: str = "item"  # Variable name for current item
    parallel: bool = False  # Execute iterations in parallel
    max_workers: Optional[int] = None  # Max parallel workers (None = unlimited)
    output_variable: Optional[str] = None  # Store loop results in this variable name
    
    def __init__(
        self, 
        step: Any = None,
        steps: Optional[List[Any]] = None,
        over: Optional[str] = None,
        from_csv: Optional[str] = None,
        from_file: Optional[str] = None,
        var_name: str = "item",
        parallel: bool = False,
        max_workers: Optional[int] = None,
        output_variable: Optional[str] = None
    ):
        # Validation: cannot have both step and steps
        if step is not None and steps is not None:
            raise ValueError("Cannot specify both 'step' and 'steps'")
        # Validation: must have at least one
        if step is None and steps is None:
            raise ValueError("Loop requires 'step' or 'steps'")
        # Validation: steps cannot be empty
        if steps is not None and len(steps) == 0:
            raise ValueError("Loop 'steps' cannot be empty")
        
        self.step = step
        self.steps = steps
        self.over = over
        self.from_csv = from_csv
        self.from_file = from_file
        self.var_name = var_name
        self.parallel = parallel
        self.max_workers = max_workers
        self.output_variable = output_variable

@dataclass
class Repeat:
    """
    Repeat a step until a condition is met (evaluator-optimizer pattern).
    
    Usage:
        workflow = Workflow(steps=[
            repeat(
                generator,
                until=lambda ctx: "done" in ctx.previous_result.lower(),
                max_iterations=5
            )
        ])
    """
    step: Any = None
    until: Optional[Callable[[WorkflowContext], bool]] = None
    max_iterations: int = 10
    
    def __init__(
        self,
        step: Any,
        until: Optional[Callable[[WorkflowContext], bool]] = None,
        max_iterations: int = 10
    ):
        self.step = step
        self.until = until
        self.max_iterations = max_iterations

# Convenience functions for cleaner API
def route(routes: Dict[str, List], default: Optional[List] = None) -> Route:
    """Create a routing decision point."""
    return Route(routes=routes, default=default)

def parallel(steps: List) -> Parallel:
    """Execute steps in parallel."""
    return Parallel(steps=steps)

def loop(step: Any = None, steps: Optional[List[Any]] = None,
         over: Optional[str] = None, from_csv: Optional[str] = None, 
         from_file: Optional[str] = None, var_name: str = "item",
         parallel: bool = False, max_workers: Optional[int] = None,
         output_variable: Optional[str] = None) -> Loop:
    """Loop over items executing step(s) for each.
    
    Args:
        step: Single step (agent/function) to execute for each item (backward compat)
        steps: Multiple steps to execute sequentially for each item (NEW)
        over: Variable name containing the list to iterate over
        from_csv: Path to CSV file to read items from
        from_file: Path to text file with one item per line
        var_name: Variable name for current item (default: "item")
        parallel: If True, execute iterations in parallel (default: False)
        max_workers: Max parallel workers when parallel=True (default: None = unlimited)
        output_variable: Variable name to store all loop outputs (default: None = "loop_outputs")
    
    Returns:
        Loop object configured for iteration
    """
    return Loop(step=step, steps=steps, over=over, from_csv=from_csv, from_file=from_file, 
                var_name=var_name, parallel=parallel, max_workers=max_workers,
                output_variable=output_variable)

def repeat(step: Any, until: Optional[Callable[[WorkflowContext], bool]] = None,
           max_iterations: int = 10) -> Repeat:
    """Repeat step until condition is met."""
    return Repeat(step=step, until=until, max_iterations=max_iterations)

@dataclass
class Include:
    """
    Include another recipe or workflow as a workflow step.
    
    Enables modular composition - recipes can include other recipes or workflows.
    
    Usage:
        workflow = Workflow(steps=[
            content_writer,
            include("wordpress-publisher"),  # Include another recipe
            include(workflow=my_other_workflow),  # Include another workflow
        ])
        
    YAML syntax:
        steps:
          - agent: content_writer
          - include: wordpress-publisher
          
        # Or with configuration:
          - include:
              recipe: wordpress-publisher
              input: "{{previous_output}}"
    """
    recipe: Optional[str] = None  # Recipe name or path
    workflow: Optional[Any] = None  # Workflow instance for composition
    input: Optional[str] = None  # Input override (default: previous_output)
    
    def __init__(self, recipe: Optional[str] = None, workflow: Optional[Any] = None, input: Optional[str] = None):
        if recipe is None and workflow is None:
            raise ValueError("Either 'recipe' or 'workflow' must be provided")
        self.recipe = recipe
        self.workflow = workflow
        self.input = input

def include(recipe: Optional[str] = None, workflow: Optional[Any] = None, input: Optional[str] = None) -> Include:
    """Include another recipe or workflow as a workflow step.
    
    Args:
        recipe: Recipe name or path to include
        workflow: Workflow instance to include
        input: Input override (default: previous_output)
    
    Returns:
        Include object for workflow composition
    """
    return Include(recipe=recipe, workflow=workflow, input=input)

# Maximum nesting depth for patterns (prevents stack overflow)
MAX_NESTING_DEPTH = 5

@dataclass
class If:
    """
    Conditional branching pattern for workflows.
    
    Evaluates a condition and executes either then_steps or else_steps.
    
    Usage:
        workflow = Workflow(steps=[
            if_(
                condition="{{score}} > 80",
                then_steps=[approve_step],
                else_steps=[reject_step]
            )
        ])
        
    YAML syntax:
        steps:
          - if:
              condition: "{{score}} > 80"
              then:
                - agent: approver
              else:
                - agent: rejector
    
    Supported condition formats:
        - Numeric comparison: "{{var}} > 80", "{{var}} >= 50", "{{var}} < 10"
        - String equality: "{{var}} == approved", "{{var}} != rejected"
        - Contains check: "error in {{message}}", "{{status}} contains success"
        - Boolean: "{{flag}}" (truthy check)
    """
    condition: str = ""  # Condition expression with {{var}} placeholders
    then_steps: List[Any] = field(default_factory=list)  # Steps if condition is true
    else_steps: Optional[List[Any]] = None  # Steps if condition is false (optional)
    
    def __init__(
        self,
        condition: str,
        then_steps: List[Any],
        else_steps: Optional[List[Any]] = None
    ):
        self.condition = condition
        self.then_steps = then_steps
        self.else_steps = else_steps or []

def when(
    condition: str,
    then_steps: List[Any],
    else_steps: Optional[List[Any]] = None
) -> If:
    """
    Create a conditional branching step.
    
    This is the preferred function for conditional branching (cleaner name than if_).
    
    Args:
        condition: Condition expression with {{var}} placeholders
        then_steps: Steps to execute if condition is true
        else_steps: Steps to execute if condition is false (optional)
    
    Returns:
        If object for conditional execution
    
    Example:
        when(
            condition="{{score}} > 80",
            then_steps=[approve_agent],
            else_steps=[reject_agent]
        )
    """
    return If(condition=condition, then_steps=then_steps, else_steps=else_steps)

def if_(
    condition: str,
    then_steps: List[Any],
    else_steps: Optional[List[Any]] = None
) -> If:
    """
    Create a conditional branching step.
    
    Alias for :func:`when`. Both functions work identically.
    
    Args:
        condition: Condition expression with {{var}} placeholders
        then_steps: Steps to execute if condition is true
        else_steps: Steps to execute if condition is false (optional)
    
    Returns:
        If object for conditional execution
    """
    return If(condition=condition, then_steps=then_steps, else_steps=else_steps)


@dataclass
class AgentFlow:
    """
    A complete workflow with multiple steps (deterministic pipeline).
    
    AgentFlow-centric API with consolidated feature parameters.
    All params follow precedence: Instance > Config > String > Bool > Default
    
    Usage:
        from praisonaiagents import AgentFlow, Agent
        
        # Simple flow
        flow = AgentFlow(
            steps=[Agent(instructions="Write content"), Agent(instructions="Edit content")],
            output="verbose",
        )
        result = flow.run("Write about AI")
        
        # With consolidated configs
        from praisonaiagents.workflows import WorkflowOutputConfig, WorkflowPlanningConfig
        
        flow = AgentFlow(
            steps=[...],
            output=WorkflowOutputConfig(verbose=True, stream=True),
            planning=WorkflowPlanningConfig(llm="gpt-4o", reasoning=True),
            hooks=WorkflowHooksConfig(on_step_complete=my_callback),
        )
    
    Note:
        The class was renamed from `Workflow` to `AgentFlow` in v1.0.
        `Workflow` and `Pipeline` remain as silent aliases for backward compatibility.
    """
    name: str = "AgentFlow"
    description: str = ""
    steps: List = field(default_factory=list)  # Can be Task, Agent, or function
    variables: Dict[str, Any] = field(default_factory=dict)
    file_path: Optional[str] = None
    
    # Default configuration for all steps
    default_agent_config: Optional[Dict[str, Any]] = None  # Default agent for all steps
    llm: Optional[str] = None  # Default LLM model (renamed from default_llm for consistency)
    
    # Process type: "sequential" (default) or "hierarchical" (manager-based validation)
    process: str = "sequential"  # "sequential", "hierarchical"
    manager_llm: Optional[str] = None  # LLM for manager agent (hierarchical mode)
    
    # ============================================================
    # CONSOLIDATED FEATURE PARAMS (workflow-centric API)
    # Precedence: Instance > Config > Array > Dict > String > Bool > Default
    # Workflow is "Agents in an organised way" - supports same params as Agents
    # ============================================================
    output: Optional[Any] = None  # Union[str, WorkflowOutputConfig] - verbose/stream
    planning: Optional[Any] = False  # Union[bool, str, WorkflowPlanningConfig] - planning mode
    memory: Optional[Any] = None  # Union[bool, WorkflowMemoryConfig, instance] - memory
    hooks: Optional[Any] = None  # WorkflowHooksConfig - lifecycle callbacks
    context: Optional[Any] = True  # Union[bool, ManagerConfig, ContextManager] - context management (enabled by default for workflows)
    # NEW: Agent-like consolidated params for feature parity
    autonomy: Optional[Any] = None  # Union[bool, AutonomyConfig] - agent autonomy
    knowledge: Optional[Any] = None  # Union[bool, List[str], KnowledgeConfig] - RAG
    guardrails: Optional[Any] = None  # Union[bool, Callable, GuardrailConfig] - validation
    web: Optional[Any] = None  # Union[bool, WebConfig] - web search/fetch
    reflection: Optional[Any] = None  # Union[bool, ReflectionConfig] - self-reflection
    execution: Optional[Any] = None  # Union[str, ExecutionConfig] - execution control
    caching: Optional[Any] = None  # Union[bool, CachingConfig] - caching
    learn: Optional[Any] = None  # Union[bool, LearnConfig] - continuous learning
    
    # ============================================================
    # ROBUSTNESS PARAMS (debugging & audit trail)
    # ============================================================
    history: bool = False  # Enable execution history tracking for debugging
    
    # Status tracking
    status: str = "not_started"  # not_started, running, completed, failed
    step_statuses: Dict[str, str] = field(default_factory=dict)  # {step_name: status}
    
    # Private resolved fields (set in __post_init__)
    _verbose: bool = field(default=False, repr=False)
    _stream: bool = field(default=True, repr=False)
    _output_config: Optional[Any] = field(default=None, repr=False)  # Full OutputConfig for propagation
    _planning_enabled: bool = field(default=False, repr=False)
    _planning_llm: Optional[str] = field(default=None, repr=False)
    _reasoning: bool = field(default=False, repr=False)
    _memory_config: Optional[Any] = field(default=None, repr=False)
    _hooks_config: Optional[Any] = field(default=None, repr=False)
    _context_manager: Optional[Any] = field(default=None, repr=False)
    _context_manager_initialized: bool = field(default=False, repr=False)
    _session_dedup_cache: Optional[Any] = field(default=None, repr=False)  # Shared session deduplication cache
    # NEW: Resolved configs for agent-like params (for propagation to steps/agents)
    _autonomy_config: Optional[Any] = field(default=None, repr=False)
    _knowledge_config: Optional[Any] = field(default=None, repr=False)
    _guardrails_config: Optional[Any] = field(default=None, repr=False)
    _web_config: Optional[Any] = field(default=None, repr=False)
    _reflection_config: Optional[Any] = field(default=None, repr=False)
    # Execution history for debugging (only populated when history=True)
    _execution_history: List[Dict[str, Any]] = field(default_factory=list, repr=False)
    
    def __post_init__(self):
        """Resolve consolidated params to internal values."""
        from .workflow_configs import (
            resolve_output_config, resolve_planning_config,
            resolve_memory_config, resolve_hooks_config,
        )
        
        # Resolve output param - now uses OUTPUT_PRESETS (same as Agent)
        output_cfg = resolve_output_config(self.output)
        self._output_config = output_cfg  # Store full config for propagation
        self._verbose = getattr(output_cfg, 'verbose', False) if output_cfg else False
        self._stream = getattr(output_cfg, 'stream', True) if output_cfg else True
        
        # Enable status/trace output if configured (same as Agent)
        if output_cfg:
            status_trace = getattr(output_cfg, 'status_trace', False)
            actions_trace = getattr(output_cfg, 'actions_trace', False)
            json_output = getattr(output_cfg, 'json_output', False)
            simple_output = getattr(output_cfg, 'simple_output', False)
            metrics = getattr(output_cfg, 'metrics', False)
            
            if status_trace:
                try:
                    from ..output.trace import enable_trace_output, is_trace_output_enabled
                    if not is_trace_output_enabled():
                        enable_trace_output(use_color=True, show_timestamps=True)
                except ImportError:
                    pass
            elif actions_trace:
                try:
                    from ..output.status import enable_status_output, is_status_output_enabled
                    if not is_status_output_enabled():
                        output_format = "jsonl" if json_output else "text"
                        enable_status_output(
                            redact=True,
                            use_color=True,
                            format=output_format,
                            show_timestamps=not simple_output,
                            show_metrics=metrics
                        )
                except ImportError:
                    pass
        
        # Resolve planning param
        planning_cfg = resolve_planning_config(self.planning)
        if planning_cfg:
            self._planning_enabled = planning_cfg.enabled
            self._planning_llm = planning_cfg.llm
            self._reasoning = planning_cfg.reasoning
        else:
            self._planning_enabled = False
            self._planning_llm = None
            self._reasoning = False
        
        # Resolve memory param
        self._memory_config = resolve_memory_config(self.memory)
        
        # Resolve hooks param
        self._hooks_config = resolve_hooks_config(self.hooks)
        
        # Resolve NEW agent-like params using canonical resolver
        # These are stored for propagation to steps/agents
        from ..config.param_resolver import resolve, ArrayMode
        from ..config.presets import (
            WEB_PRESETS, REFLECTION_PRESETS, GUARDRAIL_PRESETS,
            AUTONOMY_PRESETS, KNOWLEDGE_PRESETS,
        )
        try:
            from ..config.feature_configs import (
                AutonomyConfig, KnowledgeConfig, GuardrailConfig,
                WebConfig, ReflectionConfig,
            )
        except ImportError:
            AutonomyConfig = KnowledgeConfig = GuardrailConfig = None
            WebConfig = ReflectionConfig = None
        
        # Resolve autonomy
        if AutonomyConfig and self.autonomy is not None:
            self._autonomy_config = resolve(
                value=self.autonomy, param_name="autonomy",
                config_class=AutonomyConfig, presets=AUTONOMY_PRESETS,
                default=None,
            )
        
        # Resolve knowledge
        if KnowledgeConfig and self.knowledge is not None:
            self._knowledge_config = resolve(
                value=self.knowledge, param_name="knowledge",
                config_class=KnowledgeConfig, presets=KNOWLEDGE_PRESETS,
                array_mode=ArrayMode.SOURCES, default=None,
            )
        
        # Resolve guardrails
        if GuardrailConfig and self.guardrails is not None:
            self._guardrails_config = resolve(
                value=self.guardrails, param_name="guardrails",
                config_class=GuardrailConfig, presets=GUARDRAIL_PRESETS,
                default=None,
            )
        
        # Resolve web
        if WebConfig and self.web is not None:
            self._web_config = resolve(
                value=self.web, param_name="web",
                config_class=WebConfig, presets=WEB_PRESETS,
                default=None,
            )
        
        # Resolve reflection
        if ReflectionConfig and self.reflection is not None:
            self._reflection_config = resolve(
                value=self.reflection, param_name="reflection",
                config_class=ReflectionConfig, presets=REFLECTION_PRESETS,
                default=None,
            )
        
        # Initialize session deduplication cache for cross-agent deduplication
        if self.context:
            try:
                from ..context.manager import SessionDeduplicationCache
                self._session_dedup_cache = SessionDeduplicationCache()
            except ImportError:
                pass
    
    def get_history(self) -> List[Dict[str, Any]]:
        """Return execution trace for debugging.
        
        Returns a list of step execution records, each containing:
        - step: Step name
        - timestamp: ISO format timestamp
        - success: Whether the step succeeded
        - output: Truncated output (first 500 chars)
        - error: Error message if failed
        
        Only populated when history=True is set on the workflow.
        """
        return self._execution_history
    
    def to_dict(self) -> Dict[str, Any]:
        return {
            "name": self.name,
            "description": self.description,
            "steps": [s.to_dict() if hasattr(s, 'to_dict') else str(s) for s in self.steps],
            "variables": self.variables,
            "file_path": self.file_path,
            "default_agent_config": self.default_agent_config,
            "default_llm": self.llm,
            "process": self.process,
            "manager_llm": self.manager_llm,
            "output": {"verbose": self._verbose, "stream": self._stream},
            "planning": {"enabled": self._planning_enabled, "llm": self._planning_llm, "reasoning": self._reasoning},
            "memory": self._memory_config.to_dict() if hasattr(self._memory_config, 'to_dict') else bool(self._memory_config),
            "hooks": self._hooks_config.to_dict() if hasattr(self._hooks_config, 'to_dict') else None,
        }
    
    # Property accessors for backward compatibility during transition
    @property
    def verbose(self) -> bool:
        """Get verbose setting from resolved output config."""
        return self._verbose
    
    @verbose.setter
    def verbose(self, value: bool):
        """Set verbose setting."""
        self._verbose = value
    
    @property
    def stream(self) -> bool:
        """Get stream setting from resolved output config."""
        return self._stream
    
    @property
    def planning_llm(self) -> Optional[str]:
        """Get planning LLM from resolved planning config."""
        return self._planning_llm
    
    @property
    def reasoning(self) -> bool:
        """Get reasoning setting from resolved planning config."""
        return self._reasoning
    
    @property
    def memory_config(self) -> Optional[Dict[str, Any]]:
        """Get memory config dict for backward compatibility."""
        if self._memory_config:
            if hasattr(self._memory_config, 'to_dict'):
                return self._memory_config.to_dict()
            elif isinstance(self._memory_config, dict):
                return self._memory_config
        return None
    
    @property
    def on_workflow_start(self) -> Optional[Callable]:
        """Get on_workflow_start callback from hooks config."""
        return self._hooks_config.on_workflow_start if self._hooks_config else None
    
    @property
    def on_workflow_complete(self) -> Optional[Callable]:
        """Get on_workflow_complete callback from hooks config."""
        return self._hooks_config.on_workflow_complete if self._hooks_config else None
    
    @property
    def on_step_start(self) -> Optional[Callable]:
        """Get on_step_start callback from hooks config."""
        return self._hooks_config.on_step_start if self._hooks_config else None
    
    @property
    def on_step_complete(self) -> Optional[Callable]:
        """Get on_step_complete callback from hooks config."""
        return self._hooks_config.on_step_complete if self._hooks_config else None
    
    @property
    def on_step_error(self) -> Optional[Callable]:
        """Get on_step_error callback from hooks config."""
        return self._hooks_config.on_step_error if self._hooks_config else None
    
    def _resolve_pydantic_class(self, class_name: str) -> Optional[Any]:
        """
        Resolve a Pydantic class from string reference (Option B for structured output).
        
        Looks for the class in:
        1. tools.py in the workflow's directory
        2. The global namespace
        
        Args:
            class_name: Name of the Pydantic class to resolve
            
        Returns:
            The Pydantic class if found, None otherwise
        """
        import sys
        
        # Try to find the class in tools.py from the workflow's directory
        if self.file_path:
            from pathlib import Path
            workflow_dir = Path(self.file_path).parent
            tools_path = workflow_dir / "tools.py"
            
            if tools_path.exists():
                try:
                    import importlib.util
                    spec = importlib.util.spec_from_file_location("tools", tools_path)
                    if spec and spec.loader:
                        tools_module = importlib.util.module_from_spec(spec)
                        spec.loader.exec_module(tools_module)
                        
                        if hasattr(tools_module, class_name):
                            cls = getattr(tools_module, class_name)
                            # Verify it's a Pydantic model
                            if hasattr(cls, 'model_json_schema'):
                                return cls
                except Exception as e:
                    logger.debug(f"Failed to load Pydantic class from tools.py: {e}")
        
        # Try to find in already-imported modules
        for module_name, module in sys.modules.items():
            if module and hasattr(module, class_name):
                cls = getattr(module, class_name)
                if hasattr(cls, 'model_json_schema'):
                    return cls
        
        return None
    
    @classmethod
    def from_template(
        cls,
        uri: str,
        config: Optional[Dict[str, Any]] = None,
        offline: bool = False,
        **kwargs
    ) -> 'Workflow':
        """
        Create a Workflow from a template.
        
        Args:
            uri: Template URI (local path, package ref, or github ref)
                Examples:
                - "./my-template" (local path)
                - "transcript-generator" (default recipes repo)
                - "github:owner/repo/template@v1.0.0" (GitHub with version)
                - "package:agent_recipes/transcript-generator" (installed package)
            config: Optional configuration overrides
            offline: If True, only use cached templates (no network)
            **kwargs: Additional Workflow constructor arguments
            
        Returns:
            Configured Workflow instance
            
        Example:
            ```python
            from praisonaiagents import Workflow
            
            # From default recipes repo
            workflow = Workflow.from_template("transcript-generator")
            result = workflow.run("./audio.mp3")
            
            # With config overrides
            workflow = Workflow.from_template(
                "data-transformer",
                config={"output_format": "json"}
            )
            ```
        """
        try:
            # Lazy import to avoid circular dependencies and keep core SDK lean
            from praisonai.templates.loader import create_workflow_from_template
            return create_workflow_from_template(uri, config=config, offline=offline, **kwargs)
        except ImportError:
            raise ImportError(
                "Template support requires the 'praisonai' package. "
                "Install with: pip install praisonai"
            )
    
    def run(
        self,
        input: str = "",
        llm: Optional[str] = None,
        verbose: bool = False,
        stream: bool = None
    ) -> Dict[str, Any]:
        """
        Run the workflow with the given input.
        
        This is the simplest way to execute a workflow:
        
        ```python
        from praisonaiagents import Workflow, Agent
        
        workflow = Workflow(
            steps=[
                Agent(name="Writer", role="Write content"),
                my_validator_function,
                Agent(name="Editor", role="Edit content"),
            ]
        )
        result = workflow.run("Write about AI")
        ```
        
        Args:
            input: The input text/prompt for the workflow
            llm: LLM model to use (default: gpt-4o-mini)
            verbose: Print step outputs
            stream: Enable streaming responses (default: use workflow's stream setting)
            
        Returns:
            Dict with 'output' (final result) and 'steps' (all step results)
        """
        # Use default LLM if not specified
        model = llm or self.llm or "gpt-4o-mini"
        logger.debug(f"Workflow using model: {model} (llm={llm}, default_llm={self.llm})")
        
        # Use workflow verbose setting if not overridden
        verbose = verbose or self.verbose
        
        # Use workflow stream setting if not overridden
        if stream is None:
            stream = self.stream
        
        # Add input to variables
        all_variables = {**self.variables, "input": input}
        
        results = []
        previous_output = None
        
        # Update workflow status
        self.status = "running"
        
        # Set YAML-approved tools in approval context (for auto-approval of dangerous tools)
        _approval_token = None
        approve_tools = getattr(self, 'approve_tools', [])
        if approve_tools:
            from ..approval import set_yaml_approved_tools
            _approval_token = set_yaml_approved_tools(approve_tools)
        
        # Call on_workflow_start callback
        if self.on_workflow_start:
            try:
                self.on_workflow_start(self, input)
            except Exception as e:
                logger.error(f"on_workflow_start callback failed: {e}")
        
        # Planning mode - create execution plan before running
        if self.planning:
            plan = self._create_plan(input, model, verbose)
            if plan and verbose:
                print(f"📋 Execution Plan: {plan}")
            all_variables["execution_plan"] = plan
        
        # Hierarchical mode - use manager-based validation
        if self.process == "hierarchical":
            if verbose:
                print("🔄 Running workflow in hierarchical mode with manager validation")
            return self._run_hierarchical(
                input=input,
                model=model,
                verbose=verbose,
                stream=stream,
                all_variables=all_variables,
                _approval_token=_approval_token
            )
        
        # Process steps (may include Route, Parallel, Loop, Repeat)
        i = 0
        while i < len(self.steps):
            step = self.steps[i]
            
            # Handle special pattern types
            if isinstance(step, Route):
                # Decision-based routing
                route_result = self._execute_route(
                    step, previous_output, input, all_variables, model, verbose, stream
                )
                results.extend(route_result["steps"])
                previous_output = route_result["output"]
                all_variables.update(route_result.get("variables", {}))
                i += 1
                continue
                
            elif isinstance(step, Parallel):
                # Parallel execution
                parallel_result = self._execute_parallel(
                    step, previous_output, input, all_variables, model, verbose, stream
                )
                results.extend(parallel_result["steps"])
                previous_output = parallel_result["output"]
                all_variables.update(parallel_result.get("variables", {}))
                i += 1
                continue
                
            elif isinstance(step, Loop):
                # Loop over items
                loop_result = self._execute_loop(
                    step, previous_output, input, all_variables, model, verbose, stream
                )
                results.extend(loop_result["steps"])
                previous_output = loop_result["output"]
                all_variables.update(loop_result.get("variables", {}))
                i += 1
                continue
                
            elif isinstance(step, Repeat):
                # Repeat until condition
                repeat_result = self._execute_repeat(
                    step, previous_output, input, all_variables, model, verbose, stream
                )
                results.extend(repeat_result["steps"])
                previous_output = repeat_result["output"]
                all_variables.update(repeat_result.get("variables", {}))
                i += 1
                continue
                
            elif isinstance(step, Include):
                # Include another recipe
                include_result = self._execute_include(
                    step, previous_output, input, all_variables, model, verbose, stream
                )
                results.extend(include_result["steps"])
                previous_output = include_result["output"]
                all_variables.update(include_result.get("variables", {}))
                i += 1
                continue
            
            elif isinstance(step, If):
                # Conditional branching
                if_result = self._execute_if(
                    step, previous_output, input, all_variables, model, verbose, stream
                )
                results.extend(if_result["steps"])
                previous_output = if_result["output"]
                all_variables.update(if_result.get("variables", {}))
                i += 1
                continue
            
            # Normalize single step
            step = self._normalize_single_step(step, i)
            
            # Create context for handlers
            context = WorkflowContext(
                input=input,
                previous_result=str(previous_output) if previous_output else None,
                current_step=step.name,
                variables=all_variables.copy()
            )
            
            # Update step status
            if hasattr(step, 'status'):
                step.status = "running"
            self.step_statuses[step.name] = "running"
            
            # Call on_step_start callback
            if self.on_step_start:
                try:
                    self.on_step_start(step.name, context)
                except Exception as e:
                    logger.error(f"on_step_start callback failed: {e}")
            
            # Check should_run condition
            if step.should_run:
                try:
                    if not step.should_run(context):
                        if verbose:
                            print(f"⏭️ Skipped: {step.name}")
                        if hasattr(step, 'status'):
                            step.status = "skipped"
                        self.step_statuses[step.name] = "skipped"
                        i += 1
                        continue
                except Exception as e:
                    logger.error(f"should_run failed for {step.name}: {e}")
            
            # Execute step with retry and guardrail support
            output = None
            stop = False
            step_error = None
            max_retries = getattr(step, 'max_retries', 3)
            retry_count = 0
            validation_feedback = None
            
            while retry_count <= max_retries:
                step_error = None
                
                # Add validation feedback to context if retrying
                if validation_feedback:
                    context.variables["validation_feedback"] = validation_feedback
                
                try:
                    if step.handler:
                        # Custom handler function
                        result = step.handler(context)
                        if isinstance(result, StepResult):
                            output = result.output
                            stop = result.stop_workflow
                            if result.variables:
                                all_variables.update(result.variables)
                        else:
                            output = str(result)
                            
                    elif step.agent:
                        # Direct agent with tools
                        # Propagate context management to existing agent if workflow has it enabled
                        if self.context and hasattr(step.agent, '_context_manager_initialized'):
                            if not step.agent._context_manager_initialized:
                                step.agent._context_param = self.context
                            # Share session deduplication cache for cross-agent deduplication
                            if self._session_dedup_cache:
                                step.agent._session_dedup_cache = self._session_dedup_cache
                                # Also set on existing context manager if already initialized
                                if step.agent._context_manager and hasattr(step.agent._context_manager, '_session_cache'):
                                    step.agent._context_manager._session_cache = self._session_dedup_cache
                        
                        # Substitute variables in action
                        action = step.action or input
                        for key, value in all_variables.items():
                            action = action.replace(f"{{{{{key}}}}}", str(value))
                        if previous_output:
                            if "{{previous_output}}" in action:
                                action = action.replace("{{previous_output}}", str(previous_output))
                            else:
                                # Auto-append context if not explicitly referenced
                                action = f"{action}\n\nContext from previous step:\n{previous_output}"
                        action = action.replace("{{input}}", input)
                        
                        # Add reasoning prompt if enabled
                        if self.reasoning:
                            action = f"Think step by step and reason through this task:\n\n{action}"
                        
                        # Add validation feedback if retrying
                        if validation_feedback:
                            action = f"{action}\n\nPrevious attempt feedback: {validation_feedback}"
                        
                        # Check if this is a specialized agent (AudioAgent, VideoAgent, ImageAgent, OCRAgent)
                        agent_class_name = step.agent.__class__.__name__
                        if agent_class_name in ('AudioAgent', 'VideoAgent', 'ImageAgent', 'OCRAgent', 'DeepResearchAgent'):
                            # Use specialized agent dispatch
                            output = self._execute_specialized_agent(
                                step.agent, agent_class_name, action, step, all_variables, stream
                            )
                        else:
                            # Standard Agent with chat() method
                            # Handle images/attachments if present - check step.images first, then variables
                            step_attachments = getattr(step, 'images', None)
                            if not step_attachments:
                                # Check for image_path, image_url, or image in variables
                                image_var = all_variables.get('image_path') or all_variables.get('image_url') or all_variables.get('image')
                                if image_var:
                                    step_attachments = [image_var] if isinstance(image_var, str) else image_var
                                    logger.debug(f"Passing image attachment to agent: {image_var}")
                            # Get structured output options from step
                            step_output_json = getattr(step, '_output_json', None)
                            step_output_pydantic = getattr(step, '_output_pydantic', None)
                            
                            # Resolve Pydantic class from string reference (Option B)
                            # If output_pydantic is a string, try to resolve it from tools.py
                            if step_output_pydantic and isinstance(step_output_pydantic, str):
                                resolved_class = self._resolve_pydantic_class(step_output_pydantic)
                                if resolved_class:
                                    step_output_pydantic = resolved_class
                                else:
                                    logger.warning(f"Could not resolve Pydantic class '{step_output_pydantic}' from tools.py")
                                    step_output_pydantic = None
                            
                            # Build chat kwargs
                            chat_kwargs = {"stream": stream}
                            if step_attachments:
                                chat_kwargs["attachments"] = step_attachments
                            if step_output_json:
                                chat_kwargs["output_json"] = step_output_json
                            if step_output_pydantic:
                                chat_kwargs["output_pydantic"] = step_output_pydantic
                            
                            # Pass tool_choice from YAML if specified (auto, required, none)
                            # This forces the LLM to use tools when tool_choice='required'
                            yaml_tool_choice = getattr(step.agent, '_yaml_tool_choice', None)
                            if yaml_tool_choice:
                                chat_kwargs["tool_choice"] = yaml_tool_choice
                            
                            output = step.agent.chat(action, **chat_kwargs)
                            
                            # Parse JSON output if output_json was requested and output is a string
                            if step_output_json and output and isinstance(output, str):
                                output = _parse_json_output(output, step.name)
                            
                            # Handle output_pydantic if present
                            output_pydantic = getattr(step, 'output_pydantic', None)
                            if output_pydantic and output:
                                try:
                                    # Try to parse output as Pydantic model
                                    if hasattr(output_pydantic, 'model_validate_json'):
                                        parsed = output_pydantic.model_validate_json(output)
                                        output = parsed.model_dump_json()
                                except Exception as e:
                                    logger.debug(f"Pydantic parsing failed: {e}")
                            
                    elif step.action:
                        # Action with agent_config - create temporary agent
                        from ..agent.agent import Agent
                        config = step.agent_config or self.default_agent_config or {}
                        
                        # Get tools from step or config
                        step_tools = step.tools or config.get("tools", [])
                        
                        temp_agent = Agent(
                            name=config.get("name", step.name),
                            role=config.get("role", "Assistant"),
                            goal=config.get("goal", "Complete the task"),
                            llm=config.get("llm", model),
                            tools=step_tools if step_tools else None,
                            output=self.output,  # Propagate output config
                            context=self.context,  # Propagate context management
                            memory=self.memory,  # Propagate memory config
                            knowledge=self.knowledge,  # Propagate knowledge/RAG
                            guardrails=self.guardrails,  # Propagate guardrails
                            web=self.web,  # Propagate web search/fetch
                            reflection=self.reflection,  # Propagate self-reflection
                            autonomy=self.autonomy,  # Propagate autonomy settings
                            caching=self.caching,  # Propagate caching config
                            hooks=self.hooks,  # Propagate hooks/callbacks
                        )
                        # Substitute variables in action
                        action = step.action
                        for key, value in all_variables.items():
                            action = action.replace(f"{{{{{key}}}}}", str(value))
                        if previous_output:
                            if "{{previous_output}}" in action:
                                action = action.replace("{{previous_output}}", str(previous_output))
                            else:
                                # Auto-append context if not explicitly referenced
                                action = f"{action}\n\nContext from previous step:\n{previous_output}"
                        action = action.replace("{{input}}", input)
                        
                        # Add reasoning prompt if enabled
                        if self.reasoning:
                            action = f"Think step by step and reason through this task:\n\n{action}"
                        
                        if validation_feedback:
                            action = f"{action}\n\nPrevious attempt feedback: {validation_feedback}"
                        
                        output = temp_agent.chat(action, stream=stream)
                        
                        # Parse JSON output if output_json was requested
                        step_output_json = getattr(step, '_output_json', None)
                        if step_output_json and output and isinstance(output, str):
                            output = _parse_json_output(output, step.name)
                        
                except Exception as e:
                    step_error = e
                    output = f"Error: {e}"
                    if self.on_step_error:
                        try:
                            self.on_step_error(step.name, e)
                        except Exception:
                            pass
                
                # Check guardrail if present (guardrails is canonical, guardrail is deprecated)
                guardrail = getattr(step, 'guardrails', None) or getattr(step, 'guardrail', None)
                if guardrail and output and not step_error:
                    try:
                        is_valid, feedback = guardrail(StepResult(output=output))
                        if not is_valid:
                            validation_feedback = str(feedback)
                            retry_count += 1
                            if verbose:
                                print(f"⚠️ {step.name} failed validation (attempt {retry_count}/{max_retries}): {feedback}")
                            continue  # Retry
                    except Exception as e:
                        logger.error(f"Guardrail failed for {step.name}: {e}")
                
                # Success - break out of retry loop
                break
            
            # Update step status
            if step_error:
                if hasattr(step, 'status'):
                    step.status = "failed"
                self.step_statuses[step.name] = "failed"
            else:
                if hasattr(step, 'status'):
                    step.status = "completed"
                self.step_statuses[step.name] = "completed"
            
            # Create step result for callback
            step_result = StepResult(output=output or "", stop_workflow=stop)
            
            # Call on_step_complete callback
            if self.on_step_complete:
                try:
                    self.on_step_complete(step.name, step_result)
                except Exception as e:
                    logger.error(f"on_step_complete callback failed: {e}")
            
            # Handle output_file - save output to file
            if hasattr(step, 'output_file') and step.output_file and output:
                try:
                    import os
                    output_path = step.output_file
                    # Substitute variables in path
                    for key, value in all_variables.items():
                        output_path = output_path.replace(f"{{{{{key}}}}}", str(value))
                    # Create directory if needed
                    os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else ".", exist_ok=True)
                    with open(output_path, "w") as f:
                        f.write(str(output))
                    if verbose:
                        print(f"📁 Saved output to: {output_path}")
                except Exception as e:
                    logger.error(f"Failed to save output to file: {e}")
            
            # Store result
            results.append({
                "step": step.name,
                "output": output,
                "status": self.step_statuses.get(step.name, "completed"),
                "retries": retry_count
            })
            previous_output = output
            
            if verbose:
                print(f"✅ {step.name}: {str(output)}")
            
            # Handle early stop
            if stop:
                if verbose:
                    print(f"🛑 Workflow stopped at: {step.name}")
                break
            
            # Store output in variables
            var_name = step.output_variable or f"{step.name}_output"
            all_variables[var_name] = output
            
            # Validate output and warn about issues
            if output is None:
                logger.warning(f"⚠️  Step '{step.name}': Output is None. Agent may not have returned expected format.")
                if verbose:
                    print(f"⚠️  WARNING: Step '{step.name}' output is None!")
            else:
                # Check type against output_json schema if defined
                expected_schema = getattr(step, '_output_json', None)
                if expected_schema and isinstance(expected_schema, dict):
                    expected_type = expected_schema.get('type')
                    actual_type = type(output).__name__
                    if expected_type == 'object' and not isinstance(output, dict):
                        logger.warning(f"⚠️  Step '{step.name}': Expected object/dict, got {actual_type}")
                        if verbose:
                            print(f"⚠️  Step '{step.name}': Expected 'object', received '{actual_type}'")
                    elif expected_type == 'array' and not isinstance(output, list):
                        logger.warning(f"⚠️  Step '{step.name}': Expected array/list, got {actual_type}")
                        if verbose:
                            print(f"⚠️  Step '{step.name}': Expected 'array', received '{actual_type}'")
            
            i += 1
        
        # Update workflow status
        self.status = "completed"
        
        # Reset YAML-approved tools context if it was set
        if _approval_token is not None:
            from ..approval import reset_yaml_approved_tools
            reset_yaml_approved_tools(_approval_token)
        
        final_result = {
            "output": previous_output,
            "steps": results,
            "variables": all_variables,
            "status": self.status
        }
        
        # Call on_workflow_complete callback
        if self.on_workflow_complete:
            try:
                self.on_workflow_complete(self, final_result)
            except Exception as e:
                logger.error(f"on_workflow_complete callback failed: {e}")
        
        return final_result
    
    async def astart(
        self,
        input: str = "",
        llm: Optional[str] = None,
        verbose: bool = False
    ) -> Dict[str, Any]:
        """Async version of start() for async workflow execution.
        
        Args:
            input: The input text/prompt for the workflow
            llm: LLM model to use (default: gpt-4o-mini)
            verbose: Print step outputs
            
        Returns:
            Dict with 'output' (final result) and 'steps' (all step results)
        """
        import asyncio
        
        # Run the synchronous version in a thread pool
        # Use copy_context_to_callable to propagate contextvars (needed for trace emission)
        from ..trace.context_events import copy_context_to_callable
        loop = asyncio.get_event_loop()
        result = await loop.run_in_executor(
            None,
            copy_context_to_callable(lambda: self.run(input, llm, verbose))
        )
        return result
    
    async def arun(
        self,
        input: str = "",
        llm: Optional[str] = None,
        verbose: bool = False
    ) -> Dict[str, Any]:
        """Alias for astart() for backward compatibility."""
        return await self.astart(input, llm, verbose)
    
    def _run_hierarchical(
        self,
        input: str,
        model: str,
        verbose: bool,
        stream: bool,
        all_variables: Dict[str, Any],
        _approval_token: Optional[Any] = None
    ) -> Dict[str, Any]:
        """Run workflow with hierarchical manager-based validation.
        
        In hierarchical mode, a manager agent validates each step's output
        before proceeding to the next step. If the manager rejects a step,
        the workflow stops with status='failed' and includes a failure_reason.
        
        This pattern is adapted from Process.hierarchical() for Workflow use.
        
        Args:
            input: The workflow input
            model: LLM model to use for steps
            verbose: Print step outputs
            stream: Enable streaming
            all_variables: Variables dict
            _approval_token: Token for YAML-approved tools
            
        Returns:
            Dict with output, steps, variables, status, and optionally failure_reason
        """
        from ..llm import LLM
        
        results = []
        previous_output = input
        failure_reason = None
        
        manager_model = self.manager_llm or model
        
        normalized_steps = self._normalize_steps()
        
        for i, step in enumerate(normalized_steps):
            step_name = getattr(step, 'name', f'step_{i}')
            
            if self.on_step_start:
                try:
                    self.on_step_start(self, step, i)
                except Exception as e:
                    logger.error(f"on_step_start callback failed: {e}")
            
            if hasattr(step, 'status'):
                step.status = "running"
            self.step_statuses[step_name] = "running"
            
            try:
                # Use _execute_single_step_internal which returns a dict
                step_result_internal = self._execute_single_step_internal(
                    step, previous_output, input, all_variables, model, verbose, i, stream
                )
                output = step_result_internal.get("output", "")
                stop = step_result_internal.get("stop", False)
                step_vars = step_result_internal.get("variables", {})
                all_variables.update(step_vars)
                
                step_result = {
                    "step": step_name,
                    "output": output,
                    "status": "completed"
                }
                results.append(step_result)
                
                if hasattr(step, 'status'):
                    step.status = "completed"
                self.step_statuses[step_name] = "completed"
                
                if self.on_step_complete:
                    try:
                        self.on_step_complete(self, step, step_result)
                    except Exception as e:
                        logger.error(f"on_step_complete callback failed: {e}")
                
                if stop:
                    if verbose:
                        print(f"🛑 Workflow stopped at: {step_name}")
                    break
                
                # Check if step has tools assigned
                step_has_tools = bool(getattr(step, 'tools', None) or (hasattr(step, 'agent') and getattr(step.agent, 'tools', None)))
                
                # Build tool evidence guidance for manager
                tool_evidence_guidance = ""
                if step_has_tools:
                    tool_evidence_guidance = """
4. TOOL USAGE EVIDENCE: If the step required tool usage (e.g., search_web), check for EVIDENCE in the output:
   - URLs or links (e.g., https://..., [Source](url))
   - Structured data with sources/citations
   - Specific facts with references
   - Multiple distinct items from external sources
   NOTE: The agent may have called tools successfully even if it doesn't say "I called the tool".
   Look for RESULTS from tools, not explicit statements about calling them."""
                
                validation_prompt = f"""You are reviewing the output of a workflow step.

Step Name: {step_name}
Step Action: {getattr(step, 'action', 'Execute task')}
Expected Output: {getattr(step, 'expected_output', 'Task completed successfully')}

Agent Output:
{output}

Did this step complete successfully? Consider:
1. Does the output address the task?
2. Is the output meaningful (not an error message)?
3. Does it meet the expected output criteria?{tool_evidence_guidance}

IMPORTANT: Approve if the output contains substantive content that addresses the task.
Only reject if the output is clearly an error, empty, or completely off-topic.

Respond with JSON: {{"approved": true/false, "reason": "Brief explanation"}}
"""
                
                try:
                    llm_instance = LLM(model=manager_model, verbose=False)
                    validation_response = llm_instance.get_response(
                        prompt=validation_prompt,
                        system_prompt="You are a workflow quality manager. Respond only with valid JSON.",
                        output_json=True
                    )
                    
                    if isinstance(validation_response, str):
                        import json
                        try:
                            decision_data = json.loads(validation_response)
                        except json.JSONDecodeError:
                            decision_data = {"approved": True, "reason": "Could not parse response, assuming success"}
                    elif isinstance(validation_response, dict):
                        decision_data = validation_response
                    else:
                        decision_data = {"approved": True, "reason": "Unknown response format, assuming success"}
                    
                    approved = decision_data.get("approved", True)
                    reason = decision_data.get("reason", "No reason provided")
                    
                    if verbose:
                        status_icon = "✅" if approved else "❌"
                        print(f"{status_icon} Manager validation for '{step_name}': {reason}")
                    
                    if not approved:
                        failure_reason = f"Manager rejected step '{step_name}': {reason}"
                        self.status = "failed"
                        if hasattr(step, 'status'):
                            step.status = "failed"
                        self.step_statuses[step_name] = "failed"
                        step_result["status"] = "failed"
                        step_result["failure_reason"] = failure_reason
                        
                        if self.on_step_error:
                            try:
                                self.on_step_error(self, step, Exception(failure_reason))
                            except Exception as e:
                                logger.error(f"on_step_error callback failed: {e}")
                        break
                        
                except Exception as e:
                    logger.warning(f"Manager validation failed for step '{step_name}': {e}. Continuing workflow.")
                
                previous_output = output
                
                # Store output in variables for next step's variable substitution
                # This is critical for {{agent_name}}_output references to work
                var_name = f"{step_name}_output"
                all_variables[var_name] = output
                
            except Exception as e:
                error_msg = str(e)
                if hasattr(step, 'status'):
                    step.status = "failed"
                self.step_statuses[step_name] = "failed"
                
                step_result = {
                    "step": step_name,
                    "output": f"Error: {error_msg}",
                    "status": "failed"
                }
                results.append(step_result)
                
                if self.on_step_error:
                    try:
                        self.on_step_error(self, step, e)
                    except Exception as callback_e:
                        logger.error(f"on_step_error callback failed: {callback_e}")
                
                failure_reason = f"Step '{step_name}' failed with error: {error_msg}"
                self.status = "failed"
                break
        
        if failure_reason is None:
            self.status = "completed"
        
        if _approval_token is not None:
            from ..approval import reset_yaml_approved_tools
            reset_yaml_approved_tools(_approval_token)
        
        final_result = {
            "output": previous_output,
            "steps": results,
            "variables": all_variables,
            "status": self.status
        }
        
        if failure_reason:
            final_result["failure_reason"] = failure_reason
        
        if self.on_workflow_complete:
            try:
                self.on_workflow_complete(self, final_result)
            except Exception as e:
                logger.error(f"on_workflow_complete callback failed: {e}")
        
        return final_result
    
    def _normalize_steps(self) -> List['Task']:
        """Convert mixed steps (Agent, function, Task) to Task list.
        
        This method uses _normalize_single_step to ensure consistent normalization
        and avoid duplicated code paths (DRY principle).
        """
        return [self._normalize_single_step(step, i) for i, step in enumerate(self.steps)]
    
    def _create_plan(self, input: str, model: str, verbose: bool) -> Optional[str]:
        """Create an execution plan for the workflow using LLM.
        
        Args:
            input: The workflow input
            model: LLM model to use
            verbose: Print verbose output
            
        Returns:
            Execution plan as string, or None if planning fails
        """
        try:
            from ..agent.agent import Agent
            
            # Describe the steps
            step_descriptions = []
            for i, step in enumerate(self.steps):
                if isinstance(step, Task):
                    step_descriptions.append(f"{i+1}. {step.name}: {step.description or step.action or 'Execute step'}")
                elif hasattr(step, 'name'):
                    role = getattr(step, 'role', 'Agent')
                    step_descriptions.append(f"{i+1}. {step.name} ({role})")
                elif callable(step):
                    step_descriptions.append(f"{i+1}. {getattr(step, '__name__', 'function')}")
                elif isinstance(step, (Route, Parallel, Loop, Repeat)):
                    step_descriptions.append(f"{i+1}. {type(step).__name__} pattern")
                else:
                    step_descriptions.append(f"{i+1}. Step {i+1}")
            
            steps_text = "\n".join(step_descriptions)
            
            planning_prompt = f"""You are a workflow planner. Given the following workflow steps and input, create a brief execution plan.

Workflow: {self.name}
Description: {self.description}

Steps:
{steps_text}

Input: {input}

Create a brief execution plan (2-3 sentences) describing how to best accomplish this task using the available steps. Focus on the key objectives and any important considerations."""

            planner = Agent(
                name="Planner",
                role="Workflow Planner",
                goal="Create efficient execution plans",
                llm=self.planning_llm or model,
            )
            
            plan = planner.chat(planning_prompt)
            return plan
            
        except Exception as e:
            logger.error(f"Planning failed: {e}")
            return None
    
    def _execute_specialized_agent(
        self,
        agent: Any,
        agent_class_name: str,
        action: str,
        step: 'Task',
        variables: Dict[str, Any],
        stream: bool
    ) -> Any:
        """
        Execute a specialized agent (AudioAgent, VideoAgent, ImageAgent, OCRAgent).
        
        Dispatches to the appropriate method based on agent type and action.
        Falls back to chat() for standard Agent class.
        
        Args:
            agent: The agent instance
            agent_class_name: Name of the agent class
            action: The action/prompt to execute
            step: The workflow step containing additional config
            variables: Current workflow variables
            stream: Whether to stream output
            
        Returns:
            Output from the agent execution
        """
        # Get yaml_action which specifies the method to call
        yaml_action = getattr(agent, '_yaml_action', None) or action
        
        # AudioAgent - TTS (speech) or STT (transcribe)
        if agent_class_name == 'AudioAgent':
            if yaml_action == 'speech' or 'speech' in yaml_action.lower():
                # Text-to-Speech
                text = variables.get('text', action)
                output_file = variables.get('output') or variables.get('audio_output') or 'output.mp3'
                voice = variables.get('voice', 'alloy')
                result = agent.speech(text, output=output_file, voice=voice)
                # Store the output file path in variables for next step
                variables['_last_audio_file'] = output_file
                return result
            elif yaml_action == 'transcribe' or 'transcribe' in yaml_action.lower():
                # Speech-to-Text - check multiple variable names for input file
                input_file = (
                    variables.get('audio_file') or 
                    variables.get('audio') or 
                    variables.get('_last_audio_file') or  # From previous TTS step
                    variables.get('input')
                )
                # If input is empty string or template, use default
                if not input_file or input_file == '{{input}}' or 'Context from previous' in str(input_file):
                    input_file = 'output.mp3'  # Default TTS output
                language = variables.get('language', 'en')
                return agent.transcribe(input_file, language=language)
            else:
                # Default to speech if action contains text
                return agent.speech(action, output=variables.get('output', 'output.mp3'))
        
        # VideoAgent - generate
        elif agent_class_name == 'VideoAgent':
            prompt = variables.get('prompt', action)
            output_file = variables.get('output', 'output.mp4')
            return agent.generate(prompt, output=output_file)
        
        # ImageAgent - generate with output path support
        elif agent_class_name == 'ImageAgent':
            prompt = variables.get('prompt', action)
            output_path = variables.get('output') or variables.get('image_output')
            result = agent.generate(prompt)
            # Extract URL from result and store in variables for next step
            if hasattr(result, 'data') and result.data:
                image_url = result.data[0].url if hasattr(result.data[0], 'url') else None
                if image_url:
                    variables['_last_image_url'] = image_url
                    # Save to file if output path specified
                    if output_path:
                        try:
                            import urllib.request
                            urllib.request.urlretrieve(image_url, output_path)
                            variables['_last_image_file'] = output_path
                        except Exception as e:
                            logger.warning(f"Failed to save image to {output_path}: {e}")
            return result
        
        # OCRAgent - extract with retry logic for external API failures
        elif agent_class_name == 'OCRAgent':
            source = variables.get('source') or variables.get('url') or variables.get('document')
            # If no explicit source variable, try to extract URL from action text
            if not source:
                import re
                url_pattern = r'https?://[^\s<>"{}|\\^`\[\]]+'
                urls = re.findall(url_pattern, action)
                source = urls[0] if urls else action
            
            # Retry logic for external API failures
            max_retries = 2
            last_error = None
            for attempt in range(max_retries + 1):
                try:
                    return agent.extract(source)
                except Exception as e:
                    last_error = e
                    error_str = str(e).lower()
                    # Only retry on transient errors (network, timeout, fetch failures)
                    if any(x in error_str for x in ['fetch', 'timeout', 'connection', 'network']):
                        if attempt < max_retries:
                            import time
                            time.sleep(1 * (attempt + 1))  # Exponential backoff
                            continue
                    # Non-retryable error, raise immediately
                    raise
            # All retries exhausted
            raise last_error
        
        # DeepResearchAgent - research
        elif agent_class_name == 'DeepResearchAgent':
            query = variables.get('query', action)
            return agent.research(query)
        
        # Default: standard Agent with chat() method
        else:
            return agent.chat(action, stream=stream)
    
    def _normalize_single_step(self, step: Any, index: int) -> 'Task':
        """Normalize a single step to Task.
        
        Supports:
        - Task objects (passed through)
        - Agent objects (wrapped with agent reference and tools)
        - Specialized agents (AudioAgent, VideoAgent, ImageAgent, OCRAgent)
        - Callable functions (wrapped as handler)
        - Strings (used as action)
        """
        if isinstance(step, Task):
            return step
        
        # Check for standard Agent (has chat method) or specialized agents
        agent_class_name = step.__class__.__name__
        is_specialized_agent = agent_class_name in ('AudioAgent', 'VideoAgent', 'ImageAgent', 'OCRAgent', 'DeepResearchAgent')
        
        if hasattr(step, 'chat') or is_specialized_agent:
            # It's an Agent - wrap with agent reference and preserve tools
            agent_tools = getattr(step, 'tools', None)
            # Check for _yaml_action from YAML parser (canonical: action, alias: description)
            yaml_action = getattr(step, '_yaml_action', None)
            # Check for _yaml_output_variable from YAML parser
            yaml_output_variable = getattr(step, '_yaml_output_variable', None)
            # Check for _yaml_output_json from YAML parser (structured output - Option A)
            yaml_output_json = getattr(step, '_yaml_output_json', None)
            # Check for _yaml_output_pydantic from YAML parser (structured output - Option B)
            yaml_output_pydantic = getattr(step, '_yaml_output_pydantic', None)
            # Check for _yaml_step_name from YAML parser
            yaml_step_name = getattr(step, '_yaml_step_name', None)
            # Use yaml_action if set, otherwise fall back to previous_output/input
            default_action = "{{previous_output}}" if index > 0 else "{{input}}"
            action = yaml_action if yaml_action else default_action
            # Build output config with all structured output options
            output_config = {}
            if yaml_output_variable:
                output_config["variable"] = yaml_output_variable
            if yaml_output_json:
                output_config["json_model"] = yaml_output_json
            if yaml_output_pydantic:
                output_config["pydantic_model"] = yaml_output_pydantic
            # Only pass output_config if it has values
            output_json_model = output_config.get('json_model') if output_config else None
            output_pydantic_model = output_config.get('pydantic_model') if output_config else None
            # Extract remaining YAML step-level attributes (previously dead stores)
            yaml_guardrail = getattr(step, '_yaml_guardrail', None)
            yaml_max_retries = getattr(step, '_yaml_max_retries', None)
            yaml_expected_output = getattr(step, '_yaml_expected_output', None)
            yaml_output_file = getattr(step, '_yaml_output_file', None)
            yaml_skip_on_failure = getattr(step, '_yaml_skip_on_failure', None)
            yaml_retry_delay = getattr(step, '_yaml_retry_delay', None)
            yaml_create_directory = getattr(step, '_yaml_create_directory', None)
            yaml_callback = getattr(step, '_yaml_callback', None)
            yaml_async_execution = getattr(step, '_yaml_async_execution', None)

            # Build Task kwargs — only include YAML overrides when present
            task_kwargs = dict(
                name=yaml_step_name or getattr(step, 'name', f'agent_{index+1}'),
                agent=step,
                tools=agent_tools,
                action=action,
                output_json=output_json_model,
                output_pydantic=output_pydantic_model,
            )
            if yaml_guardrail is not None:
                task_kwargs['guardrails'] = yaml_guardrail
            if yaml_max_retries is not None:
                task_kwargs['max_retries'] = yaml_max_retries
            if yaml_expected_output is not None:
                task_kwargs['expected_output'] = yaml_expected_output
            if yaml_output_file is not None:
                task_kwargs['output_file'] = yaml_output_file
            if yaml_skip_on_failure is not None:
                task_kwargs['skip_on_failure'] = yaml_skip_on_failure
            if yaml_retry_delay is not None:
                task_kwargs['retry_delay'] = yaml_retry_delay
            if yaml_create_directory is not None:
                task_kwargs['create_directory'] = yaml_create_directory
            if yaml_callback is not None:
                task_kwargs['callback'] = yaml_callback
            if yaml_async_execution is not None:
                task_kwargs['async_execution'] = yaml_async_execution

            return Task(**task_kwargs)
        elif callable(step):
            return Task(
                name=getattr(step, '__name__', f'step_{index+1}'),
                handler=step
            )
        else:
            return Task(
                name=f'step_{index+1}',
                action=str(step)
            )
    
    def _execute_single_step_internal(
        self, 
        step: Any, 
        previous_output: Optional[str],
        input: str,
        all_variables: Dict[str, Any],
        model: str,
        verbose: bool,
        index: int = 0,
        stream: bool = True,
        depth: int = 0
    ) -> Dict[str, Any]:
        """Execute a single step and return result.
        
        Args:
            step: The step to execute (Agent, function, or pattern like Loop/Parallel/Route/If)
            previous_output: Output from previous step
            input: Original workflow input
            all_variables: Current workflow variables
            model: LLM model to use
            verbose: Whether to print verbose output
            index: Step index
            stream: Whether to stream responses
            depth: Current nesting depth (for pattern recursion safety)
        
        Returns:
            Dict with 'step', 'output', 'stop', 'variables'
        """
        # Check nesting depth limit for nested patterns
        if depth > MAX_NESTING_DEPTH:
            raise ValueError(
                f"Maximum nesting depth ({MAX_NESTING_DEPTH}) exceeded. "
                "Simplify your workflow or reduce pattern nesting."
            )
        
        # Handle nested patterns (Loop, Parallel, Route, Repeat, If)
        if isinstance(step, Loop):
            loop_result = self._execute_loop(
                step, previous_output, input, all_variables, model, verbose, stream, depth=depth+1
            )
            return {
                "step": f"loop_{index}",
                "output": loop_result.get("output", ""),
                "stop": False,
                "variables": loop_result.get("variables", all_variables)
            }
        
        if isinstance(step, Parallel):
            parallel_result = self._execute_parallel(
                step, previous_output, input, all_variables, model, verbose, stream, depth=depth+1
            )
            return {
                "step": f"parallel_{index}",
                "output": parallel_result.get("output", ""),
                "stop": False,
                "variables": parallel_result.get("variables", all_variables)
            }
        
        if isinstance(step, Route):
            route_result = self._execute_route(
                step, previous_output, input, all_variables, model, verbose, stream, depth=depth+1
            )
            return {
                "step": f"route_{index}",
                "output": route_result.get("output", ""),
                "stop": False,
                "variables": route_result.get("variables", all_variables)
            }
        
        if isinstance(step, Repeat):
            repeat_result = self._execute_repeat(
                step, previous_output, input, all_variables, model, verbose, stream, depth=depth+1
            )
            return {
                "step": f"repeat_{index}",
                "output": repeat_result.get("output", ""),
                "stop": False,
                "variables": repeat_result.get("variables", all_variables)
            }
        
        if isinstance(step, If):
            if_result = self._execute_if(
                step, previous_output, input, all_variables, model, verbose, stream, depth=depth+1
            )
            return {
                "step": f"if_{index}",
                "output": if_result.get("output", ""),
                "stop": False,
                "variables": if_result.get("variables", all_variables)
            }
        
        # Handle Include steps (for include inside loop)
        if isinstance(step, Include):
            include_result = self._execute_include(
                step, previous_output, input, all_variables, model, verbose, stream
            )
            # Return in the expected format for single step
            combined_output = include_result.get("output", "")
            return {
                "step": f"include:{step.recipe}",
                "output": combined_output,
                "stop": False,
                "variables": include_result.get("variables", all_variables)
            }
        
        normalized = self._normalize_single_step(step, index)
        
        context = WorkflowContext(
            input=input,
            previous_result=str(previous_output) if previous_output else None,
            current_step=normalized.name,
            variables=all_variables.copy()
        )
        
        output = None
        stop = False
        
        if normalized.handler:
            try:
                result = normalized.handler(context)
                if isinstance(result, StepResult):
                    output = result.output
                    stop = result.stop_workflow
                    if result.variables:
                        all_variables.update(result.variables)
                else:
                    output = str(result)
            except Exception as e:
                output = f"Error: {e}"
        elif normalized.agent:
            try:
                # Propagate context management to existing agent if workflow has it enabled
                if self.context and hasattr(normalized.agent, '_context_manager_initialized'):
                    if not normalized.agent._context_manager_initialized:
                        normalized.agent._context_param = self.context
                    # Share session deduplication cache for cross-agent deduplication
                    if self._session_dedup_cache:
                        normalized.agent._session_dedup_cache = self._session_dedup_cache
                        # Also set on existing context manager if already initialized
                        if normalized.agent._context_manager and hasattr(normalized.agent._context_manager, '_session_cache'):
                            normalized.agent._context_manager._session_cache = self._session_dedup_cache
                
                action = normalized.action or input
                # Substitute variables
                for key, value in all_variables.items():
                    action = action.replace(f"{{{{{key}}}}}", str(value))
                if previous_output:
                    if "{{previous_output}}" in action:
                        action = action.replace("{{previous_output}}", str(previous_output))
                    else:
                        # Auto-append context if not explicitly referenced
                        action = f"{action}\n\nContext from previous step:\n{previous_output}"
                action = action.replace("{{input}}", input)
                
                # Check if this is a specialized agent (AudioAgent, VideoAgent, ImageAgent, OCRAgent)
                agent_class_name = normalized.agent.__class__.__name__
                output = self._execute_specialized_agent(
                    normalized.agent, agent_class_name, action, normalized, all_variables, stream
                )
                
                # Parse JSON output if output_json was requested
                step_output_json = getattr(normalized, '_output_json', None)
                if step_output_json and output and isinstance(output, str):
                    output = _parse_json_output(output, normalized.name)
            except Exception as e:
                output = f"Error: {e}"
        elif normalized.action:
            try:
                from ..agent.agent import Agent
                config = normalized.agent_config or self.default_agent_config or {}
                temp_agent = Agent(
                    name=config.get("name", normalized.name),
                    role=config.get("role", "Assistant"),
                    goal=config.get("goal", "Complete the task"),
                    llm=config.get("llm", model),
                    output=self.output,  # Propagate output config
                    context=self.context,  # Propagate context management
                    memory=self.memory,  # Propagate memory config
                    knowledge=self.knowledge,  # Propagate knowledge/RAG
                    guardrails=self.guardrails,  # Propagate guardrails
                    web=self.web,  # Propagate web search/fetch
                    reflection=self.reflection,  # Propagate self-reflection
                    autonomy=self.autonomy,  # Propagate autonomy settings
                    caching=self.caching,  # Propagate caching config
                    hooks=self.hooks,  # Propagate hooks/callbacks
                )
                action = normalized.action
                for key, value in all_variables.items():
                    action = action.replace(f"{{{{{key}}}}}", str(value))
                if previous_output:
                    if "{{previous_output}}" in action:
                        action = action.replace("{{previous_output}}", str(previous_output))
                    else:
                        # Auto-append context if not explicitly referenced
                        action = f"{action}\n\nContext from previous step:\n{previous_output}"
                
                output = temp_agent.chat(action, stream=stream)
                
                # Parse JSON output if output_json was requested
                step_output_json = getattr(normalized, '_output_json', None)
                if step_output_json and output and isinstance(output, str):
                    output = _parse_json_output(output, normalized.name)
            except Exception as e:
                output = f"Error: {e}"
        
        if verbose:
            print(f"✅ {normalized.name}: {str(output)}")
        
        return {
            "step": normalized.name,
            "output": output,
            "stop": stop,
            "variables": all_variables
        }
    
    def _execute_route(
        self,
        route_step: Route,
        previous_output: Optional[str],
        input: str,
        all_variables: Dict[str, Any],
        model: str,
        verbose: bool,
        stream: bool = True,
        depth: int = 0
    ) -> Dict[str, Any]:
        """Execute routing based on previous output."""
        results = []
        output = previous_output
        
        # Find matching route
        matched_route = None
        prev_lower = str(previous_output).lower() if previous_output else ""
        
        for key in route_step.routes:
            if key.lower() in prev_lower or key == "default":
                if key != "default":
                    matched_route = route_step.routes[key]
                    break
        
        if matched_route is None:
            matched_route = route_step.default or []
        
        if verbose:
            route_name = next((k for k in route_step.routes if route_step.routes[k] == matched_route), "default")
            print(f"🔀 Routing to: {route_name}")
        
        # Execute matched route steps
        for idx, step in enumerate(matched_route):
            step_result = self._execute_single_step_internal(
                step, output, input, all_variables, model, verbose, idx, stream=stream, depth=depth+1
            )
            results.append({"step": step_result["step"], "output": step_result["output"]})
            output = step_result["output"]
            all_variables.update(step_result.get("variables", {}))
            
            if step_result.get("stop"):
                break
        
        return {"steps": results, "output": output, "variables": all_variables}
    
    def _llm_summarize_for_parallel(
        self,
        content: str,
        num_branches: int,
        model: str,
        verbose: bool = False
    ) -> str:
        """
        Use LLM to create a concise summary of content for parallel distribution.
        
        This reduces token usage when the same context is passed to multiple parallel branches.
        """
        from ..context.tokens import estimate_tokens_heuristic
        
        # Only summarize if content is large enough
        tokens = estimate_tokens_heuristic(content)
        if tokens < 1500:
            return content
        
        # Target summary size: ~500 tokens to leave room for each branch's work
        target_tokens = min(500, tokens // num_branches)
        
        try:
            import litellm
            
            prompt = f"""Summarize the following content concisely, preserving all key information, data, and actionable items. 
Keep the summary under {target_tokens * 4} characters. Focus on facts, numbers, and specific details.

CONTENT:
{content[:8000]}

CONCISE SUMMARY:"""
            
            response = litellm.completion(
                model=model,
                messages=[{"role": "user", "content": prompt}],
                max_tokens=target_tokens,
                temperature=0.1,
            )
            
            summary = response.choices[0].message.content.strip()
            
            if verbose:
                print(f"  🤖 LLM summarized context: {tokens:,} → {estimate_tokens_heuristic(summary):,} tokens")
            
            return summary
            
        except Exception as e:
            logger.debug(f"LLM summarization failed, using truncation: {e}")
            raise  # Let caller handle fallback
    
    def _execute_parallel(
        self,
        parallel_step: Parallel,
        previous_output: Optional[str],
        input: str,
        all_variables: Dict[str, Any],
        model: str,
        verbose: bool,
        stream: bool = True,
        depth: int = 0
    ) -> Dict[str, Any]:
        """Execute steps in parallel (simulated with sequential for now)."""
        import concurrent.futures
        
        results = []
        outputs = []
        
        if verbose:
            print(f"⚡ Running {len(parallel_step.steps)} steps in parallel...")
        
        # Optimize: Use LLM-based summarization before distributing to parallel branches
        # This prevents rate limits and reduces token waste
        optimized_previous = previous_output
        num_branches = len(parallel_step.steps)
        if previous_output and len(previous_output) > 3000:
            from ..context.tokens import estimate_tokens_heuristic
            tokens = estimate_tokens_heuristic(previous_output)
            if tokens > 1000:
                # Try LLM-based summarization first, fall back to truncation
                try:
                    optimized_previous = self._llm_summarize_for_parallel(previous_output, num_branches, model, verbose)
                except Exception:
                    # Fallback to truncation-based summarization
                    max_chars = min(2500, len(previous_output) // max(num_branches, 2))
                    if len(previous_output) > max_chars:
                        optimized_previous = previous_output[:max_chars * 2 // 3] + "\n\n[... context summarized for parallel efficiency ...]\n\n" + previous_output[-max_chars // 3:]
                
                if verbose and optimized_previous != previous_output:
                    new_tokens = estimate_tokens_heuristic(optimized_previous)
                    saved = tokens - new_tokens
                    print(f"  📦 Optimized context for {num_branches} parallel branches: {tokens:,} → {new_tokens:,} tokens (saved {saved:,} per branch)")
        
        # Use ThreadPoolExecutor for parallel execution
        # IMPORTANT: Limit max_workers to prevent rate limit issues (max 3 concurrent branches)
        from ..trace.context_events import copy_context_to_callable, get_context_emitter
        
        effective_workers = min(3, len(parallel_step.steps))  # Cap at 3 to prevent rate limits
        with concurrent.futures.ThreadPoolExecutor(max_workers=effective_workers) as executor:
            futures = []
            for idx, step in enumerate(parallel_step.steps):
                # Wrap execution to propagate context and set branch_id for parallel tracking
                def execute_with_branch(step=step, idx=idx, opt_prev=optimized_previous):
                    emitter = get_context_emitter()
                    emitter.set_branch(f"parallel_{idx}")
                    try:
                        return self._execute_single_step_internal(
                            step, opt_prev, input, all_variables.copy(), model, False, idx, stream, depth=depth+1
                        )
                    finally:
                        emitter.clear_branch()
                
                future = executor.submit(copy_context_to_callable(execute_with_branch))
                futures.append((idx, future))
            
            for idx, future in futures:
                try:
                    step_result = future.result()
                    results.append({"step": step_result["step"], "output": step_result["output"]})
                    outputs.append(step_result["output"])
                except Exception as e:
                    results.append({"step": f"parallel_{idx}", "output": f"Error: {e}"})
                    outputs.append(f"Error: {e}")
        
        # Combine outputs
        combined_output = "\n---\n".join(str(o) for o in outputs)
        all_variables["parallel_outputs"] = outputs
        
        if verbose:
            print(f"✅ Parallel complete: {len(outputs)} results")
        
        return {"steps": results, "output": combined_output, "variables": all_variables}
    
    def _execute_loop(
        self,
        loop_step: Loop,
        previous_output: Optional[str],
        input: str,
        all_variables: Dict[str, Any],
        model: str,
        verbose: bool,
        stream: bool = True,
        depth: int = 0
    ) -> Dict[str, Any]:
        """Execute step for each item in loop.
        
        When loop_step.parallel is True, executes iterations concurrently
        using ThreadPoolExecutor for faster processing.
        """
        import csv
        import concurrent.futures
        
        results = []
        outputs = []
        items = []
        
        # Get items from variable, CSV, or file
        if loop_step.over:
            items = all_variables.get(loop_step.over, [])
            if isinstance(items, str):
                # Try to parse as JSON list first (handles agent output like '["a", "b", "c"]')
                items = self._parse_list_from_string(items)
        elif loop_step.from_csv:
            try:
                with open(loop_step.from_csv, 'r', encoding='utf-8') as f:
                    reader = csv.DictReader(f)
                    items = list(reader)
            except Exception as e:
                logger.error(f"Failed to read CSV {loop_step.from_csv}: {e}")
                items = []
        elif loop_step.from_file:
            try:
                with open(loop_step.from_file, 'r', encoding='utf-8') as f:
                    items = [line.strip() for line in f if line.strip()]
            except Exception as e:
                logger.error(f"Failed to read file {loop_step.from_file}: {e}")
                items = []
        
        num_items = len(items)
        
        # Determine steps to run (multi-step or single step)
        steps_to_run = loop_step.steps if loop_step.steps else [loop_step.step]
        is_multi_step = loop_step.steps is not None and len(loop_step.steps) > 1
        
        if loop_step.parallel and num_items > 1:
            # Parallel execution - cap workers to prevent rate limits
            max_workers = min(loop_step.max_workers or num_items, 3)  # Cap at 3 to prevent rate limits
            if verbose:
                step_info = f" ({len(steps_to_run)} steps each)" if is_multi_step else ""
                print(f"⚡🔁 Parallel looping over {num_items} items{step_info} (max_workers={max_workers})...")
            
            # Optimize: Aggressively summarize large previous_output before distributing to parallel branches
            # This reduces token waste and prevents rate limit issues
            optimized_previous = previous_output
            if previous_output and len(previous_output) > 3000:
                from ..context.tokens import estimate_tokens_heuristic
                tokens = estimate_tokens_heuristic(previous_output)
                # Target: max 800 tokens per branch to stay well under rate limits
                if tokens > 1000:
                    max_chars = min(2500, len(previous_output) // max(num_items, 2))
                    if len(previous_output) > max_chars:
                        # Extract key information: first part (context) + last part (recent output)
                        optimized_previous = previous_output[:max_chars * 2 // 3] + "\n\n[... context summarized for parallel efficiency ...]\n\n" + previous_output[-max_chars // 3:]
                        if verbose:
                            new_tokens = estimate_tokens_heuristic(optimized_previous)
                            saved = tokens - new_tokens
                            print(f"  📦 Optimized context for {num_items} parallel branches: {tokens:,} → {new_tokens:,} tokens (saved {saved:,} per branch)")
            
            # Use copy_context_to_callable to propagate contextvars (needed for trace emission)
            from ..trace.context_events import copy_context_to_callable, get_context_emitter
            
            def execute_item(idx_item_tuple, opt_prev=optimized_previous):
                """Execute step(s) for a single item in thread."""
                idx, item = idx_item_tuple
                
                # Set branch context for parallel tracking
                emitter = get_context_emitter()
                emitter.set_branch(f"loop_{idx}")
                
                try:
                    # CRITICAL: Deep copy variables to ensure thread isolation (per-branch context isolation)
                    import copy
                    loop_vars = copy.deepcopy(all_variables)
                    loop_vars[loop_step.var_name] = item
                    loop_vars["loop_index"] = idx
                    
                    # Also expand nested item properties for template access (e.g., {{item.title}})
                    if isinstance(item, dict):
                        for key, value in item.items():
                            loop_vars[f"{loop_step.var_name}.{key}"] = value
                    
                    # Execute all steps sequentially within this iteration
                    iteration_output = opt_prev
                    iteration_results = []
                    for step_idx, step in enumerate(steps_to_run):
                        step_result = self._execute_single_step_internal(
                            step, iteration_output, input, loop_vars, model, False, step_idx, stream=False, depth=depth+1
                        )
                        iteration_output = step_result.get("output")
                        iteration_results.append(step_result)
                        # Update variables if step set any
                        if step_result.get("variables"):
                            loop_vars.update(step_result["variables"])
                    
                    # Return final output (last step's output)
                    final_result = {
                        "step": f"loop_{idx}",
                        "output": iteration_output,
                        "steps": iteration_results
                    }
                    return idx, final_result
                finally:
                    emitter.clear_branch()
            
            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
                futures = [
                    executor.submit(copy_context_to_callable(lambda pair=(idx, item): execute_item(pair)))
                    for idx, item in enumerate(items)
                ]
                
                # Collect results in order
                indexed_results = []
                for future in concurrent.futures.as_completed(futures):
                    try:
                        idx, step_result = future.result()
                        indexed_results.append((idx, step_result))
                        if verbose:
                            print(f"  ✓ Item {idx + 1}/{num_items} complete")
                    except Exception as e:
                        logger.error(f"Parallel loop iteration failed: {e}")
                        indexed_results.append((idx, {"step": f"loop_{idx}", "output": f"Error: {e}"}))
                
                # Sort by index to maintain order
                indexed_results.sort(key=lambda x: x[0])
                
                for idx, step_result in indexed_results:
                    results.append({"step": f"{step_result['step']}_{idx}", "output": step_result["output"]})
                    outputs.append(step_result["output"])
            
            if verbose:
                print(f"✅ Parallel loop complete: {len(outputs)} results")
        else:
            # Sequential execution (original behavior)
            if verbose:
                step_info = f" ({len(steps_to_run)} steps each)" if is_multi_step else ""
                print(f"🔁 Looping over {num_items} items{step_info}...")
            
            for idx, item in enumerate(items):
                # Add current item to variables
                import copy
                loop_vars = copy.deepcopy(all_variables)
                loop_vars[loop_step.var_name] = item
                loop_vars["loop_index"] = idx
                
                # Also expand nested item properties for template access (e.g., {{item.title}})
                if isinstance(item, dict):
                    for key, value in item.items():
                        loop_vars[f"{loop_step.var_name}.{key}"] = value
                
                # Execute all steps sequentially within this iteration
                iteration_output = previous_output
                for step_idx, step in enumerate(steps_to_run):
                    step_result = self._execute_single_step_internal(
                        step, iteration_output, input, loop_vars, model, verbose, step_idx, stream=stream, depth=depth+1
                    )
                    iteration_output = step_result.get("output")
                    # Update variables if step set any
                    if step_result.get("variables"):
                        loop_vars.update(step_result["variables"])
                
                results.append({"step": f"loop_{idx}", "output": iteration_output})
                outputs.append(iteration_output)
                previous_output = iteration_output
        # Store outputs in user-specified variable or default to loop_outputs
        output_var_name = loop_step.output_variable or "loop_outputs"
        all_variables[output_var_name] = outputs
        all_variables["loop_outputs"] = outputs  # Also keep for backward compatibility
        combined_output = "\n".join(str(o) for o in outputs) if outputs else ""
        
        # Validate outputs and warn about issues
        none_count = sum(1 for o in outputs if o is None)
        if none_count > 0:
            logger.warning(f"⚠️  Loop '{output_var_name}': {none_count}/{len(outputs)} outputs are None. "
                          f"Check if agent returned expected format.")
            if verbose:
                print(f"⚠️  WARNING: {none_count}/{len(outputs)} loop outputs are None!")
        
        # Check for type consistency
        if outputs and len(outputs) > 0:
            expected_type = loop_step.step._output_json if hasattr(loop_step.step, '_output_json') else None
            if expected_type:
                expected_schema_type = expected_type.get('type') if isinstance(expected_type, dict) else None
                for i, o in enumerate(outputs):
                    if o is None:
                        continue
                    actual_type = type(o).__name__
                    if expected_schema_type == 'object' and not isinstance(o, dict):
                        logger.warning(f"⚠️  Loop output[{i}]: Expected object/dict, got {actual_type}")
                        if verbose:
                            print(f"⚠️  Loop output[{i}]: Expected 'object', received '{actual_type}'")
                    elif expected_schema_type == 'array' and not isinstance(o, list):
                        logger.warning(f"⚠️  Loop output[{i}]: Expected array/list, got {actual_type}")
                        if verbose:
                            print(f"⚠️  Loop output[{i}]: Expected 'array', received '{actual_type}'")
        
        # Debug logging for output_variable
        if verbose:
            print(f"📦 Loop stored {len(outputs)} results in variable: '{output_var_name}'")
        
        return {"steps": results, "output": combined_output, "variables": all_variables}
    
    def _parse_list_from_string(self, text: str) -> List[Any]:
        """
        Parse a list from a string, handling multiple formats.
        
        Supports:
        1. Pure JSON array: '["a", "b", "c"]'
        2. JSON array embedded in text: 'Here are topics: ["a", "b", "c"]'
        3. Newline-separated items (fallback)
        
        Args:
            text: String that may contain a list
            
        Returns:
            List of items, or [text] if no list found
        """
        import json
        import re
        
        if not text or not text.strip():
            return []
        
        text = text.strip()
        
        # 1. Try direct JSON parse (pure JSON array)
        try:
            parsed = json.loads(text)
            if isinstance(parsed, list):
                return parsed
        except json.JSONDecodeError:
            pass
        
        # 2. Try to extract JSON array from text using regex
        # Match JSON arrays like ["item1", "item2"] or ['item1', 'item2']
        json_array_pattern = r'\[(?:[^\[\]]*(?:"[^"]*"|\'[^\']*\')?[^\[\]]*)*\]'
        matches = re.findall(json_array_pattern, text)
        
        for match in matches:
            try:
                # Try parsing as JSON
                parsed = json.loads(match)
                if isinstance(parsed, list) and len(parsed) > 0:
                    return parsed
            except json.JSONDecodeError:
                # Try replacing single quotes with double quotes
                try:
                    fixed = match.replace("'", '"')
                    parsed = json.loads(fixed)
                    if isinstance(parsed, list) and len(parsed) > 0:
                        return parsed
                except json.JSONDecodeError:
                    continue
        
        # 3. Fallback: wrap as single item
        return [text]
    
    def _execute_repeat(
        self,
        repeat_step: Repeat,
        previous_output: Optional[str],
        input: str,
        all_variables: Dict[str, Any],
        model: str,
        verbose: bool,
        stream: bool = True,
        depth: int = 0
    ) -> Dict[str, Any]:
        """Repeat step until condition is met."""
        results = []
        output = previous_output
        
        if verbose:
            print(f"🔄 Repeating up to {repeat_step.max_iterations} times...")
        
        for iteration in range(repeat_step.max_iterations):
            step_result = self._execute_single_step_internal(
                repeat_step.step, output, input, all_variables, model, verbose, iteration, stream=stream, depth=depth+1
            )
            results.append({"step": f"{step_result['step']}_{iteration}", "output": step_result["output"]})
            output = step_result["output"]
            all_variables.update(step_result.get("variables", {}))
            
            # Check until condition
            if repeat_step.until:
                context = WorkflowContext(
                    input=input,
                    previous_result=str(output) if output else None,
                    current_step=f"repeat_{iteration}",
                    variables=all_variables.copy()
                )
                try:
                    if repeat_step.until(context):
                        if verbose:
                            print(f"✅ Repeat condition met at iteration {iteration + 1}")
                        break
                except Exception as e:
                    logger.error(f"Repeat until condition failed: {e}")
            
            if step_result.get("stop"):
                break
        
        all_variables["repeat_iterations"] = iteration + 1
        return {"steps": results, "output": output, "variables": all_variables}
    
    def _execute_if(
        self,
        if_step: If,
        previous_output: Optional[str],
        input: str,
        all_variables: Dict[str, Any],
        model: str,
        verbose: bool,
        stream: bool = True,
        depth: int = 0
    ) -> Dict[str, Any]:
        """Execute conditional branching based on condition evaluation.
        
        Args:
            if_step: The If step with condition and branches
            previous_output: Output from previous step
            input: Original workflow input
            all_variables: Current workflow variables
            model: LLM model to use
            verbose: Verbose output
            stream: Enable streaming
            depth: Current nesting depth
            
        Returns:
            Dict with 'steps', 'output', and 'variables'
        """
        results = []
        output = previous_output
        
        # Evaluate the condition
        condition_result = self._evaluate_condition(if_step.condition, all_variables, previous_output)
        
        if verbose:
            branch = "then" if condition_result else "else"
            print(f"🔀 Condition '{if_step.condition}' → {condition_result}, executing {branch} branch")
        
        # Select the appropriate branch
        steps_to_execute = if_step.then_steps if condition_result else if_step.else_steps
        
        # Execute the selected branch
        for idx, step in enumerate(steps_to_execute):
            step_result = self._execute_single_step_internal(
                step, output, input, all_variables, model, verbose, idx, stream=stream, depth=depth
            )
            results.append({"step": step_result["step"], "output": step_result["output"]})
            output = step_result["output"]
            all_variables.update(step_result.get("variables", {}))
            
            if step_result.get("stop"):
                break
        
        return {"steps": results, "output": output, "variables": all_variables}
    
    def _evaluate_condition(
        self,
        condition: str,
        variables: Dict[str, Any],
        previous_output: Optional[str] = None
    ) -> bool:
        """Evaluate a condition expression with variable substitution.
        
        This method delegates to the shared evaluate_condition() function
        from praisonaiagents.conditions for DRY condition handling.
        
        Supported formats:
            - Numeric comparison: "{{var}} > 80", "{{var}} >= 50", "{{var}} < 10"
            - String equality: "{{var}} == approved", "{{var}} != rejected"
            - Contains check: "error in {{message}}", "{{status}} contains success"
            - Boolean: "{{flag}}" (truthy check)
            - Nested property: "{{item.score}} >= 60"
        
        Args:
            condition: Condition expression with {{var}} placeholders
            variables: Current workflow variables
            previous_output: Output from previous step
            
        Returns:
            Boolean result of condition evaluation
        """
        from praisonaiagents.conditions.evaluator import evaluate_condition
        return evaluate_condition(condition, variables, previous_output)
    
    def _execute_include(
        self,
        include_step: Include,
        previous_output: Optional[str],
        input: str,
        all_variables: Dict[str, Any],
        model: str,
        verbose: bool,
        stream: bool = True,
        visited_recipes: Optional[set] = None
    ) -> Dict[str, Any]:
        """Execute an included recipe as a workflow step.
        
        Args:
            include_step: The Include step with recipe name
            previous_output: Output from previous step
            input: Original workflow input
            all_variables: Current workflow variables
            model: LLM model to use
            verbose: Verbose output
            stream: Enable streaming
            visited_recipes: Set of already-visited recipe names (for cycle detection)
            
        Returns:
            Dict with 'steps', 'output', and 'variables'
        """
        recipe_name = include_step.recipe
        
        # Cycle detection
        if visited_recipes is None:
            visited_recipes = set()
        
        if recipe_name in visited_recipes:
            error_msg = f"Circular include detected: {recipe_name} was already included in this execution chain"
            logger.error(error_msg)
            return {
                "steps": [{"step": f"include:{recipe_name}", "output": f"Error: {error_msg}"}],
                "output": error_msg,
                "variables": all_variables
            }
        
        visited_recipes.add(recipe_name)
        
        if verbose:
            print(f"📦 Including recipe: {recipe_name}")
        
        try:
            # Try to resolve recipe path using agent_recipes if available
            recipe_path = None
            
            try:
                from agent_recipes import get_template_path
                recipe_path = get_template_path(recipe_name)
            except ImportError:
                # agent_recipes not installed, try local path
                from pathlib import Path
                if Path(recipe_name).exists():
                    recipe_path = Path(recipe_name)
                else:
                    # Try as relative path to current working directory
                    import os
                    cwd = Path(os.getcwd())
                    potential_path = cwd / recipe_name
                    if potential_path.exists():
                        recipe_path = potential_path
            
            if not recipe_path:
                error_msg = f"Recipe not found: {recipe_name}"
                logger.error(error_msg)
                return {
                    "steps": [{"step": f"include:{recipe_name}", "output": f"Error: {error_msg}"}],
                    "output": error_msg,
                    "variables": all_variables
                }
            
            # Find the YAML file (agents.yaml or workflow.yaml)
            recipe_yaml = None
            from pathlib import Path
            recipe_path = Path(recipe_path)
            
            for yaml_name in ["agents.yaml", "workflow.yaml", "TEMPLATE.yaml"]:
                yaml_path = recipe_path / yaml_name
                if yaml_path.exists():
                    recipe_yaml = yaml_path
                    break
            
            if not recipe_yaml:
                error_msg = f"No workflow YAML found in {recipe_path}"
                logger.error(error_msg)
                return {
                    "steps": [{"step": f"include:{recipe_name}", "output": f"Error: {error_msg}"}],
                    "output": error_msg,
                    "variables": all_variables
                }
            
            # Load tools from the recipe's tools.py if present
            tool_registry = {}
            tools_py = recipe_path / "tools.py"
            if tools_py.exists():
                try:
                    import importlib.util
                    spec = importlib.util.spec_from_file_location("recipe_tools", tools_py)
                    recipe_module = importlib.util.module_from_spec(spec)
                    spec.loader.exec_module(recipe_module)
                    
                    # Extract callable tools
                    for name in dir(recipe_module):
                        if not name.startswith("_"):
                            obj = getattr(recipe_module, name)
                            if callable(obj):
                                tool_registry[name] = obj
                except Exception as e:
                    logger.warning(f"Failed to load tools from {tools_py}: {e}")
            
            # Determine input for the included recipe
            recipe_input = include_step.input or previous_output or input
            
            # Convert non-string inputs to JSON string for variable substitution
            if recipe_input and not isinstance(recipe_input, str):
                recipe_input = json.dumps(recipe_input)
            
            # Substitute variables in the input (only if it's a string)
            if recipe_input and isinstance(recipe_input, str):
                for key, value in all_variables.items():
                    recipe_input = recipe_input.replace(f"{{{{{key}}}}}", str(value))
                if previous_output:
                    prev_str = previous_output if isinstance(previous_output, str) else json.dumps(previous_output)
                    recipe_input = recipe_input.replace("{{previous_output}}", prev_str)
                recipe_input = recipe_input.replace("{{input}}", input)
            
            # Parse and execute the included workflow
            from .yaml_parser import YAMLWorkflowParser
            parser = YAMLWorkflowParser(tool_registry=tool_registry)
            included_workflow = parser.parse_file(str(recipe_yaml))
            
            # Merge parent variables into included workflow
            included_workflow.variables.update(all_variables)
            included_workflow.variables["previous_output"] = previous_output
            
            # Execute the included workflow
            result = included_workflow.run(
                input=recipe_input or "",
                llm=model,
                verbose=verbose,
                stream=stream
            )
            
            if verbose:
                print(f"✅ Included recipe {recipe_name} completed")
            
            # Return result in expected format
            return {
                "steps": result.get("steps", []),
                "output": result.get("output", ""),
                "variables": {**all_variables, **result.get("variables", {})}
            }
            
        except Exception as e:
            logger.error(f"Failed to execute included recipe {recipe_name}: {e}")
            return {
                "steps": [{"step": f"include:{recipe_name}", "output": f"Error: {e}"}],
                "output": f"Error executing {recipe_name}: {e}",
                "variables": all_variables
            }
        finally:
            # Remove from visited after execution completes (allow re-use in different branches)
            visited_recipes.discard(recipe_name)
    
    def start(self, input: str = "", **kwargs) -> Dict[str, Any]:
        """Alias for run() for consistency with Agents."""
        return self.run(input, **kwargs)

# Backward compatibility aliases (silent - no deprecation warnings)
# AgentFlow is the primary name (v1.0+)
# Workflow and Pipeline are silent aliases
Workflow = AgentFlow
Pipeline = AgentFlow

class WorkflowManager:
    """
    Manages workflow discovery, loading, and execution.
    
    Workflows are defined in markdown files with YAML frontmatter
    and step definitions.
    
    Example workflow file (.praisonai/workflows/deploy.md):
        ```markdown
        ---
        name: Deploy to Production
        description: Deploy the application to production
        variables:
          environment: production
          branch: main
        ---
        
        ## Step 1: Run Tests
        Run all tests to ensure code quality.
        
        ```action
        Run the test suite with pytest
        ```
        
        ## Step 2: Build
        Build the application for production.
        
        ```action
        Build the application with production settings
        ```
        
        ## Step 3: Deploy
        Deploy to the production server.
        
        ```action
        Deploy to {{environment}} from {{branch}}
        ```
        ```
    
    Example usage:
        ```python
        manager = WorkflowManager(workspace_path="/path/to/project")
        
        # List available workflows
        workflows = manager.list_workflows()
        
        # Get a specific workflow
        deploy = manager.get_workflow("deploy")
        
        # Execute a workflow
        results = manager.execute(
            "deploy",
            executor=lambda prompt: agent.chat(prompt),
            variables={"branch": "release-1.0"}
        )
        ```
    """
    
    from ..paths import DEFAULT_DIR_NAME as _DIR_NAME
    WORKFLOWS_DIR = f"{_DIR_NAME}/workflows"
    SUPPORTED_EXTENSIONS = [".md", ".yaml", ".yml"]
    
    def __init__(
        self,
        workspace_path: Optional[str] = None,
        verbose: int = 0
    ):
        """
        Initialize WorkflowManager.
        
        Args:
            workspace_path: Path to workspace/project root
            verbose: Verbosity level
        """
        self.workspace_path = Path(workspace_path) if workspace_path else Path.cwd()
        self.verbose = verbose
        
        self._workflows: Dict[str, Workflow] = {}
        self._loaded = False
    
    def _log(self, msg: str, level: int = logging.INFO):
        """Log message if verbose."""
        if self.verbose >= 1:
            logger.log(level, msg)
    
    def _ensure_loaded(self):
        """Lazy load workflows on first access."""
        if not self._loaded:
            self._discover_workflows()
            self._loaded = True
    
    def _parse_frontmatter(self, content: str) -> tuple[Dict[str, Any], str]:
        """Parse YAML frontmatter from markdown content."""
        frontmatter = {}
        body = content
        
        if content.startswith("---"):
            parts = content.split("---", 2)
            if len(parts) >= 3:
                yaml_content = parts[1].strip()
                body = parts[2].strip()
                
                # Simple YAML parsing
                for line in yaml_content.split("\n"):
                    line = line.strip()
                    if ":" in line and not line.startswith("#"):
                        key, value = line.split(":", 1)
                        key = key.strip()
                        value = value.strip()
                        
                        # Handle basic types
                        if value.lower() in ("true", "false"):
                            value = value.lower() == "true"
                        elif value.isdigit():
                            value = int(value)
                        elif value.startswith('"') and value.endswith('"'):
                            value = value[1:-1]
                        
                        frontmatter[key] = value
        
        return frontmatter, body
    
    def _parse_steps(self, body: str) -> List[Task]:
        """Parse workflow steps from markdown body."""
        steps = []
        
        # Split by ## headers
        step_pattern = re.compile(r'^##\s+(?:Step\s+\d+[:\s]*)?(.+)$', re.MULTILINE)
        action_pattern = re.compile(r'```action\s*\n(.*?)\n```', re.DOTALL)
        condition_pattern = re.compile(r'```condition\s*\n(.*?)\n```', re.DOTALL)
        agent_pattern = re.compile(r'```agent\s*\n(.*?)\n```', re.DOTALL)
        tools_pattern = re.compile(r'```tools\s*\n(.*?)\n```', re.DOTALL)
        context_from_pattern = re.compile(r'context_from:\s*\[([^\]]+)\]', re.IGNORECASE)
        retain_context_pattern = re.compile(r'retain_full_context:\s*(true|false)', re.IGNORECASE)
        output_var_pattern = re.compile(r'output_variable:\s*(\w+)', re.IGNORECASE)
        next_steps_pattern = re.compile(r'next_steps:\s*\[([^\]]+)\]', re.IGNORECASE)
        loop_over_pattern = re.compile(r'loop_over:\s*(\w+)', re.IGNORECASE)
        loop_var_pattern = re.compile(r'loop_var:\s*(\w+)', re.IGNORECASE)
        branch_pattern = re.compile(r'```branch\s*\n(.*?)\n```', re.DOTALL)
        
        # New patterns for route, parallel, images, output_file
        route_pattern = re.compile(r'```route\s*\n(.*?)\n```', re.DOTALL)
        parallel_pattern = re.compile(r'```parallel\s*\n(.*?)\n```', re.DOTALL)
        images_pattern = re.compile(r'```images\s*\n(.*?)\n```', re.DOTALL)
        output_file_pattern = re.compile(r'output_file:\s*(.+)', re.IGNORECASE)
        repeat_pattern = re.compile(r'```repeat\s*\n(.*?)\n```', re.DOTALL)
        
        # Find all step headers
        matches = list(step_pattern.finditer(body))
        
        for i, match in enumerate(matches):
            step_name = match.group(1).strip()
            
            # Get content until next step or end
            start = match.end()
            end = matches[i + 1].start() if i + 1 < len(matches) else len(body)
            step_content = body[start:end].strip()
            
            # Extract action
            action_match = action_pattern.search(step_content)
            action = action_match.group(1).strip() if action_match else ""
            
            # Extract condition
            condition_match = condition_pattern.search(step_content)
            condition = condition_match.group(1).strip() if condition_match else None
            
            # Extract agent config
            agent_config = None
            agent_match = agent_pattern.search(step_content)
            if agent_match:
                agent_config = self._parse_agent_config(agent_match.group(1).strip())
            
            # Extract tools list
            tools = None
            tools_match = tools_pattern.search(step_content)
            if tools_match:
                tools_str = tools_match.group(1).strip()
                # Parse tools as list of strings (tool names)
                tools = [t.strip().strip('"\'') for t in tools_str.split('\n') if t.strip()]
            
            # Extract context_from
            context_from = None
            context_from_match = context_from_pattern.search(step_content)
            if context_from_match:
                context_from = [s.strip().strip('"\'') for s in context_from_match.group(1).split(',')]
            
            # Extract retain_full_context
            retain_full_context = True
            retain_match = retain_context_pattern.search(step_content)
            if retain_match:
                retain_full_context = retain_match.group(1).lower() == 'true'
            
            # Extract output_variable
            output_variable = None
            output_var_match = output_var_pattern.search(step_content)
            if output_var_match:
                output_variable = output_var_match.group(1)
            
            # Extract next_steps for branching
            next_steps = None
            next_steps_match = next_steps_pattern.search(step_content)
            if next_steps_match:
                next_steps = [s.strip().strip('"\'') for s in next_steps_match.group(1).split(',')]
            
            # Extract branch_condition
            branch_condition = None
            branch_match = branch_pattern.search(step_content)
            if branch_match:
                branch_condition = self._parse_branch_condition(branch_match.group(1).strip())
            
            # Extract loop_over
            loop_over = None
            loop_over_match = loop_over_pattern.search(step_content)
            if loop_over_match:
                loop_over = loop_over_match.group(1)
            
            # Extract loop_var
            loop_var = "item"
            loop_var_match = loop_var_pattern.search(step_content)
            if loop_var_match:
                loop_var = loop_var_match.group(1)
            
            # Extract route pattern (same as branch_condition)
            route_match = route_pattern.search(step_content)
            if route_match and not branch_condition:
                branch_condition = self._parse_branch_condition(route_match.group(1).strip())
            
            # Extract parallel pattern
            parallel_steps = None
            parallel_match = parallel_pattern.search(step_content)
            if parallel_match:
                parallel_str = parallel_match.group(1).strip()
                # Parse as list of step names
                parallel_steps = [s.strip().lstrip('- ').strip('"\'') for s in parallel_str.split('\n') if s.strip()]
                # Set next_steps to parallel steps if not already set
                if not next_steps:
                    next_steps = parallel_steps
            
            # Extract images
            images = None
            images_match = images_pattern.search(step_content)
            if images_match:
                images_str = images_match.group(1).strip()
                images = [s.strip() for s in images_str.split('\n') if s.strip()]
            
            # Extract output_file
            output_file = None
            output_file_match = output_file_pattern.search(step_content)
            if output_file_match:
                output_file = output_file_match.group(1).strip()
            
            # Extract repeat config
            repeat_config = None
            repeat_match = repeat_pattern.search(step_content)
            if repeat_match:
                repeat_config = self._parse_agent_config(repeat_match.group(1).strip())
            
            # Get description (text before action block)
            description = step_content
            if action_match:
                description = step_content[:action_match.start()].strip()
            
            # Clean up description (remove code blocks and config lines)
            description = re.sub(r'```.*?```', '', description, flags=re.DOTALL)
            description = re.sub(r'context_from:.*', '', description)
            description = re.sub(r'retain_full_context:.*', '', description)
            description = re.sub(r'output_variable:.*', '', description)
            description = re.sub(r'next_steps:.*', '', description)
            description = re.sub(r'loop_over:.*', '', description)
            description = re.sub(r'loop_var:.*', '', description)
            description = re.sub(r'output_file:.*', '', description)
            description = description.strip()
            
            # Set max_retries from repeat config
            max_retries = 3
            if repeat_config and 'max_iterations' in repeat_config:
                max_retries = int(repeat_config['max_iterations'])
            
            # Build consolidated context config if needed
            context_config = None
            if context_from or not retain_full_context:
                # Only create config if context_from is set OR retain_full_context is False (non-default)
                context_config = TaskContextConfig(
                    from_steps=context_from,
                    retain_full=retain_full_context
                )
            
            # Build consolidated output config if needed
            output_config = None
            if output_variable or output_file:
                output_config = TaskOutputConfig(
                    variable=output_variable,
                    file=output_file
                )
            
            # Build consolidated execution config if needed
            execution_config = None
            if max_retries != 3:  # Only if non-default
                execution_config = TaskExecutionConfig(
                    max_retries=max_retries
                )
            
            # Build consolidated routing config if needed
            routing_config = None
            if next_steps or branch_condition:
                routing_config = TaskRoutingConfig(
                    next_steps=next_steps,
                    branches=branch_condition
                )
            
            steps.append(Task(
                name=step_name,
                description=description,
                action=action,
                condition=condition,
                agent_config=agent_config,
                tools=tools,
                context=context_config,
                output=output_config,
                execution=execution_config,
                routing=routing_config,
                loop_over=loop_over,
                loop_var=loop_var,
                images=images
            ))
        
        return steps
    
    def _parse_branch_condition(self, branch_str: str) -> Dict[str, List[str]]:
        """Parse branch condition from a code block."""
        condition = {}
        for line in branch_str.split('\n'):
            line = line.strip()
            if ':' in line and not line.startswith('#'):
                key, value = line.split(':', 1)
                key = key.strip()
                value = value.strip()
                
                # Parse as list
                if value.startswith('[') and value.endswith(']'):
                    value = [s.strip().strip('"\'') for s in value[1:-1].split(',')]
                else:
                    value = [value.strip('"\'')]
                
                condition[key] = value
        return condition
    
    def _parse_agent_config(self, agent_str: str) -> Dict[str, Any]:
        """Parse agent configuration from a code block."""
        config = {}
        for line in agent_str.split('\n'):
            line = line.strip()
            if ':' in line and not line.startswith('#'):
                key, value = line.split(':', 1)
                key = key.strip()
                value = value.strip()
                
                # Handle basic types
                if value.lower() in ('true', 'false'):
                    value = value.lower() == 'true'
                elif value.isdigit():
                    value = int(value)
                elif value.startswith('"') and value.endswith('"'):
                    value = value[1:-1]
                elif value.startswith("'") and value.endswith("'"):
                    value = value[1:-1]
                
                config[key] = value
        return config
    
    def _load_workflow(self, file_path: Path) -> Optional[Workflow]:
        """Load a workflow from a file."""
        try:
            content = file_path.read_text(encoding="utf-8")
            frontmatter, body = self._parse_frontmatter(content)
            
            name = frontmatter.get("name", file_path.stem)
            description = frontmatter.get("description", "")
            variables = frontmatter.get("variables", {})
            
            # Parse variables if it's a string (simple YAML)
            if isinstance(variables, str):
                variables = {}
            
            # Parse workflow-level configuration (llm preferred, default_llm for backward compat)
            llm = frontmatter.get("llm") or frontmatter.get("default_llm")
            planning_enabled = frontmatter.get("planning", False)
            planning_llm = frontmatter.get("planning_llm")
            memory_config = frontmatter.get("memory_config")
            default_agent_config = frontmatter.get("default_agent_config")
            
            steps = self._parse_steps(body)
            
            # Build consolidated planning config if needed
            planning_config = None
            if planning_enabled or planning_llm:
                planning_config = WorkflowPlanningConfig(
                    enabled=planning_enabled,
                    llm=planning_llm
                )
            
            # Build consolidated memory config if needed
            memory_cfg = None
            if memory_config:
                if isinstance(memory_config, dict):
                    memory_cfg = WorkflowMemoryConfig(**memory_config)
                else:
                    memory_cfg = memory_config
            
            workflow = Workflow(
                name=name,
                description=description,
                steps=steps,
                variables=variables,
                file_path=str(file_path),
                llm=llm,
                planning=planning_config,
                memory=memory_cfg,
                default_agent_config=default_agent_config
            )
            
            self._log(f"Loaded workflow '{name}' with {len(steps)} steps")
            return workflow
            
        except Exception as e:
            self._log(f"Error loading workflow {file_path}: {e}", logging.WARNING)
            return None
    
    def _discover_workflows(self):
        """Discover all workflows in the workspace."""
        self._workflows = {}
        
        workflows_dir = self.workspace_path / self.WORKFLOWS_DIR.replace("/", os.sep)
        
        if not workflows_dir.exists():
            return
        
        for ext in self.SUPPORTED_EXTENSIONS:
            for file_path in workflows_dir.glob(f"*{ext}"):
                workflow = self._load_workflow(file_path)
                if workflow:
                    self._workflows[workflow.name.lower()] = workflow
        
        self._log(f"Discovered {len(self._workflows)} workflows")
    
    def reload(self):
        """Reload all workflows from disk."""
        self._loaded = False
        self._ensure_loaded()
    
    def list_workflows(self) -> List[Workflow]:
        """List all available workflows."""
        self._ensure_loaded()
        return list(self._workflows.values())
    
    def get_workflow(self, name: str) -> Optional[Workflow]:
        """Get a workflow by name."""
        self._ensure_loaded()
        return self._workflows.get(name.lower())
    
    def _substitute_variables(
        self,
        text: str,
        variables: Dict[str, Any]
    ) -> str:
        """
        Substitute {{variable}} placeholders in text.
        
        Delegates to shared substitute_variables() utility for DRY compliance.
        
        Resolution order:
        1. Dynamic variable providers ({{now}}, {{today}}, {{uuid}}, etc.)
        2. Static variables from workflow variables dict
        3. Keep original placeholder if not found
        """
        from praisonaiagents.utils.variables import substitute_variables
        return substitute_variables(text, variables)
    
    def _build_step_context(
        self,
        step: Task,
        step_index: int,
        results: List[Dict[str, Any]],
        variables: Dict[str, Any]
    ) -> str:
        """
        Build context for a step from previous step outputs.
        
        Args:
            step: Current workflow step
            step_index: Index of current step
            results: Results from previous steps
            variables: Current variables dict
            
        Returns:
            Context string to prepend to action
        """
        if step_index == 0 or not results:
            return ""
        
        context_parts = []
        
        # Use getattr with defaults for workflow-specific attributes that may not exist on Task
        context_from = getattr(step, 'context_from', None)
        retain_full_context = getattr(step, 'retain_full_context', False)
        
        if context_from:
            # Include only specified steps
            for step_name in context_from:
                for prev_result in results:
                    if prev_result["step"] == step_name and prev_result.get("output"):
                        context_parts.append(f"## {step_name} Output:\n{prev_result['output']}")
        elif retain_full_context:
            # Include all previous outputs
            for prev_result in results:
                if prev_result.get("output") and prev_result.get("status") == "success":
                    context_parts.append(f"## {prev_result['step']} Output:\n{prev_result['output']}")
        else:
            # Include only the last step's output
            last_result = results[-1]
            if last_result.get("output") and last_result.get("status") == "success":
                context_parts.append(f"## Previous Output:\n{last_result['output']}")
        
        if context_parts:
            return "# Context from previous steps:\n\n" + "\n\n".join(context_parts) + "\n\n"
        return ""
    
    def _update_variables_with_output(
        self,
        step: Task,
        output: str,
        variables: Dict[str, Any],
        results: List[Dict[str, Any]]
    ) -> None:
        """
        Update variables dict with step output for variable substitution.
        
        Args:
            step: Current workflow step
            output: Step output
            variables: Variables dict to update
            results: Results list for previous_output
        """
        # Store output in custom variable name if specified
        if step.output_variable:
            variables[step.output_variable] = output
        
        # Store output with step name (normalized)
        step_var_name = f"{step.name.lower().replace(' ', '_')}_output"
        variables[step_var_name] = output
        
        # Update previous_output for next step
        variables["previous_output"] = output
    
    def load_yaml(
        self,
        file_path: Union[str, Path],
        tool_registry: Optional[Dict[str, Callable]] = None
    ) -> "Workflow":
        """
        Load a workflow from a YAML file.
        
        Args:
            file_path: Path to the YAML workflow file
            tool_registry: Optional dictionary mapping tool names to callable functions
            
        Returns:
            Workflow object ready for execution
            
        Example:
            ```python
            manager = WorkflowManager()
            workflow = manager.load_yaml("research_workflow.yaml")
            result = workflow.start("Research AI trends")
            ```
        """
        from .yaml_parser import YAMLWorkflowParser
        
        parser = YAMLWorkflowParser(tool_registry=tool_registry)
        workflow = parser.parse_file(file_path)
        
        # Store in manager's workflow cache
        workflow_name = workflow.name.lower().replace(' ', '_')
        self._workflows[workflow_name] = workflow
        
        return workflow
    
    def execute_yaml(
        self,
        file_path: Union[str, Path],
        input_data: Optional[str] = None,
        variables: Optional[Dict[str, Any]] = None,
        tool_registry: Optional[Dict[str, Callable]] = None,
        verbose: bool = False
    ) -> Dict[str, Any]:
        """
        Load and execute a YAML workflow file.
        
        Args:
            file_path: Path to the YAML workflow file
            input_data: Initial input for the workflow
            variables: Additional variables to merge with workflow variables
            tool_registry: Optional dictionary mapping tool names to callable functions
            verbose: Enable verbose output
            
        Returns:
            Workflow execution results
            
        Example:
            ```python
            manager = WorkflowManager()
            result = manager.execute_yaml(
                "research_workflow.yaml",
                input_data="Research AI trends",
                variables={"topic": "Machine Learning"}
            )
            print(result["output"])
            ```
        """
        workflow = self.load_yaml(file_path, tool_registry=tool_registry)
        
        # Merge additional variables
        if variables:
            workflow.variables.update(variables)
        
        # Set verbose if specified
        if verbose:
            workflow.verbose = verbose
        
        # Execute the workflow
        return workflow.start(input_data or "")
    
    async def aexecute_yaml(
        self,
        file_path: Union[str, Path],
        input_data: Optional[str] = None,
        variables: Optional[Dict[str, Any]] = None,
        tool_registry: Optional[Dict[str, Callable]] = None,
        verbose: bool = False
    ) -> Dict[str, Any]:
        """
        Async version of execute_yaml.
        
        Args:
            file_path: Path to the YAML workflow file
            input_data: Initial input for the workflow
            variables: Additional variables to merge with workflow variables
            tool_registry: Optional dictionary mapping tool names to callable functions
            verbose: Enable verbose output
            
        Returns:
            Workflow execution results
        """
        workflow = self.load_yaml(file_path, tool_registry=tool_registry)
        
        # Merge additional variables
        if variables:
            workflow.variables.update(variables)
        
        # Set verbose if specified
        if verbose:
            workflow.verbose = verbose
        
        # Execute the workflow asynchronously
        return await workflow.astart(input_data or "")
    
    def execute(
        self,
        workflow_name: str,
        executor: Optional[Callable[[str], str]] = None,
        variables: Optional[Dict[str, Any]] = None,
        on_step: Optional[Callable[[Task, int], None]] = None,
        on_result: Optional[Callable[[Task, str], None]] = None,
        default_agent: Optional[Any] = None,
        default_llm: Optional[str] = None,
        memory: Optional[Any] = None,
        planning: bool = False,
        stream: bool = False,
        verbose: int = 0,
        checkpoint: Optional[str] = None,
        resume: Optional[str] = None
    ) -> Dict[str, Any]:
        """
        Execute a workflow with context passing between steps.
        
        Args:
            workflow_name: Name of the workflow to execute
            executor: Function to execute each step (e.g., agent.chat). Optional if default_agent provided.
            variables: Variables to substitute in steps
            on_step: Callback before each step (step, index)
            on_result: Callback after each step (step, result)
            default_agent: Default agent to use for steps without agent_config
            default_llm: Default LLM model for agent creation
            memory: Shared memory instance
            planning: Enable planning mode
            stream: Enable streaming output
            verbose: Verbosity level
            checkpoint: Save checkpoint after each step with this name
            resume: Resume from checkpoint with this name
            
        Returns:
            Execution results with step outputs and status
        """
        workflow = self.get_workflow(workflow_name)
        if not workflow:
            return {
                "success": False,
                "error": f"Workflow '{workflow_name}' not found",
                "results": []
            }
        
        # Merge variables
        all_variables = {**workflow.variables}
        if variables:
            all_variables.update(variables)
        
        # Use workflow-level defaults if not provided
        if default_llm is None:
            default_llm = workflow.llm
        if planning is False and workflow.planning:
            planning = workflow.planning
        
        results = []
        success = True
        start_step = 0
        
        # Resume from checkpoint if specified
        if resume:
            checkpoint_data = self._load_checkpoint(resume)
            if checkpoint_data:
                results = checkpoint_data.get("results", [])
                all_variables = checkpoint_data.get("variables", all_variables)
                start_step = checkpoint_data.get("completed_steps", 0)
                self._log(f"Resuming workflow from step {start_step + 1}")
        
        # Build step lookup for branching
        step_lookup = {step.name: (i, step) for i, step in enumerate(workflow.steps)}
        
        current_step_idx = start_step
        max_iterations = len(workflow.steps) * 10  # Prevent infinite loops
        iteration = 0
        
        while current_step_idx < len(workflow.steps) and iteration < max_iterations:
            iteration += 1
            step = workflow.steps[current_step_idx]
            i = current_step_idx
            
            # Check condition
            if step.condition:
                condition = self._substitute_variables(step.condition, all_variables)
                # Simple condition evaluation (could be enhanced)
                if condition.lower() in ("false", "no", "skip", "0"):
                    results.append({
                        "step": step.name,
                        "status": "skipped",
                        "output": None
                    })
                    current_step_idx += 1
                    continue
            
            # Handle loop_over - iterate over a variable
            if step.loop_over and step.loop_over in all_variables:
                loop_items = all_variables[step.loop_over]
                if isinstance(loop_items, (list, tuple)):
                    loop_results = []
                    for item_idx, item in enumerate(loop_items):
                        # Set loop variable
                        all_variables[step.loop_var] = item
                        all_variables["_loop_index"] = item_idx
                        
                        # Execute step for this item
                        step_result = self._execute_single_step(
                            step=step,
                            step_idx=i,
                            results=results,
                            all_variables=all_variables,
                            executor=executor,
                            default_agent=default_agent,
                            default_llm=default_llm,
                            memory=memory,
                            planning=planning,
                            # verbose parameter removed - Agent no longer accepts it
                            on_step=on_step,
                            on_result=on_result
                        )
                        loop_results.append(step_result)
                        
                        if not step_result["success"] and step.on_error == "stop":
                            success = False
                            break
                    
                    # Store all loop results
                    results.append({
                        "step": step.name,
                        "status": "success" if all(r["success"] for r in loop_results) else "partial",
                        "output": [r["output"] for r in loop_results],
                        "loop_results": loop_results
                    })
                    
                    # Clean up loop variables
                    all_variables.pop(step.loop_var, None)
                    all_variables.pop("_loop_index", None)
                else:
                    self._log(f"loop_over variable '{step.loop_over}' is not iterable")
                
                current_step_idx += 1
                continue
            
            # Callback before step
            if on_step:
                on_step(step, i)
            
            # Execute single step
            step_result = self._execute_single_step(
                step=step,
                step_idx=i,
                results=results,
                all_variables=all_variables,
                executor=executor,
                default_agent=default_agent,
                default_llm=default_llm,
                memory=memory,
                planning=planning,
                # verbose parameter removed - Agent no longer accepts it
                on_step=None,  # Already called above
                on_result=on_result
            )
            
            # Handle skipped steps
            if step_result.get("skipped"):
                results.append({
                    "step": step.name,
                    "status": "skipped",
                    "output": None
                })
                current_step_idx += 1
                continue
            
            results.append({
                "step": step.name,
                "status": "success" if step_result["success"] else "failed",
                "output": step_result["output"],
                "error": step_result.get("error")
            })
            
            # Handle early stop 
            if step_result.get("stop"):
                self._log(f"Workflow stopped early at step '{step.name}'")
                break
            
            # Save checkpoint after each step if enabled
            if checkpoint:
                self._save_checkpoint(
                    name=checkpoint,
                    workflow_name=workflow_name,
                    completed_steps=i + 1,
                    results=results,
                    variables=all_variables
                )
            
            # Handle failure
            if not step_result["success"]:
                if step.on_error == "stop":
                    success = False
                    break
                elif step.on_error == "continue":
                    current_step_idx += 1
                    continue
            
            # Handle branching
            next_step_idx = None
            # Use getattr with defaults for workflow-specific attributes that may not exist on Task
            branch_condition = getattr(step, 'branch_condition', None)
            next_steps = getattr(step, 'next_steps', None)
            if branch_condition and step_result["output"]:
                # Evaluate branch condition based on output
                output_lower = str(step_result["output"]).lower()
                for branch_key, branch_targets in branch_condition.items():
                    if branch_key.lower() in output_lower or output_lower.startswith(branch_key.lower()):
                        if branch_targets and branch_targets[0] in step_lookup:
                            next_step_idx, _ = step_lookup[branch_targets[0]]
                            break
            elif next_steps:
                # Use explicit next_steps
                if next_steps[0] in step_lookup:
                    next_step_idx, _ = step_lookup[next_steps[0]]
            
            # Move to next step
            if next_step_idx is not None:
                current_step_idx = next_step_idx
            else:
                current_step_idx += 1
        
        return {
            "success": success,
            "workflow": workflow.name,
            "results": results,
            "variables": all_variables
        }
    
    def _execute_single_step(
        self,
        step: Task,
        step_idx: int,
        results: List[Dict[str, Any]],
        all_variables: Dict[str, Any],
        executor: Optional[Callable[[str], str]] = None,
        default_agent: Optional[Any] = None,
        default_llm: Optional[str] = None,
        memory: Optional[Any] = None,
        planning: bool = False,
        verbose: int = 0,
        on_step: Optional[Callable[[Task, int], None]] = None,
        on_result: Optional[Callable[[Task, str], None]] = None,
        original_input: str = ""
    ) -> Dict[str, Any]:
        """Execute a single workflow step."""
        # Get previous step output
        previous_output = results[-1].get("output") if results else None
        
        # Create context for step handlers
        context = WorkflowContext(
            input=original_input,
            previous_result=str(previous_output) if previous_output else None,
            current_step=step.name,
            variables=all_variables.copy()
        )
        
        # Check should_run condition if provided
        if step.should_run:
            try:
                if not step.should_run(context):
                    return {
                        "success": True,
                        "output": None,
                        "skipped": True,
                        "stop": False
                    }
            except Exception as e:
                self._log(f"should_run check for step '{step.name}' failed: {e}")
        
        # If step has a custom handler function
        if step.handler:
            try:
                result = step.handler(context)
                # Handle StepResult
                if isinstance(result, StepResult):
                    # Update variables from result
                    if result.variables:
                        all_variables.update(result.variables)
                    return {
                        "success": True,
                        "output": result.output,
                        "stop": result.stop_workflow,  # Early termination flag
                        "error": None
                    }
                else:
                    return {
                        "success": True,
                        "output": str(result),
                        "stop": False,
                        "error": None
                    }
            except Exception as e:
                return {
                    "success": False,
                    "output": None,
                    "stop": False,
                    "error": str(e)
                }
        
        # Build context from previous steps
        context = self._build_step_context(step, step_idx, results, all_variables)
        
        # Substitute variables in action
        action = self._substitute_variables(step.action, all_variables)
        
        # Prepend context to action if available
        if context:
            action = f"{context}# Current Task:\n{action}"
        
        # Get or create executor for this step
        step_executor = executor
        if step_executor is None:
            # Use step's direct agent if provided
            step_agent = step.agent
            
            # Otherwise create agent from config
            if step_agent is None:
                step_agent = self._create_step_agent(
                    step=step,
                    default_agent=default_agent,
                    default_llm=default_llm,
                    memory=memory,
                    planning=planning,
                    verbose=verbose
                )
            
            if step_agent:
                if planning and hasattr(step_agent, 'start'):
                    def agent_executor(prompt, agent=step_agent):
                        return agent.start(prompt)
                    step_executor = agent_executor
                else:
                    def agent_executor(prompt, agent=step_agent):
                        return agent.chat(prompt)
                    step_executor = agent_executor
        
        if step_executor is None:
            return {
                "success": False,
                "output": None,
                "stop": False,
                "error": f"No executor available for step '{step.name}'"
            }
        
        # Execute step with retries
        retries = 0
        step_success = False
        output = None
        error = None
        
        while retries <= step.max_retries and not step_success:
            try:
                output = step_executor(action)
                step_success = True
            except Exception as e:
                error = str(e)
                retries += 1
                if retries <= step.max_retries:
                    self._log(f"Step '{step.name}' failed, retrying ({retries}/{step.max_retries})")
        
        # Update variables with step output
        if output and step_success:
            self._update_variables_with_output(step, output, all_variables, results)
        
        # Callback after step
        if on_result and output:
            on_result(step, output)
        
        return {
            "success": step_success,
            "output": output,
            "stop": False,  # Normal execution doesn't request stop
            "error": error
        }
    
    async def aexecute(
        self,
        workflow_name: str,
        executor: Optional[Callable[[str], str]] = None,
        variables: Optional[Dict[str, Any]] = None,
        on_step: Optional[Callable[[Task, int], None]] = None,
        on_result: Optional[Callable[[Task, str], None]] = None,
        default_agent: Optional[Any] = None,
        default_llm: Optional[str] = None,
        memory: Optional[Any] = None,
        planning: bool = False,
        stream: bool = False,
        verbose: int = 0
    ) -> Dict[str, Any]:
        """
        Async version of execute() for workflow execution.
        
        Args:
            workflow_name: Name of the workflow to execute
            executor: Async function to execute each step. Optional if default_agent provided.
            variables: Variables to substitute in steps
            on_step: Callback before each step (step, index)
            on_result: Callback after each step (step, result)
            default_agent: Default agent to use for steps without agent_config
            default_llm: Default LLM model for agent creation
            memory: Shared memory instance
            planning: Enable planning mode
            stream: Enable streaming output
            verbose: Verbosity level
            
        Returns:
            Execution results with step outputs and status
        """
        import asyncio
        
        workflow = self.get_workflow(workflow_name)
        if not workflow:
            return {
                "success": False,
                "error": f"Workflow '{workflow_name}' not found",
                "results": []
            }
        
        # Merge variables
        all_variables = {**workflow.variables}
        if variables:
            all_variables.update(variables)
        
        # Use workflow-level defaults if not provided
        if default_llm is None:
            default_llm = workflow.llm
        if planning is False and workflow.planning:
            planning = workflow.planning
        
        results = []
        success = True
        
        for i, step in enumerate(workflow.steps):
            # Check condition
            if step.condition:
                condition = self._substitute_variables(step.condition, all_variables)
                if condition.lower() in ("false", "no", "skip", "0"):
                    results.append({
                        "step": step.name,
                        "status": "skipped",
                        "output": None
                    })
                    continue
            
            # Callback before step
            if on_step:
                on_step(step, i)
            
            # Build context from previous steps
            context = self._build_step_context(step, i, results, all_variables)
            
            # Substitute variables in action
            action = self._substitute_variables(step.action, all_variables)
            
            # Prepend context to action if available
            if context:
                action = f"{context}# Current Task:\n{action}"
            
            # Get or create executor for this step
            step_executor = executor
            if step_executor is None:
                # Create agent for this step
                step_agent = self._create_step_agent(
                    step=step,
                    default_agent=default_agent,
                    default_llm=default_llm,
                    memory=memory,
                    verbose=verbose
                )
                if step_agent:
                    # Check if agent has async chat method (and it's actually async)
                    if hasattr(step_agent, 'achat') and asyncio.iscoroutinefunction(getattr(step_agent, 'achat', None)):
                        async def async_agent_executor(prompt, agent=step_agent):
                            return await agent.achat(prompt)
                        step_executor = async_agent_executor
                    else:
                        def sync_agent_executor(prompt, agent=step_agent):
                            return agent.chat(prompt)
                        step_executor = sync_agent_executor
            
            if step_executor is None:
                return {
                    "success": False,
                    "error": f"No executor available for step '{step.name}'. Provide executor or default_agent.",
                    "results": results
                }
            
            # Execute step
            retries = 0
            step_success = False
            output = None
            error = None
            
            while retries <= step.max_retries and not step_success:
                try:
                    # Check if executor is async
                    if asyncio.iscoroutinefunction(step_executor):
                        output = await step_executor(action)
                    else:
                        output = step_executor(action)
                    step_success = True
                except Exception as e:
                    error = str(e)
                    retries += 1
                    if retries <= step.max_retries:
                        self._log(f"Step '{step.name}' failed, retrying ({retries}/{step.max_retries})")
            
            # Update variables with step output for next steps
            if output and step_success:
                self._update_variables_with_output(step, output, all_variables, results)
            
            # Callback after step
            if on_result and output:
                on_result(step, output)
            
            results.append({
                "step": step.name,
                "status": "success" if step_success else "failed",
                "output": output,
                "error": error
            })
            
            # Handle failure
            if not step_success:
                if step.on_error == "stop":
                    success = False
                    break
                elif step.on_error == "continue":
                    continue
        
        return {
            "success": success,
            "workflow": workflow.name,
            "results": results,
            "variables": all_variables
        }
    
    def _create_step_agent(
        self,
        step: Task,
        default_agent: Optional[Any] = None,
        default_llm: Optional[str] = None,
        memory: Optional[Any] = None,
        planning: bool = False,
        verbose: int = 0
    ) -> Optional[Any]:
        """
        Create an agent for a specific step.
        
        Args:
            step: Workflow step with optional agent_config
            default_agent: Default agent to use if no config
            default_llm: Default LLM model
            memory: Shared memory instance
            planning: Enable planning mode for the agent
            verbose: Verbosity level
            
        Returns:
            Agent instance or None
        """
        if step.agent_config:
            try:
                from ..agent.agent import Agent
                
                config = step.agent_config.copy()
                # Remove verbose as Agent no longer accepts it
                config.pop("verbose", None)
                config.setdefault("name", f"{step.name}Agent")
                config.setdefault("llm", default_llm)
                # config.setdefault("verbose", verbose)  # Agent no longer accepts verbose
                config.setdefault("planning", planning)
                
                if step.tools:
                    config["tools"] = step.tools
                if memory:
                    config["memory"] = memory
                
                return Agent(**config)
            except Exception as e:
                self._log(f"Failed to create agent for step '{step.name}': {e}", logging.ERROR)
                return default_agent
        
        return default_agent
    
    def create_workflow(
        self,
        name: str,
        description: str = "",
        steps: Optional[List[Dict[str, str]]] = None,
        variables: Optional[Dict[str, Any]] = None
    ) -> Workflow:
        """
        Create a new workflow file.
        
        Args:
            name: Workflow name
            description: Workflow description
            steps: List of step dicts with name, description, action
            variables: Default variables
            
        Returns:
            Created Workflow object
        """
        workflows_dir = self.workspace_path / self.WORKFLOWS_DIR.replace("/", os.sep)
        workflows_dir.mkdir(parents=True, exist_ok=True)
        
        file_path = workflows_dir / f"{name.lower().replace(' ', '_')}.md"
        
        # Build content
        lines = ["---"]
        lines.append(f"name: {name}")
        if description:
            lines.append(f"description: {description}")
        if variables:
            lines.append("variables:")
            for k, v in variables.items():
                lines.append(f"  {k}: {v}")
        lines.append("---")
        lines.append("")
        
        # Add steps
        workflow_steps = []
        if steps:
            for i, step in enumerate(steps, 1):
                step_name = step.get("name", f"Step {i}")
                step_desc = step.get("description", "")
                step_action = step.get("action", "")
                
                lines.append(f"## Step {i}: {step_name}")
                if step_desc:
                    lines.append(step_desc)
                lines.append("")
                lines.append("```action")
                lines.append(step_action)
                lines.append("```")
                lines.append("")
                
                workflow_steps.append(Task(
                    name=step_name,
                    description=step_desc,
                    action=step_action
                ))
        
        # Write file
        file_path.write_text("\n".join(lines), encoding="utf-8")
        
        # Create and register workflow
        workflow = Workflow(
            name=name,
            description=description,
            steps=workflow_steps,
            variables=variables or {},
            file_path=str(file_path)
        )
        
        self._workflows[name.lower()] = workflow
        self._log(f"Created workflow '{name}' at {file_path}")
        
        return workflow
    
    def get_stats(self) -> Dict[str, Any]:
        """Get workflow statistics."""
        self._ensure_loaded()
        
        total_steps = sum(len(w.steps) for w in self._workflows.values())
        
        return {
            "total_workflows": len(self._workflows),
            "total_steps": total_steps,
            "workflows": [w.name for w in self._workflows.values()]
        }
    
    # -------------------------------------------------------------------------
    #                          Checkpoint Methods
    # -------------------------------------------------------------------------
    
    def _get_checkpoints_dir(self) -> Path:
        """Get the checkpoints directory path."""
        from ..paths import DEFAULT_DIR_NAME
        checkpoints_dir = self.workspace_path / DEFAULT_DIR_NAME / "checkpoints"
        checkpoints_dir.mkdir(parents=True, exist_ok=True)
        return checkpoints_dir
    
    def _save_checkpoint(
        self,
        name: str,
        workflow_name: str,
        completed_steps: int,
        results: List[Dict[str, Any]],
        variables: Dict[str, Any]
    ) -> str:
        """
        Save workflow checkpoint for later resumption.
        
        Args:
            name: Checkpoint name
            workflow_name: Name of the workflow
            completed_steps: Number of completed steps
            results: Results from completed steps
            variables: Current variable state
            
        Returns:
            Path to checkpoint file
        """
        import json
        import time
        from datetime import datetime
        
        checkpoint_file = self._get_checkpoints_dir() / f"{name}.json"
        
        checkpoint_data = {
            "name": name,
            "workflow_name": workflow_name,
            "completed_steps": completed_steps,
            "results": results,
            "variables": variables,
            "saved_at": time.time(),
            "saved_at_iso": datetime.now().isoformat()
        }
        
        with open(checkpoint_file, 'w') as f:
            json.dump(checkpoint_data, f, indent=2, default=str)
        
        self._log(f"Saved checkpoint '{name}' at step {completed_steps}")
        return str(checkpoint_file)
    
    def _load_checkpoint(self, name: str) -> Optional[Dict[str, Any]]:
        """
        Load a workflow checkpoint.
        
        Args:
            name: Checkpoint name
            
        Returns:
            Checkpoint data or None if not found
        """
        import json
        
        checkpoint_file = self._get_checkpoints_dir() / f"{name}.json"
        
        if not checkpoint_file.exists():
            self._log(f"Checkpoint '{name}' not found")
            return None
        
        with open(checkpoint_file, 'r') as f:
            return json.load(f)
    
    def list_checkpoints(self) -> List[Dict[str, Any]]:
        """
        List all saved checkpoints.
        
        Returns:
            List of checkpoint info dicts
        """
        import json
        
        checkpoints = []
        checkpoints_dir = self._get_checkpoints_dir()
        
        for checkpoint_file in checkpoints_dir.glob("*.json"):
            try:
                with open(checkpoint_file, 'r') as f:
                    data = json.load(f)
                checkpoints.append({
                    "name": data.get("name", checkpoint_file.stem),
                    "workflow": data.get("workflow_name", ""),
                    "completed_steps": data.get("completed_steps", 0),
                    "saved_at": data.get("saved_at_iso", "")
                })
            except Exception:
                continue
        
        checkpoints.sort(key=lambda x: x.get("saved_at", ""), reverse=True)
        return checkpoints
    
    def delete_checkpoint(self, name: str) -> bool:
        """
        Delete a checkpoint.
        
        Args:
            name: Checkpoint name
            
        Returns:
            True if deleted successfully
        """
        checkpoint_file = self._get_checkpoints_dir() / f"{name}.json"
        
        if checkpoint_file.exists():
            checkpoint_file.unlink()
            self._log(f"Deleted checkpoint '{name}'")
            return True
        return False

def create_workflow_manager(
    workspace_path: Optional[str] = None,
    **kwargs
) -> WorkflowManager:
    """
    Create a WorkflowManager instance.
    
    Args:
        workspace_path: Path to workspace
        **kwargs: Additional configuration
        
    Returns:
        WorkflowManager instance
    """
    return WorkflowManager(workspace_path=workspace_path, **kwargs)
