"""Simplified YAML Pipeline Builder with Plugin Architecture.

Philosophy:
- Core builder: Minimal YAML → DirectedGraph parser
- Preprocessing plugins: Environment variables, templating, validation
- Entity plugins: Macros, nodes (each entity type handles its own YAML)

Plugins provide clear value:
- EnvironmentVariablePlugin: Resolve ${VAR} and ${VAR:default}
- TemplatePlugin: Jinja2 templating in YAML values
- SchemaValidationPlugin: Validate schemas before execution
"""

import os
import re
from contextlib import contextmanager, suppress
from functools import lru_cache, singledispatch
from pathlib import Path
from typing import TYPE_CHECKING, Any, Protocol, TypeGuard, cast

import yaml
from jinja2 import TemplateSyntaxError, UndefinedError
from jinja2.sandbox import SandboxedEnvironment

from hexdag.core.configurable import ConfigurableMacro
from hexdag.core.domain.dag import DirectedGraph, NodeSpec
from hexdag.core.logging import get_logger
from hexdag.core.pipeline_builder.pipeline_config import PipelineConfig
from hexdag.core.pipeline_builder.yaml_validator import YamlValidator
from hexdag.core.resolver import ResolveError, register_runtime, resolve

if TYPE_CHECKING:
    from collections.abc import Callable

logger = get_logger(__name__)


class YamlPipelineBuilderError(Exception):
    """YAML pipeline building errors."""

    pass


# ============================================================================
# Type Guards
# ============================================================================


def _is_dict_config(value: Any) -> TypeGuard[dict[str, Any]]:
    """Type guard to verify value is a dictionary."""
    return isinstance(value, dict)


# ============================================================================
# Plugin Protocol
# ============================================================================


class PreprocessPlugin(Protocol):
    """Plugin for preprocessing YAML before building (env vars, templating, etc.)."""

    def process(self, config: dict[str, Any]) -> dict[str, Any]:
        """Process entire config, returning modified version."""
        ...


class EntityPlugin(Protocol):
    """Plugin for building specific entity types (macros, nodes, etc.)."""

    def can_handle(self, node_config: dict[str, Any]) -> bool:
        """Return True if this plugin can handle the node config."""
        ...

    def build(
        self, node_config: dict[str, Any], builder: "YamlPipelineBuilder", graph: DirectedGraph
    ) -> NodeSpec | None:
        """Build entity from config.

        Args
        -----
            node_config: The node configuration dictionary.
            builder: The YamlPipelineBuilder instance.
            graph: The DirectedGraph instance.

        Return NodeSpec or None if handled (e.g., macro merged into graph)."""
        ...


# ============================================================================
# Core Builder
# ============================================================================


class YamlPipelineBuilder:
    """YAML → DirectedGraph builder with plugin support.

    Workflow:
    1. Parse YAML
    2. Select environment
    3. Validate structure
    4. Run preprocessing plugins (env vars, templates)
    5. Build graph using entity plugins
    6. Extract pipeline config
    """

    def __init__(self, base_path: Path | None = None) -> None:
        """Initialize builder.

        Args:
            base_path: Base directory for resolving includes (default: cwd)
        """
        self.base_path = base_path or Path.cwd()
        self.validator = YamlValidator()

        # Plugins
        self.preprocess_plugins: list[PreprocessPlugin] = []
        self.entity_plugins: list[EntityPlugin] = []

        self._register_default_plugins()

    def _register_default_plugins(self) -> None:
        """Register default plugins for common use cases."""
        # Preprocessing plugins (run before building)
        self.preprocess_plugins.append(IncludePreprocessPlugin(base_path=self.base_path))
        self.preprocess_plugins.append(EnvironmentVariablePlugin())
        self.preprocess_plugins.append(TemplatePlugin())

        # Entity plugins (build specific entity types)
        self.entity_plugins.append(MacroDefinitionPlugin())  # Process Macro definitions first
        self.entity_plugins.append(MacroEntityPlugin())  # Then macro invocations
        self.entity_plugins.append(NodeEntityPlugin(self))  # Finally regular nodes

    @contextmanager
    def _temporary_base_path(self, new_base: Path) -> Any:
        """Context manager for temporarily changing base_path.

        Args:
            new_base: Temporary base path to use

        Yields:
            None
        """
        original_base = self.base_path
        self.base_path = new_base

        # Update include plugin base paths
        for plugin in self.preprocess_plugins:
            if isinstance(plugin, IncludePreprocessPlugin):
                plugin.base_path = new_base

        try:
            yield
        finally:
            # Always restore original state
            self.base_path = original_base
            for plugin in self.preprocess_plugins:
                if isinstance(plugin, IncludePreprocessPlugin):
                    plugin.base_path = original_base

    # --- Public API ---

    def build_from_yaml_file(
        self, yaml_path: str, use_cache: bool = True
    ) -> tuple[DirectedGraph, PipelineConfig]:
        """Build from YAML file.

        Args:
            yaml_path: Path to YAML file
            use_cache: Whether to use cached YAML parsing

        Returns:
            Tuple of (DirectedGraph, PipelineConfig)
        """
        yaml_file = Path(yaml_path)
        yaml_content = yaml_file.read_text(encoding="utf-8")

        # Use context manager to temporarily change base_path for relative includes
        with self._temporary_base_path(yaml_file.parent):
            return self.build_from_yaml_string(yaml_content, use_cache=use_cache)

    def build_from_yaml_string(
        self, yaml_content: str, use_cache: bool = True, environment: str | None = None
    ) -> tuple[DirectedGraph, PipelineConfig]:
        """Build DirectedGraph + PipelineConfig from YAML string."""
        # Step 1: Parse YAML
        documents = self._parse_yaml(yaml_content, use_cache=use_cache)

        # Step 2: Process ALL documents for macro definitions first
        #         This allows macros to be defined in multi-document YAML
        #         before the pipeline that uses them
        for doc in documents:
            if isinstance(doc, dict) and doc.get("kind") == "Macro":
                # Process includes for macro definitions (but skip template rendering)
                # Templates in macros should be preserved for expansion-time rendering
                processed_doc = doc
                for plugin in self.preprocess_plugins:
                    # Skip TemplatePlugin for Macro definitions - templates should be preserved
                    if not isinstance(plugin, TemplatePlugin):
                        processed_doc = plugin.process(processed_doc)
                # Validate and process macro definition
                processed_doc = self._validate_config(processed_doc)
                # Register macro (MacroDefinitionPlugin handles this)
                self._process_macro_definitions([processed_doc])

        # Step 3: Select pipeline environment
        config = self._select_environment(documents, environment)

        # Skip if selected document is a Macro (already processed above)
        if config.get("kind") == "Macro":
            # Return empty graph for macro-only documents
            logger.info("Document contains only macro definitions, no pipeline to build")
            return DirectedGraph(), PipelineConfig(
                ports={}, type_ports={}, policies={}, metadata={}, nodes=[]
            )

        # Step 4: Preprocess FIRST (includes must resolve before validation)
        for plugin in self.preprocess_plugins:
            config = plugin.process(config)

        # Step 4.5: Register user-defined aliases BEFORE validation
        # so that custom node kinds can pass validation
        self._register_aliases(config)

        # Step 5: Validate structure (after includes are resolved)
        config = self._validate_config(config)

        # Step 6: Build graph
        graph = self._build_graph(config)

        # Step 7: Extract pipeline config
        pipeline_config = self._extract_pipeline_config(config)

        logger.info(
            "✅ Built pipeline '{name}' with {nodes} nodes, {ports} ports, {policies} policies",
            name=pipeline_config.metadata.get("name", "unknown"),
            nodes=len(graph.nodes),
            ports=len(pipeline_config.ports),
            policies=len(pipeline_config.policies),
        )

        return graph, pipeline_config

    # --- Core Logic ---

    def _parse_yaml(self, yaml_content: str, use_cache: bool) -> list[dict[str, Any]]:
        """Parse YAML into list of documents."""
        if "---" in yaml_content:
            return list(yaml.safe_load_all(yaml_content))
        parsed = _parse_yaml_cached(yaml_content) if use_cache else yaml.safe_load(yaml_content)
        return [parsed]

    def _select_environment(
        self, documents: list[dict[str, Any]], environment: str | None
    ) -> dict[str, Any]:
        """Select document by environment name.

        Skips Macro definitions when selecting the Pipeline document.
        """
        # Filter out Macro definitions - they're processed separately
        pipeline_docs = [doc for doc in documents if doc.get("kind") != "Macro"]

        if not pipeline_docs:
            # No pipeline documents, return first macro (for macro-only files)
            return documents[0]

        if environment:
            for doc in pipeline_docs:
                if doc.get("metadata", {}).get("namespace") == environment:
                    logger.info(f"Selected environment '{environment}' from multi-document YAML")
                    return doc

            available_envs = [
                doc.get("metadata", {}).get("namespace", "default") for doc in pipeline_docs
            ]
            raise YamlPipelineBuilderError(
                f"Environment '{environment}' not found in YAML. "
                f"Available environments: {', '.join(available_envs)}"
            )

        if len(pipeline_docs) > 1:
            logger.warning(
                f"Multi-document YAML detected ({len(pipeline_docs)} pipeline documents) "
                "but no environment specified. "
                "Using first pipeline. Specify environment parameter to select specific config."
            )

        return pipeline_docs[0]

    def _validate_config(self, config: Any) -> dict[str, Any]:
        """Validate YAML structure."""
        if not isinstance(config, dict):
            raise YamlPipelineBuilderError(
                f"YAML document must be a dictionary, got {type(config).__name__}"
            )

        self._validate_manifest_format(config)

        result = self.validator.validate(config)
        if not result.is_valid:
            errors = "\n".join(f"  ERROR: {error}" for error in result.errors)
            raise YamlPipelineBuilderError(f"YAML validation failed:\n{errors}")

        for warning in result.warnings:
            logger.warning(f"YAML validation warning: {warning}")

        return config

    @staticmethod
    def _validate_manifest_format(config: dict[str, Any]) -> None:
        """Validate declarative manifest format."""
        if "kind" not in config:
            raise YamlPipelineBuilderError(
                "YAML must use declarative manifest format with 'kind' field. "
                "Example:\n"
                "apiVersion: v1\n"
                "kind: Pipeline\n"
                "metadata:\n"
                "  name: my-pipeline\n"
                "spec:\n"
                "  nodes: [...]"
            )

        # Macro definitions have different structure (no spec field)
        kind = config.get("kind")
        if kind == "Macro":
            # Macro has: metadata, parameters, nodes (no spec)
            if "metadata" not in config:
                raise YamlPipelineBuilderError("Macro definition must have 'metadata' field")
            if "nodes" not in config:
                raise YamlPipelineBuilderError("Macro definition must have 'nodes' field")
        else:
            # Pipeline and other kinds require spec
            if "spec" not in config:
                raise YamlPipelineBuilderError("Manifest YAML must have 'spec' field")

            if "metadata" not in config:
                raise YamlPipelineBuilderError("Manifest YAML must have 'metadata' field")

    def _process_macro_definitions(self, macro_configs: list[dict[str, Any]]) -> None:
        """Process macro definitions and register them.

        Parameters
        ----------
        macro_configs : list[dict[str, Any]]
            List of validated macro configuration dictionaries
        """
        # Create temporary graph (not used for macros, but required by plugin interface)
        temp_graph = DirectedGraph()

        # Find MacroDefinitionPlugin
        macro_plugin = next(
            (p for p in self.entity_plugins if isinstance(p, MacroDefinitionPlugin)), None
        )

        if macro_plugin is None:
            raise YamlPipelineBuilderError(
                "MacroDefinitionPlugin not found. Cannot process macro definitions."
            )

        for macro_config in macro_configs:
            # Wrap in a fake node config format expected by entity plugin
            node_config = macro_config
            macro_plugin.build(node_config, self, temp_graph)

    def _register_aliases(self, config: dict[str, Any]) -> None:
        """Register user-defined aliases from spec.aliases before validation.

        This allows custom node kinds to be used in YAML without requiring
        full module paths.

        Parameters
        ----------
        config : dict[str, Any]
            The YAML configuration dict
        """
        from hexdag.core.resolver import register_alias

        spec = config.get("spec", {})
        aliases = spec.get("aliases", {})

        for alias, full_path in aliases.items():
            register_alias(alias, full_path)
            logger.debug(f"Registered alias: {alias} -> {full_path}")

    def _build_graph(self, config: dict[str, Any]) -> DirectedGraph:
        """Build DirectedGraph using entity plugins."""
        graph = DirectedGraph()
        spec = config.get("spec", {})
        nodes_list = spec.get("nodes", [])

        for node_config in nodes_list:
            # Find plugin that can handle this entity
            for plugin in self.entity_plugins:
                if plugin.can_handle(node_config):
                    result = plugin.build(node_config, self, graph)
                    if result is not None:
                        graph += result
                    break
            else:
                # No plugin handled it - error
                kind = node_config.get("kind", "unknown")
                raise YamlPipelineBuilderError(f"No plugin can handle kind: {kind}")

        return graph

    @staticmethod
    def _extract_pipeline_config(config: dict[str, Any]) -> PipelineConfig:
        """Extract PipelineConfig from YAML."""
        spec = config.get("spec", {})
        metadata = config.get("metadata", {})

        return PipelineConfig(
            ports=spec.get("ports", {}),
            type_ports=spec.get("type_ports", {}),
            policies=spec.get("policies", {}),
            metadata=metadata,
            nodes=spec.get("nodes", []),
        )


# ============================================================================
# Entity Plugins - Each handles one entity type
# ============================================================================


class MacroDefinitionPlugin:
    """Plugin for handling Macro definitions (kind: Macro).

    This plugin processes YAML macro definitions and registers them in the
    component registry for later invocation. Macro definitions don't add
    nodes to the graph - they just register reusable templates.

    Examples
    --------
    YAML macro definition::

        apiVersion: hexdag/v1
        kind: Macro
        metadata:
          name: retry_workflow
          description: Retry logic with exponential backoff
        parameters:
          - name: max_retries
            type: int
            default: 3
        nodes:
          - kind: function_node
            metadata:
              name: "{{name}}_attempt"
            spec:
              fn: "{{fn}}"
    """

    def can_handle(self, node_config: dict[str, Any]) -> bool:
        """Handle Macro kind."""
        return node_config.get("kind") == "Macro"

    def build(
        self, node_config: dict[str, Any], builder: YamlPipelineBuilder, graph: DirectedGraph
    ) -> NodeSpec | None:
        """Register YAML macro in component registry.

        Parameters
        ----------
        node_config : dict[str, Any]
            Macro definition configuration
        builder : YamlPipelineBuilder
            Builder instance (unused)
        graph : DirectedGraph
            Graph instance (unused - definitions don't add nodes)

        Returns
        -------
        None
            Macro definitions don't add nodes to the graph

        Raises
        ------
        YamlPipelineBuilderError
            If macro definition is invalid
        """
        # Import here to avoid circular dependency
        from hexdag.core.yaml_macro import YamlMacro, YamlMacroConfig, YamlMacroParameterSpec

        # Extract metadata
        metadata = node_config.get("metadata", {})
        macro_name = metadata.get("name")
        if not macro_name:
            raise YamlPipelineBuilderError("Macro definition missing 'metadata.name'")

        macro_description = metadata.get("description")
        _ = metadata.get("namespace", "user")  # Reserved for future namespace support

        # Extract parameters
        raw_parameters = node_config.get("parameters", [])
        parameters = [YamlMacroParameterSpec(**p) for p in raw_parameters]

        # Extract nodes
        nodes = node_config.get("nodes", [])
        if not nodes:
            raise YamlPipelineBuilderError(
                f"Macro '{macro_name}' has no nodes. Macros must define at least one node."
            )

        # Extract outputs (optional)
        outputs = node_config.get("outputs")

        # Create YamlMacroConfig
        macro_config = YamlMacroConfig(
            macro_name=macro_name,
            macro_description=macro_description,
            parameters=parameters,
            nodes=nodes,
            outputs=outputs,
        )

        # Register macro at runtime
        # Create a dynamic class that pre-fills the YamlMacro config
        config_dict = macro_config.model_dump()

        class DynamicYamlMacro(YamlMacro):
            """Dynamically generated YamlMacro with pre-filled configuration."""

            def __init__(self, **kwargs: Any) -> None:
                # Merge pre-filled config with any override kwargs
                merged_config = {**config_dict, **kwargs}
                super().__init__(**merged_config)

        # Set class name for better debugging
        DynamicYamlMacro.__name__ = f"YamlMacro_{macro_name}"
        DynamicYamlMacro.__qualname__ = f"YamlMacro_{macro_name}"

        # Register in runtime storage (for YAML-defined macros)
        register_runtime(macro_name, DynamicYamlMacro)

        logger.info(
            f"✅ Registered YAML macro '{macro_name}' "
            f"({len(parameters)} parameters, {len(nodes)} nodes)"
        )

        # Return None - macro definitions don't add nodes to the graph
        return None


class MacroEntityPlugin:
    """Plugin for handling macro_invocation entities."""

    def can_handle(self, node_config: dict[str, Any]) -> bool:
        """Handle macro_invocation kind."""
        return node_config.get("kind") == "macro_invocation"

    def build(
        self, node_config: dict[str, Any], builder: YamlPipelineBuilder, graph: DirectedGraph
    ) -> NodeSpec | None:
        """Expand macro into subgraph and merge into main graph."""
        instance_name = node_config["metadata"]["name"]
        spec = node_config.get("spec", {})
        macro_ref = spec.get("macro")
        if not macro_ref:
            raise YamlPipelineBuilderError(f"Macro '{instance_name}' missing spec.macro field")

        # macro_ref is the full module path (e.g., hexdag.builtin.macros.ReasoningAgentMacro)
        # or a runtime-registered name for YAML-defined macros

        # Get config params for macro initialization
        config_params = spec.get("config", {}).copy()
        inputs = spec.get("inputs", {})
        dependencies = spec.get("dependencies", [])

        # Resolve macro class - either full module path or runtime-registered name
        try:
            macro_cls = resolve(macro_ref)
        except ResolveError as e:
            raise YamlPipelineBuilderError(f"Macro '{macro_ref}' not found: {e}") from e

        # Instantiate macro with config params
        try:
            macro_instance_obj = macro_cls(**config_params)
        except Exception as e:
            raise YamlPipelineBuilderError(f"Failed to instantiate macro '{macro_ref}': {e}") from e

        # Validate it's actually a macro
        if not isinstance(macro_instance_obj, ConfigurableMacro):
            type_name = type(macro_instance_obj).__name__
            raise YamlPipelineBuilderError(
                f"Component '{macro_ref}' is not a ConfigurableMacro (got {type_name})"
            )

        macro_instance: ConfigurableMacro = macro_instance_obj

        # Expand macro - merge config and inputs for validation
        # config params are the parameter values for the macro
        # inputs are the value mappings, so they should be merged
        macro_inputs = {**config_params, **inputs}

        # Expand macro
        try:
            subgraph = macro_instance.expand(
                instance_name=instance_name, inputs=macro_inputs, dependencies=dependencies
            )
        except ValueError:
            # Re-raise validation errors directly (e.g., required parameter, enum validation)
            raise
        except Exception as e:
            raise YamlPipelineBuilderError(
                f"Failed to expand macro '{macro_ref}' (instance '{instance_name}'): {e}"
            ) from e

        # Merge subgraph into main graph
        self._merge_subgraph(graph, subgraph, dependencies)

        logger.info(
            "✅ Expanded macro '{macro}' as '{instance}' ({nodes} nodes)",
            macro=macro_ref,
            instance=instance_name,
            nodes=len(subgraph.nodes),
        )

        # Return None - subgraph already merged into graph
        return None

    @staticmethod
    def _merge_subgraph(
        graph: DirectedGraph, subgraph: DirectedGraph, external_deps: list[str]
    ) -> None:
        """Merge subgraph into main graph with external dependencies.

        Optimized to avoid unnecessary graph copies when no external dependencies exist.
        """
        if not external_deps:
            # Fast path: direct merge when no external dependencies
            graph |= subgraph
        else:
            # Only process entry nodes that need external dependencies
            # Use in-place merge for better performance
            for node in subgraph.nodes.values():
                if not subgraph.get_dependencies(node.name):
                    # Entry node - add external dependencies
                    graph += node.after(*external_deps)
                else:
                    # Internal node - add as-is
                    graph += node


class NodeEntityPlugin:
    """Plugin for handling all node types (llm, function, agent, etc.)."""

    def __init__(self, builder: YamlPipelineBuilder):
        """Initialize with reference to builder for shared state."""
        self.builder = builder

    def can_handle(self, node_config: dict[str, Any]) -> bool:
        """Handle everything except macro_invocation."""
        return node_config.get("kind") != "macro_invocation"

    def build(
        self, node_config: dict[str, Any], builder: YamlPipelineBuilder, graph: DirectedGraph
    ) -> NodeSpec:
        """Build node from config.

        The 'kind' field must be a full module path to the node factory class.
        Example: hexdag.builtin.nodes.LLMNode
        """
        # Validate structure
        if "kind" not in node_config:
            raise YamlPipelineBuilderError("Node missing 'kind' field")
        if "metadata" not in node_config or "name" not in node_config["metadata"]:
            raise YamlPipelineBuilderError(
                f"Node '{node_config.get('kind')}' missing metadata.name"
            )

        kind = node_config["kind"]
        node_id = node_config["metadata"]["name"]
        spec = node_config.get("spec", {}).copy()
        # Dependencies can be at node level or inside spec (for backwards compatibility)
        deps = node_config.get("dependencies", []) or spec.pop("dependencies", [])

        # Resolve factory class from full module path
        try:
            factory_obj = resolve(kind)
        except ResolveError as e:
            raise YamlPipelineBuilderError(f"Cannot resolve node kind '{kind}': {e}") from e

        # Validate it's callable
        if not callable(factory_obj):
            raise YamlPipelineBuilderError(
                f"Node factory '{kind}' is not callable (got {type(factory_obj).__name__})"
            )

        # Handle factory classes vs factory functions
        # Factory classes need to be instantiated first, then called
        # Factory functions can be called directly
        if isinstance(factory_obj, type):
            # It's a class - instantiate then call
            factory_instance = factory_obj()
            factory = cast("Callable[..., NodeSpec]", factory_instance)
        else:
            # It's already a callable (function or instance)
            factory = cast("Callable[..., NodeSpec]", factory_obj)  # type: ignore[unreachable]

        # Create node - pass name as first positional arg
        node: NodeSpec = factory(node_id, **spec)

        # Add dependencies
        if deps:
            node = node.after(*deps) if isinstance(deps, list) else node.after(deps)

        return node


# ============================================================================
# Preprocessing Plugins
# ============================================================================


class IncludePreprocessPlugin:
    """Resolve !include directives for YAML file inclusion.

    Supports two syntaxes:
    1. Simple include: !include path/to/file.yaml
    2. Anchor include: !include path/to/file.yaml#anchor_name

    Security:
    - Only allows relative paths (no absolute paths)
    - Prevents directory traversal attacks (no ../ beyond project root)
    - Detects circular includes

    For comprehensive examples, see notebooks/03_yaml_includes_and_composition.ipynb
    """

    def __init__(self, base_path: Path | None = None, max_depth: int = 10):
        """Initialize include plugin.

        Args:
            base_path: Base directory for relative includes (changeable via context manager)
            max_depth: Maximum include nesting depth to prevent circular includes
        """
        self.base_path = base_path or Path.cwd()
        self.project_root = self.base_path  # Fixed project root for security validation
        self.max_depth = max_depth

    def process(self, config: dict[str, Any]) -> dict[str, Any]:
        """Process !include directives recursively."""
        # Create new include stack for this processing run (thread-safe)
        include_stack: list[Path] = []
        result = self._resolve_includes(
            config, self.base_path, depth=0, include_stack=include_stack
        )
        if not _is_dict_config(result):
            raise TypeError(
                f"Include processing must return a dictionary, got {type(result).__name__}. "
                "Check that your included files resolve to valid YAML dictionaries."
            )
        return result

    def _resolve_includes(
        self, obj: Any, current_base: Path, depth: int, include_stack: list[Path]
    ) -> dict[str, Any] | list[Any] | Any:
        """Recursively resolve !include directives.

        Args:
            obj: Object to process (dict, list, or primitive)
            current_base: Base path for resolving relative includes
            depth: Current recursion depth
            include_stack: Stack of currently processing files (for circular detection)

        Returns:
            Processed object with includes resolved
        """
        if depth > self.max_depth:
            raise YamlPipelineBuilderError(
                f"Include nesting too deep (max {self.max_depth}). "
                f"Possible circular include in: {' -> '.join(str(p) for p in include_stack)}"
            )

        if isinstance(obj, dict):
            # Check for !include directive
            if "!include" in obj and len(obj) == 1:
                include_spec = obj["!include"]
                return self._load_include(include_spec, current_base, depth, include_stack)

            # Recurse into dict values
            return {
                k: self._resolve_includes(v, current_base, depth, include_stack)
                for k, v in obj.items()
            }

        if isinstance(obj, list):
            # Process each list item and flatten nested lists from includes
            result = []
            for item in obj:
                resolved = self._resolve_includes(item, current_base, depth, include_stack)
                # Flatten: if an include returns a list, extend rather than append
                if isinstance(resolved, list):
                    result.extend(resolved)
                else:
                    result.append(resolved)
            return result

        return obj

    def _load_include(
        self, include_spec: str, current_base: Path, depth: int, include_stack: list[Path]
    ) -> Any:
        """Load content from included file.

        Args:
            include_spec: Include specification (e.g., "file.yaml" or "file.yaml#anchor")
            current_base: Base path for resolving relative paths
            depth: Current recursion depth
            include_stack: Stack of currently processing files (for circular detection)

        Returns:
            Loaded and processed content from included file
        """
        # Parse include specification (strip whitespace for better UX)
        include_spec = include_spec.strip()
        if "#" in include_spec:
            file_path_str, anchor = include_spec.split("#", 1)
            file_path_str = file_path_str.strip()
            anchor = anchor.strip()
        else:
            file_path_str, anchor = include_spec, None

        # Resolve file path
        file_path = self._resolve_path(file_path_str, current_base)

        # Check for circular includes
        if file_path in include_stack:
            cycle = " -> ".join(str(p) for p in include_stack + [file_path])
            raise YamlPipelineBuilderError(f"Circular include detected: {cycle}")

        # Load YAML file
        try:
            include_stack.append(file_path)
            content = yaml.safe_load(file_path.read_text(encoding="utf-8"))

            # Extract anchor if specified
            if anchor:
                if not isinstance(content, dict) or anchor not in content:
                    raise YamlPipelineBuilderError(
                        f"Anchor '{anchor}' not found in {file_path}. "
                        f"Available: {list(content.keys()) if isinstance(content, dict) else 'N/A'}"
                    )
                content = content[anchor]

            # Recursively resolve includes in loaded content
            return self._resolve_includes(content, file_path.parent, depth + 1, include_stack)

        except FileNotFoundError as e:
            raise YamlPipelineBuilderError(
                f"Include file not found: {file_path}\nSearched relative to: {current_base}"
            ) from e
        except yaml.YAMLError as e:
            raise YamlPipelineBuilderError(f"Invalid YAML in included file {file_path}: {e}") from e
        finally:
            include_stack.pop()

    def _resolve_path(self, path_str: str, current_base: Path) -> Path:
        """Resolve and validate include path.

        Args:
            path_str: Path string from !include directive
            current_base: Base path for resolving relative paths

        Returns:
            Validated absolute path

        Raises:
            YamlPipelineBuilderError: If path is invalid or potentially malicious
        """
        # Prevent absolute paths
        if Path(path_str).is_absolute():
            raise YamlPipelineBuilderError(
                f"Absolute paths not allowed in !include: {path_str}\n"
                "Use relative paths only for security."
            )

        # Resolve path relative to current base
        resolved = (current_base / path_str).resolve()

        # Prevent directory traversal outside project root
        # Use the resolved project_root (not base_path) to handle symlinks properly
        resolved_root = self.project_root.resolve()
        try:
            resolved.relative_to(resolved_root)
        except ValueError as e:
            raise YamlPipelineBuilderError(
                f"Include path traverses outside project root: {path_str}\n"
                f"Project root: {resolved_root}\n"
                f"Attempted path: {resolved}"
            ) from e

        return resolved


class EnvironmentVariablePlugin:
    """Resolve ${VAR} and ${VAR:default} in YAML with deferred secret resolution.

    For KeyVault/SecretPort workflows, secret-like environment variables are
    preserved as ${VAR} for runtime resolution. This allows:
    - Building pipelines without secrets present
    - Runtime secret injection via SecretPort → Memory
    - Separation of build and deployment contexts

    Secret patterns (deferred to runtime):
    - *_API_KEY, *_SECRET, *_TOKEN, *_PASSWORD, *_CREDENTIAL
    - SECRET_*

    Non-secret variables are resolved immediately at build-time.
    """

    ENV_VAR_PATTERN = re.compile(r"\$\{([A-Z_][A-Z0-9_]*?)(?::([^}]*))?\}")

    # Secret patterns that should be deferred to runtime
    SECRET_PATTERNS = frozenset({
        r".*_API_KEY$",
        r".*_SECRET$",
        r".*_TOKEN$",
        r".*_PASSWORD$",
        r".*_CREDENTIAL$",
        r"^SECRET_.*",
    })

    def __init__(self, defer_secrets: bool = True):
        """Initialize environment variable plugin.

        Parameters
        ----------
        defer_secrets : bool, default=True
            If True, preserve ${VAR} syntax for secret-like variables,
            allowing runtime resolution from KeyVault/Memory.
            If False, all variables are resolved at build-time (legacy behavior).
        """
        self.defer_secrets = defer_secrets
        if defer_secrets:
            # Compile secret detection regex
            self._secret_regex: re.Pattern[str] | None = re.compile(
                "|".join(f"({p})" for p in self.SECRET_PATTERNS)
            )
        else:
            self._secret_regex = None

    def process(self, config: dict[str, Any]) -> dict[str, Any]:
        """Recursively resolve environment variables."""
        result = _resolve_env_vars(
            config,
            self.ENV_VAR_PATTERN,
            secret_regex=self._secret_regex,
            defer_secrets=self.defer_secrets,
        )
        if not _is_dict_config(result):
            raise TypeError(
                f"Environment variable resolution must return a dictionary, "
                f"got {type(result).__name__}"
            )
        return result


@singledispatch
def _resolve_env_vars(
    obj: Any,
    pattern: re.Pattern[str],
    secret_regex: re.Pattern[str] | None = None,
    defer_secrets: bool = True,
) -> Any:
    """Recursively resolve ${VAR} in any structure.

    Parameters
    ----------
    obj : Any
        Object to process
    pattern : re.Pattern[str]
        Regex pattern for matching ${VAR} syntax
    secret_regex : re.Pattern[str] | None
        Regex for detecting secret-like variable names
    defer_secrets : bool
        If True, preserve ${VAR} for secrets

    Returns
    -------
    Any
        - For primitives: Returns the primitive unchanged
        - For strings: Returns str | int | float | bool (with type coercion)
        - For dicts: Returns dict[str, Any]
        - For lists: Returns list[Any]
    """
    return obj


@_resolve_env_vars.register(str)
def _resolve_env_vars_str(
    obj: str,
    pattern: re.Pattern[str],
    secret_regex: re.Pattern[str] | None = None,
    defer_secrets: bool = True,
) -> str | int | float | bool:
    """Resolve ${VAR} in strings with optional secret deferral."""

    def replacer(match: re.Match[str]) -> str:
        var_name, default = match.group(1), match.group(2)

        # Check if this looks like a secret
        if defer_secrets and secret_regex and secret_regex.match(var_name):
            # Secret detected - preserve ${VAR} syntax for runtime resolution
            logger.debug(f"Deferring secret variable to runtime: {var_name}")
            return match.group(0)  # Return original ${VAR} or ${VAR:default}

        # Non-secret - resolve immediately from environment
        env_value = os.environ.get(var_name)
        if env_value is None:
            if default is not None:
                return default
            raise YamlPipelineBuilderError(f"Environment variable '${{{var_name}}}' not set")
        return env_value

    resolved = pattern.sub(replacer, obj)

    # Type coercion only if the value changed (was resolved)
    if resolved != obj and not (defer_secrets and resolved.startswith("${")):
        if resolved.lower() in ("true", "yes", "1"):
            return True
        if resolved.lower() in ("false", "no", "0"):
            return False
        with suppress(ValueError):
            return int(resolved)
        with suppress(ValueError):
            return float(resolved)
    return resolved


@_resolve_env_vars.register(dict)
def _resolve_env_vars_dict(
    obj: dict,
    pattern: re.Pattern[str],
    secret_regex: re.Pattern[str] | None = None,
    defer_secrets: bool = True,
) -> dict[str, Any]:
    """Resolve ${VAR} in dict values."""
    return {k: _resolve_env_vars(v, pattern, secret_regex, defer_secrets) for k, v in obj.items()}


@_resolve_env_vars.register(list)
def _resolve_env_vars_list(
    obj: list,
    pattern: re.Pattern[str],
    secret_regex: re.Pattern[str] | None = None,
    defer_secrets: bool = True,
) -> list[Any]:
    """Resolve ${VAR} in list items."""
    return [_resolve_env_vars(item, pattern, secret_regex, defer_secrets) for item in obj]


class TemplatePlugin:
    """Render Jinja2 templates in YAML with two-phase rendering strategy.

    **Build-time Rendering** (YAML configuration context):
    - Metadata fields (e.g., node names, descriptions)
    - Pipeline-level spec fields (e.g., variables, ports, policies)
    - Enables dynamic configuration from environment/variables

    **Runtime Rendering** (node execution context):
    - Node spec fields (e.g., template, prompt_template, initial_prompt)
    - Pipeline outputs (e.g., spec.outputs with {{node.result}} references)
    - Preserved to allow access to dependency/node outputs at runtime
    - Enables dynamic prompts and output mapping based on execution results

    Example:
        ```yaml
        spec:
          variables:
            node_name: analyzer
          nodes:
            - kind: llm_node
              metadata:
                name: "{{ spec.variables.node_name }}"  # Build-time: renders to "analyzer"
              spec:
                template: "{{input}}"  # Runtime: renders when node executes
          outputs:
            result: "{{analyzer.analysis}}"  # Runtime: rendered after pipeline completes
        ```

    Security:
        Uses SandboxedEnvironment to prevent arbitrary code execution.
    """

    def __init__(self) -> None:
        # Use sandboxed environment to prevent code execution attacks
        # Even though this is for config files, defense in depth is important
        self.env = SandboxedEnvironment(autoescape=False, keep_trailing_newline=True)

    def process(self, config: dict[str, Any]) -> dict[str, Any]:
        """Render Jinja2 templates with config as context."""
        result = _render_templates(config, config, self.env)
        if not _is_dict_config(result):
            raise TypeError(
                f"Template rendering must return a dictionary, got {type(result).__name__}"
            )
        return result


@singledispatch
def _render_templates(obj: Any, context: dict[str, Any], env: Any) -> Any:
    """Recursively render Jinja2 templates.

    Returns:
        - For primitives: Returns the primitive unchanged
        - For strings: Returns str | int | float | bool (with type coercion)
        - For dicts: Returns dict[str, Any]
        - For lists: Returns list[Any]
    """
    return obj


@_render_templates.register(str)
def _render_templates_str(obj: str, context: dict[str, Any], env: Any) -> str | int | float | bool:
    """Render Jinja2 template in string."""
    if "{{" not in obj and "{%" not in obj:
        return obj

    try:
        rendered: str = env.from_string(obj).render(context)
        # Type coercion
        with suppress(ValueError):
            return int(rendered)
        with suppress(ValueError):
            return float(rendered)
        if rendered.lower() in ("true", "yes"):
            return True
        if rendered.lower() in ("false", "no"):
            return False
        return rendered
    except TemplateSyntaxError as e:
        raise YamlPipelineBuilderError(
            f"Invalid Jinja2 template syntax: {e}\nTemplate: {obj}"
        ) from e
    except UndefinedError as e:
        raise YamlPipelineBuilderError(
            f"Undefined variable in template: {e}\nTemplate: {obj}"
        ) from e


@_render_templates.register(dict)
def _render_templates_dict(obj: dict, context: dict[str, Any], env: Any) -> dict[str, Any]:
    """Render templates in dict values.

    Skip rendering for node spec fields and pipeline outputs to avoid conflicts between
    YAML-level templating and runtime template strings.

    Strategy:
    - Node specs: Preserve for runtime rendering with dependency outputs
    - Pipeline outputs: Preserve for runtime rendering with node results
    - Metadata and config: Render at build time with YAML context
    """
    result = {}
    for k, v in obj.items():
        # Check if this is a node spec (not a Pipeline spec) by looking for the 'kind' sibling key
        # Node kinds end with '_node' (e.g., 'prompt_node', 'llm_node', 'function_node')
        # Pipeline kind is 'Pipeline' - we should NOT skip its spec
        if (
            k == "spec"
            and isinstance(v, dict)
            and "kind" in obj
            and isinstance(obj.get("kind"), str)
            and obj["kind"] != "Pipeline"
        ):
            # This is a node spec - preserve template strings for runtime rendering
            result[k] = v  # Preserve entire spec as-is
        # Also preserve 'outputs' field in Pipeline spec - references node results
        elif k == "outputs" and isinstance(v, dict):
            # Pipeline outputs reference node results (e.g., {{node.output}})
            # Preserve for runtime rendering after nodes execute
            result[k] = v
        else:
            result[k] = _render_templates(v, context, env)
    return result


@_render_templates.register(list)
def _render_templates_list(obj: list, context: dict[str, Any], env: Any) -> list[Any]:
    """Render templates in list items."""
    return [_render_templates(item, context, env) for item in obj]


# ============================================================================
# Utilities
# ============================================================================


@lru_cache(maxsize=32)
def _parse_yaml_cached(yaml_content: str) -> Any:
    """Cached YAML parsing.

    Returns dict[str, Any] in practice, but yaml.safe_load returns Any.
    """
    return yaml.safe_load(yaml_content)
