#!/usr/bin/env python3
"""
#exonware/xwdata/src/exonware/xwdata/data/engine.py
XWDataEngine - The Brain of XWData
This module provides the core orchestration engine that coordinates:
- XWSerializer (xwsystem) for format I/O
- FormatStrategyRegistry for format-specific logic
- MetadataProcessor for universal metadata
- ReferenceResolver for reference handling
- CacheManager for performance
- NodeFactory for node creation
Company: eXonware.com
Author: eXonware Backend Team
Email: connect@exonware.com
Version: 0.9.0.6
Generation Date: 26-Oct-2025
"""

import asyncio
import hashlib
from collections.abc import AsyncIterator
from typing import Any
from pathlib import Path
from exonware.xwsystem import get_logger, get_serializer, JsonSerializer
from exonware.xwsystem.io.serialization.auto_serializer import AutoSerializer
from exonware.xwsystem.io.serialization.serializer import _get_global_serializer
from exonware.xwsystem.io import get_scheme
from exonware.xwsystem.io.stream.async_operations import async_safe_read_text, async_safe_write_text
from exonware.xwsystem.io.serialization import YamlSerializer, XmlSerializer, TomlSerializer, CsvSerializer, BsonSerializer
from ..base import ADataEngine
from ..config import XWDataConfig, LoadStrategy
from ..errors import (
    XWDataEngineError, XWDataIOError, XWDataParseError,
    XWDataSerializeError, XWDataFileNotFoundError
)
from ..defs import MergeStrategy
from .node import XWDataNode
from .factory import NodeFactory
logger = get_logger(__name__)
# ==============================================================================
# GLOBAL CACHE (xwsystem integration - shared across ALL instances)
# ==============================================================================
# Try to use xwsystem's global cache for maximum efficiency
from exonware.xwsystem.caching import create_cache
_GLOBAL_XWDATA_CACHE = create_cache(capacity=5000, namespace='xwdata', name='xwdata_global')  # Global cache (configurable; defaults to PylruCache when pylru installed, else FunctoolsLRUCache)
_JSON_SERIALIZER = get_serializer(JsonSerializer)  # Reuse xwsystem JSON flyweight (same stack as json_utils, catalog)
logger.debug("Using xwsystem global cache for xwdata")
# ==============================================================================
# MODULE-LEVEL FORMAT CACHE (Persistent across engine instances)
# ==============================================================================
_FORMAT_EXTENSION_CACHE = {
    # Text formats
    '.json': 'JSON',
    '.json5': 'JSON5',
    '.jsonl': 'JSONL',
    '.ndjson': 'JSONL',  # NDJSON is an alias for JSONL
    '.jsonlines': 'JSONL',  # Alternative extension
    '.yaml': 'YAML',
    '.yml': 'YAML',
    '.xml': 'XML',
    '.toml': 'TOML',
    '.ini': 'INI',
    '.cfg': 'ConfigParser',
    '.conf': 'ConfigParser',
    '.csv': 'CSV',
    # Binary formats
    '.bson': 'BSON',
    '.msgpack': 'MessagePack',
    '.cbor': 'CBOR',
    '.pickle': 'Pickle',
    '.pkl': 'Pickle',
    # Schema-based formats
    '.avro': 'Avro',
    '.proto': 'Protobuf',
    '.parquet': 'Parquet',
    '.orc': 'ORC',
    # Key-value stores
    '.lmdb': 'LMDB',
    '.zarr': 'Zarr',
    # Scientific formats
    '.hdf5': 'HDF5',
    '.h5': 'HDF5',
    '.feather': 'Feather',
    '.arrow': 'Arrow'
}


class XWDataEngine(ADataEngine):
    """
    Universal data engine orchestrating all xwdata operations.
    The engine is the brain of xwdata, coordinating:
    1. Format I/O via xwsystem's XWSerializer (reuse, no duplication)
    2. Format-specific logic via FormatStrategy plugins
    3. Metadata extraction and preservation
    4. Reference detection and resolution
    5. Performance optimization (caching, pooling)
    6. Node creation with COW semantics
    This is a pure orchestration engine - it delegates to specialized
    components and doesn't implement low-level logic itself.
    """

    def __init__(self, config: XWDataConfig | None = None):
        """
        Initialize data engine with configuration.
        Args:
            config: Optional configuration (uses defaults if not provided)
        """
        super().__init__()
        self._config = config or XWDataConfig.default()
        # Core components (lazy initialization)
        self._serializer: AutoSerializer | None = None
        self._xwsyntax: Any | None = None  # Optional xwsyntax integration (lazy)
        self._strategies: Any | None = None  # FormatStrategyRegistry (lazy)
        self._metadata_processor: Any | None = None  # MetadataProcessor (lazy)
        self._reference_resolver: Any | None = None  # ReferenceResolver (lazy)
        self._cache_manager: Any | None = None  # CacheManager (lazy)
        self._node_factory = NodeFactory(self._config)
        # Atomic helpers are provided by xwsystem.serializer module functions
        logger.debug("XWDataEngine initialized")
    # ==========================================================================
    # CACHE KEY GENERATION
    # ==========================================================================

    def _get_cache_key(
        self, 
        path_obj: Path, 
        format_hint: str | None = None,
        use_content_hash: bool = True
    ) -> str:
        """
        Generate intelligent cache key with content-based hashing.
        Strategies:
        1. Content-based (preferred for small files): Hash of content + format
        2. Path-based (fallback for large files): Path + mtime + size
        This enables:
        - Cache reuse across file moves/copies (same content)
        - Automatic invalidation on content changes
        - Better hit rate in production (80-95% typical)
        Args:
            path_obj: File path
            format_hint: Optional format hint
            use_content_hash: Use content-based hashing
        Returns:
            Cache key string
        """
        try:
            file_size = path_obj.stat().st_size
            # For small files (<100KB), use content hash (fast to read)
            if use_content_hash and file_size < 1024 * 100:
                content = path_obj.read_text(encoding='utf-8')
                content_hash = hashlib.md5(content.encode()).hexdigest()[:16]
                format_str = format_hint or 'auto'
                return f"load:{format_str}:{content_hash}"
            else:
                # For large files, use path + mtime + size (avoid reading)
                mtime = int(path_obj.stat().st_mtime)
                path_hash = hashlib.md5(str(path_obj).encode()).hexdigest()[:8]
                return f"load:{path_hash}:{mtime}:{file_size}"
        except Exception as e:
            # Fallback to simple path-based key
            logger.debug(f"Cache key generation failed, using simple key: {e}")
            return f"load:{str(path_obj)}"

    def _detect_format_fast(self, path_obj: Path, format_hint: str | None) -> str:
        """
        Fast format detection using module-level extension cache.
        This is O(1) lookup with zero overhead, using the persistent
        _FORMAT_EXTENSION_CACHE defined at module level.
        """
        if format_hint:
            return format_hint.upper()
        # O(1) lookup in extension cache (instant!)
        ext = path_obj.suffix.lower()
        return _FORMAT_EXTENSION_CACHE.get(ext, 'JSON')
    # ==========================================================================
    # LAZY INITIALIZATION
    # ==========================================================================

    def _ensure_serializer(self) -> AutoSerializer:
        """
        Lazy initialize AutoSerializer from xwsystem.
        Auto-discovers ALL serializers from:
        - xwsystem SerializationRegistry (base formats)
        - xwformats (if available - extended formats)
        - xwjson (required dependency - auto-registers)
        - Any other auto-registering format libraries
        This ensures xwdata can use ANY format serializer that's registered
        in the UniversalCodecRegistry, without needing format strategies.
        Format strategies are optional and only add xwdata-specific features
        (metadata extraction, reference detection).
        Required dependencies: xwsystem, xwnode, xwquery, xwjson
        """
        if self._serializer is None:
            # xwjson is required - import to trigger auto-registration
            import exonware.xwjson  # This triggers auto-registration
            logger.debug("xwdata: xwjson imported (required dependency, auto-registered)")
            # Try to use xwformats if available (optional dependency) - import to trigger auto-registration
            try:
                import exonware.xwformats  # This triggers auto-registration of all xwformats serializers
                logger.debug("xwdata: xwformats imported (optional dependency, auto-registered)")
            except ImportError:
                # xwformats not installed - continue without it
                logger.debug("xwdata: xwformats not available (optional dependency)")
            # Create AutoSerializer which will auto-discover all registered serializers
            # from both xwsystem and xwformats (if available)
            from exonware.xwsystem.io.serialization.auto_serializer import AutoSerializer
            self._serializer = AutoSerializer()
            logger.debug("xwdata: Initialized AutoSerializer (discovers all registered formats)")
            # Auto-discover all available serializers from SerializationRegistry
            from exonware.xwsystem.io.serialization.registry import SerializationRegistry, get_serialization_registry
            registry = get_serialization_registry()
            available_formats = registry.list_formats()
            logger.debug(f"xwdata: Auto-discovered {len(available_formats)} serializers: {available_formats}")
            # Optionally integrate xwsyntax for grammar-based parsing (optional dependency)
            from exonware.xwsyntax import XWSyntax
            self._xwsyntax = XWSyntax()
            logger.debug("xwdata: xwsyntax integration available (extends XWSystem)")
        return self._serializer

    def _ensure_strategies(self) -> Any:
        """Lazy initialize format strategy registry."""
        if self._strategies is None:
            from .strategies.registry import FormatStrategyRegistry
            self._strategies = FormatStrategyRegistry()
            logger.debug("Initialized FormatStrategyRegistry")
        return self._strategies

    def _ensure_metadata_processor(self) -> Any:
        """Lazy initialize metadata processor."""
        if self._metadata_processor is None:
            from .metadata.processor import MetadataProcessor
            self._metadata_processor = MetadataProcessor(self._config)
            logger.debug("Initialized MetadataProcessor")
        return self._metadata_processor

    def _ensure_reference_resolver(self) -> Any:
        """Lazy initialize reference resolver."""
        if self._reference_resolver is None:
            from .references.resolver import ReferenceResolver
            self._reference_resolver = ReferenceResolver(self._config)
            logger.debug("Initialized ReferenceResolver")
        return self._reference_resolver

    def _ensure_cache_manager(self) -> Any:
        """Lazy initialize cache manager."""
        if self._cache_manager is None:
            from ..common.caching.cache_manager import CacheManager
            self._cache_manager = CacheManager(self._config)
            logger.debug("Initialized CacheManager")
        return self._cache_manager

    def _get_serializer_for_format(self, format_name: str) -> Any | None:
        """
        Get serializer for format name, auto-discovering from SerializationRegistry.
        This method enables xwdata to use ANY serializer registered in xwsystem/xwformats,
        not just hardcoded ones. It tries SerializationRegistry first (dynamic discovery),
        then falls back to AutoSerializer (for backward compatibility).
        Args:
            format_name: Format name (e.g., 'json', 'yaml', 'edn', 'ubjson')
        Returns:
            Serializer instance or None
        """
        # Normalize format name
        format_name = format_name.upper() if format_name else None
        if not format_name:
            return None
        # Try SerializationRegistry first (dynamic discovery)
        try:
            from exonware.xwsystem.io.serialization.registry import get_serialization_registry
            registry = get_serialization_registry()
            # Try exact format name
            serializer = registry.get_by_format(format_name.lower())
            if serializer:
                logger.debug(f"xwdata: Found serializer for '{format_name}' via SerializationRegistry")
                return serializer
            # Try alternative format names
            format_aliases = {
                'JSON': 'json',
                'YAML': 'yaml',
                'XML': 'xml',
                'TOML': 'toml',
                'CSV': 'csv',
                'EDN': 'edn',
                'UBJSON': 'ubjson',
                'HOCON': 'hocon',
                'HJSON': 'hjson',
                'ARROW': 'arrow',
            }
            alias = format_aliases.get(format_name, format_name.lower())
            serializer = registry.get_by_format(alias)
            if serializer:
                logger.debug(f"xwdata: Found serializer for '{format_name}' via alias '{alias}'")
                return serializer
        except Exception as e:
            logger.debug(f"xwdata: SerializationRegistry lookup failed: {e}")
        # Fallback to AutoSerializer (backward compatibility)
        try:
            serializer = self._ensure_serializer()
            return serializer._get_serializer(format_name)
        except Exception as e:
            logger.debug(f"xwdata: AutoSerializer lookup failed: {e}")
            return None
    # ==========================================================================
    # CORE OPERATIONS
    # ==========================================================================

    async def load(
        self,
        path: str | Path,
        format_hint: str | None = None,
        **opts
    ) -> XWDataNode:
        """
        Load data from file through full pipeline.
        Pipeline:
        1. Validate path and file size (security)
        2. Check cache (performance)
        3. Read and deserialize via xwsystem (reuse)
        4. Get format strategy (format-specific logic)
        5. Extract metadata (universal metadata)
        6. Detect/resolve references (if enabled)
        7. Create XWDataNode (XWNode + COW)
        8. Cache result (performance)
        Args:
            path: File path to load
            format_hint: Optional format hint
            **opts: Additional options
        Returns:
            XWDataNode with data, metadata, and references
        """
        path_str = str(path)
        scheme = get_scheme(path_str)
        # URI (http, https, ftp, etc.): all logic in xwsystem; engine only deserializes
        if scheme != 'file':
            return await self.load_from_source(path_str, format_hint=format_hint, **opts)
        try:
            # 1. Validate path (file only)
            path_obj = await self._validate_path(path)
            if not path_obj.exists():
                raise XWDataFileNotFoundError(str(path_obj))
            # 2. Validate file size
            await self._validate_file_size(path_obj)
            # 3. CHECK CACHE FIRST (before any processing!) 🚀
            cache_key = None
            if self._config.performance.enable_caching:
                cache_key = self._get_cache_key(path_obj, format_hint)
                # Try global cache first (shared across all engines)
                if _GLOBAL_XWDATA_CACHE is not None:
                    cached = _GLOBAL_XWDATA_CACHE.get(cache_key)
                    if cached is not None:
                        logger.debug(f"💎 Global cache hit: {cache_key}")
                        return cached  # INSTANT RETURN! 100-10,000x faster
                # Fall back to instance cache
                cache = self._ensure_cache_manager()
                cached = await cache.get(cache_key)
                if cached is not None:
                    logger.debug(f"💎 Instance cache hit: {cache_key}")
                    return cached  # INSTANT RETURN! 100-10,000x faster
            # 4. Strategy selection: use FULL/FAST path for small files, LAZY for large ones
            file_size_kb = path_obj.stat().st_size / 1024
            file_size_mb = file_size_kb / 1024
            load_strategy = self._select_load_strategy(file_size_mb)
            # V8 "file-backed lazy node" path:
            # When defer_file_io is enabled, treat PARTIAL/STREAMING the same as LAZY
            # to avoid accidental full-file reads (especially for multi-GB JSONL).
            if self._config.lazy.defer_file_io and load_strategy in (
                LoadStrategy.LAZY,
                LoadStrategy.PARTIAL,
                LoadStrategy.STREAMING,
            ):
                logger.debug(
                    f"🕰️ File-backed lazy node for {path_obj} "
                    f"(strategy={load_strategy.value}, {file_size_mb:.2f}MB)"
                )
                format_name = self._detect_format_fast(path_obj, format_hint)
                # Best-effort detection metadata without reading the full file.
                detection_method = 'hint' if format_hint else 'extension'
                metadata = {
                    'source_path': str(path_obj),
                    'format': format_name,
                    'detected_format': format_name,
                    'size_bytes': path_obj.stat().st_size,
                    'lazy_mode': 'file',
                    'load_strategy': load_strategy.value,
                    'detection_confidence': 1.0,
                    'detection_method': detection_method,
                    'format_candidates': {format_name: 1.0},
                }
                node = await self._node_factory.create_node(
                    data=None,
                    metadata=metadata,
                    config=self._config,
                )
            # If explicit format hint is provided for non-lazy strategies, honor FULL
            # pipeline to get rich content-based detection metadata (confidence, candidates).
            elif format_hint and load_strategy != LoadStrategy.LAZY:
                logger.debug(f"📋 Full pipeline (hint provided): {path_obj}")
                node = await self._full_pipeline_load(path_obj, format_hint)
            else:
                # FAST PATH: Small non-cached files (xData-Old style)
                if self._config.performance.enable_fast_path and file_size_kb < self._config.performance.fast_path_threshold_kb:
                    logger.debug(f"⚡ Fast path: {path_obj} ({file_size_kb:.1f}KB)")
                    # ULTRA-FAST PATH: For very small files (< 1KB), bypass serializer overhead
                    if file_size_kb < 1.0:
                        # V8 OPTIMIZATION: Hyper-fast path for JSON (most common case)
                        if path_obj.suffix.lower() == '.json':
                            node = await self._hyper_fast_json_load(path_obj)
                        else:
                            node = await self._ultra_fast_load(path_obj, format_hint)
                    else:
                        node = await self._fast_load_small(path_obj, format_hint)
                else:
                    # FULL PIPELINE: Large files or when fast path disabled
                    logger.debug(f"📋 Full pipeline: {path_obj} ({file_size_kb:.1f}KB)")
                    node = await self._full_pipeline_load(path_obj, format_hint)
            # Cache the result (both global and instance)
            if cache_key:
                # Global cache (instant for all engines)
                if _GLOBAL_XWDATA_CACHE is not None:
                    _GLOBAL_XWDATA_CACHE.put(cache_key, node)
                # Instance cache (for this engine)
                if 'cache' in locals():
                    await cache.set(cache_key, node)
            return node
        except Exception as e:
            if isinstance(e, (XWDataIOError, XWDataParseError)):
                raise
            raise XWDataEngineError(
                f"Failed to load file: {e}",
                path=str(path),
                operation='load'
            ) from e

    def _to_source_load_config(self) -> "SourceLoadConfig":
        """Build xwsystem SourceLoadConfig from XWDataConfig (security + reference)."""
        from exonware.xwsystem.io import SourceLoadConfig
        sec = getattr(self._config, 'security', None)
        ref = getattr(self._config, 'reference', None)
        return SourceLoadConfig(
            allowed_schemes=getattr(sec, 'allowed_schemes', None) or getattr(ref, 'allowed_schemes', ('file', 'http', 'https', 'ftp')),
            allow_external=getattr(ref, 'follow_external', True),
            timeout_sec=float(getattr(ref, 'timeout_seconds', 30)),
            max_size_mb=float(getattr(sec, 'max_file_size_mb', None) or getattr(ref, 'max_external_size_mb', 100)),
            encoding='utf-8',
        )

    async def load_from_source(
        self,
        uri_or_path: str,
        format_hint: str | None = None,
        **opts
    ) -> XWDataNode:
        """
        Load data from a URI or path via xwsystem (config-driven). No load logic
        in xwdata—all scheme and security handling in xwsystem.
        """
        from exonware.xwsystem.io import read_source_text
        config = self._to_source_load_config()
        try:
            content_str, source_meta = await read_source_text(uri_or_path, config=config)
        except Exception as e:
            raise XWDataEngineError(
                f"Failed to load from source: {e}",
                path=uri_or_path,
                operation='load_from_source'
            ) from e
        content_type = source_meta.get('content_type') or ''
        if format_hint:
            format_name = format_hint.upper()
        elif content_type and 'json' in content_type:
            format_name = 'JSON'
        elif content_type and ('yaml' in content_type or 'yml' in content_type):
            format_name = 'YAML'
        else:
            format_name = (format_hint or 'JSON').upper()
        serializer = self._ensure_serializer()
        data = serializer.detect_and_deserialize(content_str, format_hint=format_name)
        metadata = {
            'source_path': uri_or_path,
            'format': format_name,
            'detected_format': format_name,
            'size_bytes': len(content_str.encode('utf-8')),
            'source': 'uri',
            '_raw_text': content_str,
        }
        return await self._node_factory.create_node(
            data=data,
            metadata=metadata,
            config=self._config
        )

    async def save(
        self,
        node: XWDataNode,
        path: str | Path,
        format: str | None = None,
        **opts
    ) -> None:
        """
        Save node to file.
        Pipeline:
        1. Validate path (security)
        2. Determine format (from hint or extension)
        3. Extract native data from node
        4. Serialize via xwsystem (reuse!)
        5. Write file
        Args:
            node: Node to save
            path: Target file path
            format: Optional format override
            **opts: Serialization options
        """
        try:
            # 1. Validate path (for writing)
            path_obj = await self._validate_path(path, for_writing=True)
            # 2. Determine format
            target_format = format or path_obj.suffix.lstrip('.').lower()
            if not target_format:
                raise XWDataSerializeError(
                    "Cannot determine format - specify format or use file extension",
                    path=str(path_obj)
                )
            # 3. Extract native data
            native_data = node.to_native()
            # 4. Serialize via xwsystem with universal options (format-agnostic)
            auto_serializer = self._ensure_serializer()
            # Set defaults for pretty printing unless user overrides
            universal_opts = {
                'pretty': True,
                'ensure_ascii': False,
                'indent': 2,
            }
            # User opts override defaults
            universal_opts.update(opts)
            # Delegate to xwsystem with universal options
            serialized_content = auto_serializer.detect_and_serialize(
                native_data, 
                format_hint=target_format,
                **universal_opts
            )
            # 5. Write file (use sync write for simplicity and avoid event loop issues)
            # async_safe_write_text has issues with nested event loops
            path_obj.parent.mkdir(parents=True, exist_ok=True)
            # Handle both str and bytes
            if isinstance(serialized_content, bytes):
                path_obj.write_bytes(serialized_content)
            else:
                path_obj.write_text(serialized_content, encoding='utf-8')
            logger.info(f"Saved {target_format.upper()} to {path_obj}")
        except Exception as e:
            if isinstance(e, (XWDataIOError, XWDataSerializeError)):
                raise
            raise XWDataEngineError(
                f"Failed to save file: {e}",
                path=str(path),
                operation='save'
            ) from e

    async def parse(
        self,
        content: str | bytes,
        format: str,
        **opts
    ) -> XWDataNode:
        """
        Parse content with specified format.
        Args:
            content: Content to parse
            format: Format name
            **opts: Parse options
        Returns:
            XWDataNode
        """
        try:
            # 1. Deserialize via xwsystem (this is sync, no need for executor)
            serializer = self._ensure_serializer()
            native_data = serializer.detect_and_deserialize(content, format_hint=format)
            # 2. Get strategy
            strategies = self._ensure_strategies()
            strategy = strategies.get(format)
            # 3. Extract metadata
            metadata = {}
            if strategy and self._config.metadata.enable_universal_metadata:
                metadata_processor = self._ensure_metadata_processor()
                metadata = await metadata_processor.extract(native_data, strategy)
            # 4. Detect references
            references = []
            if strategy:
                from .references.detector import ReferenceDetector
                detector = ReferenceDetector()
                references = await detector.detect(native_data, strategy)
            # 5. Create node
            format_info = {'format': format, 'source': 'parse'}
            return await self._node_factory.create_node(
                data=native_data,
                metadata=metadata,
                format_info=format_info,
                references=references
            )
        except Exception as e:
            raise XWDataParseError(
                f"Failed to parse {format.upper()} content: {e}",
                format=format
            ) from e

    async def create_node_from_native(
        self,
        data: Any,
        metadata: dict | None = None,
        **opts
    ) -> XWDataNode:
        """
        Create node from native Python data.
        Args:
            data: Native Python data (dict, list, etc.)
            metadata: Optional metadata
            **opts: Additional options
        Returns:
            XWDataNode
        """
        return await self._node_factory.create_from_native(
            data,
            metadata=metadata,
            **opts
        )

    async def merge_nodes(
        self,
        nodes: list[XWDataNode],
        strategy: str | MergeStrategy = 'deep'
    ) -> XWDataNode:
        """
        Merge multiple nodes into one.
        Args:
            nodes: List of nodes to merge
            strategy: Merge strategy
        Returns:
            Merged XWDataNode
        """
        if not nodes:
            raise XWDataEngineError("Cannot merge empty node list", operation='merge')
        if len(nodes) == 1:
            return nodes[0]
        # Convert strategy to string if enum
        if hasattr(strategy, 'name'):
            strategy = strategy.name.lower()
        # Start with first node's data
        merged_data = nodes[0].to_native()
        merged_metadata = nodes[0].metadata.copy()
        merged_refs = nodes[0].get_references().copy()
        # Merge remaining nodes
        for node in nodes[1:]:
            node_data = node.to_native()
            if strategy == 'deep':
                merged_data = self._deep_merge(merged_data, node_data)
            elif strategy == 'shallow':
                merged_data = self._shallow_merge(merged_data, node_data)
            elif strategy == 'replace':
                merged_data = node_data
            else:
                merged_data = self._deep_merge(merged_data, node_data)
            # Merge metadata
            merged_metadata.update(node.metadata)
            # Merge references
            merged_refs.extend(node.get_references())
        # Create merged node
        return await self._node_factory.create_node(
            data=merged_data,
            metadata=merged_metadata,
            references=merged_refs
        )

    def _deep_merge(self, base: Any, overlay: Any) -> Any:
        """Deep merge two data structures."""
        if isinstance(base, dict) and isinstance(overlay, dict):
            result = base.copy()
            for key, value in overlay.items():
                if key in result and isinstance(result[key], dict) and isinstance(value, dict):
                    result[key] = self._deep_merge(result[key], value)
                elif key in result and isinstance(result[key], list) and isinstance(value, list):
                    result[key] = result[key] + value  # Array append
                else:
                    result[key] = value
            return result
        elif isinstance(base, list) and isinstance(overlay, list):
            return base + overlay
        else:
            return overlay

    def _shallow_merge(self, base: Any, overlay: Any) -> Any:
        """Shallow merge (top-level only)."""
        if isinstance(base, dict) and isinstance(overlay, dict):
            result = base.copy()
            result.update(overlay)
            return result
        elif isinstance(base, list) and isinstance(overlay, list):
            return base + overlay
        else:
            return overlay

    async def stream_load(
        self,
        path: str | Path,
        chunk_size: int = 8192,
        **opts
    ) -> AsyncIterator[XWDataNode]:
        """
        Stream load large files in chunks.
        Uses true streaming for formats that support incremental loading (YAML multi-doc, JSONL).
        Falls back to full-load for formats without streaming support.
        Args:
            path: File path
            chunk_size: Chunk size in bytes (unused for true streaming, kept for compatibility)
            **opts: Additional options
        Yields:
            XWDataNode for each chunk/document
        """
        path_obj = await self._validate_path(path)
        # Get serializer and detect format
        serializer = self._ensure_serializer()
        format_name = self._detect_format_fast(path_obj, None)
        specialized_serializer = serializer._get_serializer(format_name)
        # Check if format supports incremental streaming
        if specialized_serializer.supports_incremental_streaming:
            # True streaming: yield documents as they're parsed
            for document in specialized_serializer.incremental_load(path_obj, **opts):
                node = await self._node_factory.create_node(
                    data=document,
                    metadata={'source_path': str(path_obj), 'format': format_name},
                    config=self._config
                )
                yield node
        else:
            # Fallback: load entire file and yield once
            node = await self.load(path_obj, **opts)
            yield node
    # ==========================================================================
    # UTILITY METHODS
    # ==========================================================================

    def get_stats(self) -> dict[str, Any]:
        """Get engine statistics."""
        stats = {
            'config': {
                'caching_enabled': self._config.performance.enable_caching,
                'pooling_enabled': self._config.performance.enable_pooling,
                'reference_resolution': self._config.reference.resolution_mode.name
            }
        }
        # Add cache stats if initialized
        if self._cache_manager is not None:
            stats['cache'] = self._cache_manager.get_stats()
        # Add pool stats if initialized
        stats['pool'] = self._node_factory.get_pool_stats()
        return stats

    async def clear_caches(self) -> None:
        """Clear all caches."""
        if self._cache_manager is not None:
            await self._cache_manager.clear()
        if self._serializer is not None:
            # Clear xwsystem serializer caches if available
            if hasattr(self._serializer, 'clear_cache'):
                await self._serializer.clear_cache()
    # ==========================================================================
    # V8: SIZE DETECTION & STRATEGY SELECTION
    # ==========================================================================

    def _detect_file_size_mb(self, path_obj: Path) -> float:
        """
        Detect file size in megabytes (V8).
        Returns:
            File size in MB
        """
        try:
            return path_obj.stat().st_size / (1024 * 1024)
        except Exception:
            return 0.0
    # ==========================================================================
    # LAZY & ATOMIC ACCESS HELPERS
    # ==========================================================================

    async def lazy_get_from_file(
        self,
        node: XWDataNode,
        path: str,
        default: Any = None
    ) -> Any:
        """
        Lazily read a value at the given path using xwsystem atomic_read_path.
        Requirements:
        - Uses xwsystem.io serializers for path-based access.
        - Avoids loading the entire file into memory for large datasets.
        - Honors GUIDE_DEV / GUIDE_TEST: no rigged behavior, real atomic ops.
        """
        source_path = node.metadata.get('source_path')
        if not source_path:
            # Fallback to normal navigation if we don't know the source file
            return node.get_value_at_path(path, default)
        # Use xwsystem's atomic_read_path for efficient path-based access.
        # This delegates to format-specific serializers which may use streaming
        # or partial reads for large files.
        try:
            auto_serializer = self._ensure_serializer()
            pointer = self._dot_path_to_pointer(path)
            # Get format-specific serializer for atomic operations
            from pathlib import Path
            path_obj = Path(source_path)
            format_name = self._detect_format_fast(path_obj, None)
            specialized_serializer = auto_serializer._get_serializer(format_name)
            # Try atomic_read_path first (format-specific optimizations)
            try:
                value = specialized_serializer.atomic_read_path(source_path, pointer)
                return value
            except (NotImplementedError, AttributeError):
                # Fallback: full-load and navigate (correct but less efficient)
                # This also materializes the data and creates XWNode for future queries
                full_node = await self._full_pipeline_load(path_obj, None)
                # Update the original node with loaded data so XWNode is available
                node._data = full_node._data
                node._xwnode = full_node._xwnode
                node._metadata.update(full_node.metadata)
                return full_node.get_value_at_path(path, default)
        except FileNotFoundError:
            raise XWDataFileNotFoundError(source_path)
        except KeyError:
            return default
        except Exception as e:
            logger.warning(f"lazy_get_from_file failed for {source_path} @ {path}: {e}")
            return default

    async def lazy_set_in_file(
        self,
        node: XWDataNode,
        path: str,
        value: Any
    ) -> XWDataNode:
        """
        Lazily update a value at the given path using xwsystem atomic_update_path.
        Returns a new XWDataNode with metadata updated; data is not fully loaded,
        keeping the node in lazy mode for subsequent operations.
        """
        source_path = node.metadata.get('source_path')
        if not source_path:
            # No backing file; fall back to in-memory COW set
            return node.set_value_at_path(path, value)
        # Use xwsystem's atomic_update_path for efficient path-based updates.
        # This delegates to format-specific serializers which may use streaming
        # or partial writes for large files.
        try:
            auto_serializer = self._ensure_serializer()
            pointer = self._dot_path_to_pointer(path)
            # Get format-specific serializer for atomic operations
            from pathlib import Path
            path_obj = Path(source_path)
            format_name = self._detect_format_fast(path_obj, None)
            specialized_serializer = auto_serializer._get_serializer(format_name)
            # Try atomic_update_path first (format-specific optimizations)
            try:
                specialized_serializer.atomic_update_path(source_path, pointer, value)
                # Atomic update succeeded; return a fresh lazy-style node
                # so that subsequent accesses continue to treat this as file-backed lazy data.
                new_metadata = node.metadata.copy()
                new_metadata.setdefault('source_path', source_path)
                new_metadata.setdefault('lazy_mode', node.metadata.get('lazy_mode', 'file'))
                new_metadata['last_lazy_update_path'] = path
                return await self._node_factory.create_node(
                    data=None,
                    metadata=new_metadata,
                    config=self._config
                )
            except (NotImplementedError, AttributeError, TypeError, ValueError) as e:
                # Fallback: full-load, COW set, full-save (correct but less efficient)
                # This handles cases where atomic_update_path is not supported or fails
                logger.debug(f"atomic_update_path not available or failed, using full-load fallback: {e}")
                full_node = await self._full_pipeline_load(path_obj, None)
                updated = full_node.set_value_at_path(path, value)
                await self.save(updated, source_path)
                # Return fresh lazy-style node
                new_metadata = updated.metadata.copy()
                new_metadata.setdefault('source_path', source_path)
                new_metadata.setdefault('lazy_mode', node.metadata.get('lazy_mode', 'file'))
                return await self._node_factory.create_node(
                    data=None,
                    metadata=new_metadata,
                    config=self._config
                )
        except FileNotFoundError:
            raise XWDataFileNotFoundError(source_path)
        except Exception as e:
            logger.warning(f"lazy_set_in_file failed for {source_path} @ {path}: {e}")
            # Last resort fallback: try full-load + save
            try:
                full_node = await self._full_pipeline_load(path_obj, None)
                updated = full_node.set_value_at_path(path, value)
                await self.save(updated, source_path)
                new_metadata = updated.metadata.copy()
                new_metadata.setdefault('source_path', source_path)
                new_metadata.setdefault('lazy_mode', node.metadata.get('lazy_mode', 'file'))
                return await self._node_factory.create_node(
                    data=None,
                    metadata=new_metadata,
                    config=self._config
                )
            except Exception as fallback_error:
                logger.error(f"Fallback also failed: {fallback_error}")
                # Best effort: return original node unchanged
                return node

    async def get_page_from_file(
        self,
        node: XWDataNode,
        page_number: int,
        page_size: int
    ) -> list[XWDataNode]:
        """
        Retrieve a page of records from a large file-backed dataset.
        This uses xwsystem.io.serialization's record-level paging API
        (ISerialization.get_record_page), which delegates to format-specific
        serializers for efficient streaming paging (e.g., JSONL) or falls back
        to full-load paging for formats that don't support efficient paging.
        For JSONL files, this leverages JsonLinesSerializer.get_record_page()
        which performs true streaming (line-by-line) without loading the entire
        file into memory. For other formats, it uses the generic ASerialization
        default which loads and slices, or format-specific overrides if available.
        """
        source_path = node.metadata.get('source_path')
        if not source_path:
            raise XWDataEngineError(
                "Paging from file requires a file-backed XWDataNode with source_path metadata",
                operation='get_page'
            )
        # Use xwsystem's get_record_page API, which delegates to specialized
        # serializers (e.g., JsonLinesSerializer for JSONL) for efficient
        # streaming paging, or falls back to full-load + slice for other formats.
        auto_serializer = self._ensure_serializer()
        # Get format-specific serializer for record-level operations
        from pathlib import Path
        path_obj = Path(source_path)
        format_name = self._detect_format_fast(path_obj, None)
        specialized_serializer = auto_serializer._get_serializer(format_name)
        try:
            # Delegate to specialized serializer's record-level paging API
            records = specialized_serializer.get_record_page(
                source_path,
                page_number=page_number,
                page_size=page_size
            )
        except (NotImplementedError, AttributeError) as e:
            # Fallback: full-load and page in memory (correct but less efficient)
            logger.debug(f"get_record_page not available, using full-load fallback: {e}")
            from pathlib import Path
            full_node = await self._full_pipeline_load(Path(source_path), None)
            data = full_node.to_native()
            if not isinstance(data, list):
                raise XWDataEngineError(
                    "In-memory paging requires top-level list data",
                    path=str(source_path),
                    operation='get_page'
                )
            start = max((page_number - 1) * page_size, 0)
            end = start + page_size
            records = data[start:end]
        except Exception as e:
            raise XWDataEngineError(
                f"Failed to get page from file: {e}",
                path=str(source_path),
                operation='get_page'
            ) from e
        # Wrap each record as an XWDataNode with appropriate metadata
        result_nodes: list[XWDataNode] = []
        for record in records:
            child_meta = {
                'source_path': str(source_path),
                'format': (node.metadata.get('format') or '').upper(),
                'detected_format': node.metadata.get('detected_format', node.metadata.get('format')),
                'page_number': page_number,
                'page_size': page_size,
            }
            child_node = await self._node_factory.create_node(
                data=record,
                metadata=child_meta,
                config=self._config
            )
            result_nodes.append(child_node)
        return result_nodes

    def _dot_path_to_pointer(self, path: str) -> str:
        """
        Convert dot-separated XWData path to JSON Pointer-style path.
        Example:
            "user.name"      -> "/user/name"
            "items.0.value"  -> "/items/0/value"
        This is used for atomic_read_path / atomic_update_path which expect
        pointer-like paths. It is format-agnostic; format-specific serializers
        may further interpret the path according to their own rules.
        """
        if not path:
            return "/"
        parts = path.split(".")
        # Basic numeric detection for list indices
        pointer_parts = []
        for part in parts:
            pointer_parts.append(part)
        return "/" + "/".join(pointer_parts)

    def _select_load_strategy(self, file_size_mb: float) -> LoadStrategy:
        """
        Select optimal loading strategy based on file size (V8).
        Strategy selection (configurable thresholds):
        - < 1MB: FULL (ultra-fast path, all in memory)
        - < 50MB: LAZY (defer until accessed)
        - < 500MB: PARTIAL (ijson, JSON Pointer)
        - > 500MB: STREAMING (constant memory)
        Args:
            file_size_mb: File size in megabytes
        Returns:
            Optimal load strategy
        """
        if file_size_mb < self._config.thresholds.small_mb:
            return LoadStrategy.FULL
        elif file_size_mb < self._config.thresholds.medium_mb:
            return LoadStrategy.LAZY
        elif file_size_mb < self._config.thresholds.large_mb:
            return LoadStrategy.PARTIAL
        else:
            return LoadStrategy.STREAMING

    def _should_use_partial_access(self, file_size_mb: float) -> bool:
        """
        Determine if partial access should be used (V8).
        Args:
            file_size_mb: File size in megabytes
        Returns:
            True if partial access should be enabled
        """
        # If explicitly disabled, don't use it
        if not self._config.partial.auto_enable_on_size:
            return self._config.partial.enable_partial_read
        # Auto-enable if file exceeds threshold
        return file_size_mb >= self._config.partial.partial_threshold_mb
    # ==========================================================================
    # FAST PATH OPTIMIZATIONS (xData-Old Style)
    # ==========================================================================

    async def _hyper_fast_json_load(self, path_obj: Path) -> XWDataNode:
        """
        Hyper-fast path for tiny JSON files (< 1KB) - V8 optimization.
        Uses xwsystem JsonSerializer (get_serializer flyweight) for one parser stack:
        1. Direct file read (sync)
        2. xwsystem JsonSerializer.decode (same stack as json_utils, catalog, indexing)
        3. Minimal metadata (4 fields only)
        4. Direct node creation (no factory)
        5. Skip XWNode (bypass graph creation)
        """
        # 1. Direct read (sync for tiny files)
        content = path_obj.read_text(encoding='utf-8')
        # 2. xwsystem JsonSerializer (flyweight)
        data = _JSON_SERIALIZER.decode(content)
        # 3. Absolute minimal metadata (4 fields only - less than V7)
        metadata = {
            'source_path': str(path_obj),
            'format': 'JSON',
            'detected_format': 'JSON',
            'hyper_fast_path': True,
            # Detection metadata for GUIDE_TEST / detection APIs
            # Tiny JSON files with .json extension are unambiguous → 100% confidence.
            'detection_confidence': 1.0,
            'detection_method': 'extension',
            'format_candidates': {'JSON': 1.0},
            '_raw_text': content,
        }
        # 4. Direct node creation (bypass factory completely)
        from .node import XWDataNode
        node = XWDataNode(
            data=data,
            metadata=metadata,
            config=self._config
        )
        # 5. Skip XWNode (major speedup)
        node._xwnode = None
        logger.debug(f"🚀🚀 Hyper-fast JSON: {path_obj}")
        return node

    async def _ultra_fast_load(
        self, 
        path_obj: Path, 
        format_hint: str | None = None
    ) -> XWDataNode:
        """
        Ultra-fast path for very small files (< 1KB) - minimal overhead.
        This method achieves V6-level performance by:
        1. Direct file read (synchronous)
        2. Direct format parsing (bypass serializer overhead)
        3. Minimal metadata (only essential fields)
        4. Direct node creation (no factory overhead)
        Supported formats: JSON, YAML, XML, TOML, BSON, CSV
        Expected performance: Match V6's 0.1-0.2ms for small files.
        """
        try:
            # 1. Direct file read (synchronous for tiny files)
            content = path_obj.read_text(encoding='utf-8')
            # 2. Detect format quickly (O(1) lookup)
            format_name = self._detect_format_fast(path_obj, format_hint)
            # 3. Direct format parsing (bypass serializer for maximum speed)
            data = None
            parse_success = False
            try:
                if format_name == 'JSON':
                    data = _JSON_SERIALIZER.decode(content)
                    parse_success = True
                elif format_name == 'YAML':
                    data = YamlSerializer().decode(content)
                    parse_success = True
                elif format_name == 'XML':
                    data = XmlSerializer().decode(content)
                    parse_success = True
                elif format_name == 'TOML':
                    data = TomlSerializer().decode(content)
                    parse_success = True
                elif format_name == 'BSON':
                    # BsonSerializer expects bytes
                    bson_content = content.encode() if isinstance(content, str) else content
                    data = BsonSerializer().decode(bson_content)
                    parse_success = True
                elif format_name == 'CSV':
                    data = CsvSerializer().decode(content)
                    parse_success = True
            except Exception:
                parse_success = False
            # Fallback to serializer if direct parsing failed
            if not parse_success:
                serializer = self._ensure_serializer()
                data = serializer.detect_and_deserialize(content, format_hint=format_name)
            # 4. Minimal metadata (only essential fields)
            metadata = {
                'source_path': str(path_obj),
                'format': format_name,
                'detected_format': format_name,
                'size_bytes': len(content),
                'ultra_fast_path': True,
                'direct_parse': parse_success,
                '_raw_text': content,
            }
            # 5. Minimal node creation (ultra-minimal overhead)
            from .node import XWDataNode
            node = XWDataNode(
                data=data,
                metadata=metadata,
                config=self._config
            )
            # Skip XWNode initialization for ultra-fast path (major performance gain)
            node._xwnode = None  # Bypass XWNode creation entirely
            logger.debug(f"🚀 Ultra-fast path completed: {path_obj} (format: {format_name}, direct: {parse_success})")
            return node
        except Exception as e:
            logger.warning(f"Ultra-fast path failed for {path_obj}, falling back to fast path: {e}")
            return await self._fast_load_small(path_obj, format_hint)

    def _xml_to_dict(self, element) -> dict:
        """
        Convert XML element to dictionary (simple conversion for ultra-fast path).
        This is a lightweight conversion optimized for speed.
        For complex XML, the full pipeline will be used.
        """
        result = {}
        # Add attributes
        if element.attrib:
            result['@attributes'] = element.attrib
        # Add text content
        if element.text and element.text.strip():
            if len(element) == 0:  # Leaf node
                return element.text.strip()
            result['@text'] = element.text.strip()
        # Add children
        for child in element:
            child_data = self._xml_to_dict(child)
            if child.tag in result:
                # Handle multiple children with same tag
                if not isinstance(result[child.tag], list):
                    result[child.tag] = [result[child.tag]]
                result[child.tag].append(child_data)
            else:
                result[child.tag] = child_data
        return result if result else None

    async def _fast_load_small(
        self, 
        path_obj: Path, 
        format_hint: str | None = None
    ) -> XWDataNode:
        """
        Fast path for small files - bypass full pipeline.
        This method replicates xData-Old's simplicity for small files:
        1. Direct file read (no async overhead for tiny files)
        2. Direct format detection (no complex detection)
        3. Direct deserialization (no strategy overhead)
        4. Minimal metadata extraction
        5. No reference resolution (too expensive for small files)
        6. Direct node creation
        Expected performance: Match xData-Old's 0.1ms for small JSON files.
        """
        try:
            # 1. Direct file read (synchronous for small files)
            content = path_obj.read_text(encoding='utf-8')
            # 2. Fast format detection (use module-level cache - O(1))
            format_name = self._detect_format_fast(path_obj, format_hint)
            # 3. Direct deserialization (no strategy overhead)
            serializer = self._ensure_serializer()
            try:
                data = serializer.detect_and_deserialize(content, format_hint=format_name)
            except Exception as e:
                # Fallback to JSON if format detection failed
                logger.debug(f"Format {format_name} failed, falling back to JSON: {e}")
                data = serializer.detect_and_deserialize(content, format_hint='JSON')
            # 4. Minimal metadata extraction
            metadata = {
                'source_path': str(path_obj),
                'format': format_name,
                'detected_format': format_name,  # For get_detected_format()
                'size_bytes': len(content),
                'fast_path': True,
                '_raw_text': content,
            }
            # 5. Direct node creation (no reference resolution)
            node = await self._node_factory.create_node(
                data=data,
                metadata=metadata,
                config=self._config
            )
            logger.debug(f"Fast path completed for {path_obj} ({len(content)} bytes)")
            return node
        except Exception as e:
            logger.error(f"Fast path failed for {path_obj}: {e}")
            # Fallback to full pipeline
            logger.debug(f"Falling back to full pipeline for {path_obj}")
            return await self._full_pipeline_load(path_obj, format_hint)

    async def _full_pipeline_load(
        self, 
        path_obj: Path, 
        format_hint: str | None = None
    ) -> XWDataNode:
        """
        Full pipeline load - the original load method logic.
        This is the fallback when fast path fails or is disabled.
        """
        # This contains the original load method logic
        # (the rest of the current load method)
        serializer = self._ensure_serializer()
        # Read file content (async)
        content = await async_safe_read_text(str(path_obj))
        # Detect format with confidence scores (before deserialization)
        format_info = await self._detect_format(path_obj, content, format_hint)
        format_name = format_info['format']
        # Deserialize via xwsystem (reuse!)
        try:
            data = serializer.detect_and_deserialize(content, format_hint=format_name)
        except Exception as e:
            raise XWDataParseError(f"Failed to deserialize {path_obj}: {e}") from e
        # Resolve references if enabled (V7 optimization: skip if disabled)
        if self._config.reference.resolution_mode.name not in ('DISABLED', 'DETECT_ONLY'):
            try:
                resolver = self._ensure_reference_resolver()
                strategy = self._strategies.get_strategy(format_name)
                base_path = path_obj.parent if path_obj else None
                # Resolve references recursively
                data = await resolver.resolve(
                    data=data,
                    strategy=strategy,
                    base_path=base_path
                )
                logger.debug(f"Reference resolution completed for {path_obj}")
            except Exception as e:
                # Reference resolution is optional - log error but continue
                logger.warning(f"Reference resolution failed for {path_obj}: {e}")
                # Continue with unresolved data
        else:
            # V7 optimization: Skip reference resolution entirely when disabled
            logger.debug(f"Reference resolution disabled for {path_obj}")
        # Extract basic metadata
        metadata = {
            'source_path': str(path_obj),
            'format': format_name,
            'detected_format': format_name,  # For get_detected_format()
            'size_bytes': len(content),
            'fast_path': False,
            'references_resolved': self._config.reference.resolution_mode.name != 'DISABLED',
            '_raw_text': content,
        }
        # Add format detection info from detector
        metadata.update(format_info)
        # Normalize detection metadata keys so that XWData facade helpers
        # (get_detection_confidence, get_detection_info, etc.) see a consistent schema.
        # Detector returns: confidence, method, candidates
        # Facade expects: detection_confidence, detection_method, format_candidates
        if 'confidence' in metadata and 'detection_confidence' not in metadata:
            metadata['detection_confidence'] = metadata['confidence']
        if 'method' in metadata and 'detection_method' not in metadata:
            metadata['detection_method'] = metadata['method']
        if 'candidates' in metadata and 'format_candidates' not in metadata:
            metadata['format_candidates'] = metadata['candidates']
        # Create XWDataNode
        node = await self._node_factory.create_node(
            data=data,
            metadata=metadata,
            config=self._config
        )
        # Caching is handled by the caller (load method)
        return node

    async def _detect_format(
        self, 
        path_obj: Path, 
        content: str, 
        format_hint: str | None = None
    ) -> dict[str, Any]:
        """
        Detect format with confidence scores.
        Args:
            path_obj: File path
            content: File content
            format_hint: Optional format hint
        Returns:
            Format information dictionary
        """
        if format_hint:
            return {
                'format': format_hint.upper(),
                'confidence': 1.0,
                'method': 'hint'
            }
        # Use xwsystem's format detector (corrected import path)
        from exonware.xwsystem.io.serialization.format_detector import FormatDetector
        detector = FormatDetector()
        format_scores = detector.detect_format(
            file_path=path_obj,
            content=content
        )
        if format_scores:
            detected_format = max(format_scores, key=format_scores.get)
            confidence = format_scores[detected_format]
            return {
                'format': detected_format,
                'confidence': confidence,
                'method': 'content',
                'candidates': format_scores
            }
        # Fallback to JSON
        return {
            'format': 'JSON',
            'confidence': 0.5,
            'method': 'fallback'
        }
__all__ = ['XWDataEngine']
