"""Cache node for high-performance data caching and retrieval.

This module provides comprehensive caching capabilities supporting multiple
backends including Redis, in-memory LRU, and file-based caching with
advanced features like TTL, compression, and serialization.
"""

import asyncio
import gzip
import hashlib
import json
import logging
import time
from datetime import UTC, datetime, timedelta
from enum import Enum
from typing import Any, Dict, List, Optional, Union

from kailash.nodes.base import NodeParameter, register_node
from kailash.nodes.base_async import AsyncNode
from kailash.sdk_exceptions import NodeExecutionError
from kailash.utils.redis_validation import validate_redis_url

try:
    import redis.asyncio as redis

    REDIS_AVAILABLE = True
except ImportError:
    REDIS_AVAILABLE = False

logger = logging.getLogger(__name__)


class _PickleDeserializationError(Exception):
    """Raised when legacy pickle data is encountered.

    Pickle deserialization has been removed due to Remote Code Execution (RCE)
    risk. Data previously stored with pickle serialization must be re-serialized
    using JSON format.
    """

    pass


class CacheBackend(Enum):
    """Supported cache backend types."""

    MEMORY = "memory"
    REDIS = "redis"
    FILE = "file"
    HYBRID = "hybrid"  # Memory + Redis fallback


class SerializationFormat(Enum):
    """Data serialization formats."""

    JSON = "json"
    PICKLE = "pickle"
    STRING = "string"
    BYTES = "bytes"


class EvictionPolicy(Enum):
    """Cache eviction policies."""

    LRU = "lru"  # Least Recently Used
    LFU = "lfu"  # Least Frequently Used
    TTL = "ttl"  # Time To Live only
    FIFO = "fifo"  # First In, First Out


@register_node()
class CacheNode(AsyncNode):
    """Node for high-performance data caching and retrieval.

    This node provides comprehensive caching capabilities including:
    - Multiple backend support (Redis, in-memory, file-based, hybrid)
    - Configurable TTL (Time To Live) with automatic expiration
    - Multiple serialization formats (JSON, Pickle, String, Bytes)
    - Data compression for large values
    - Eviction policies (LRU, LFU, TTL, FIFO)
    - Atomic operations and transactions
    - Cache statistics and monitoring
    - Distributed caching with Redis
    - Fallback strategies for high availability

    Design Purpose:
    - Improve application performance through intelligent caching
    - Reduce database and API load
    - Provide configurable caching strategies
    - Support both simple and complex caching scenarios

    Examples:
        >>> # Simple key-value caching
        >>> cache = CacheNode()
        >>> result = await cache.execute(
        ...     operation="set",
        ...     key="user:123",
        ...     value={"name": "John", "email": "john@example.com"},
        ...     ttl=3600  # 1 hour
        ... )

        >>> # Batch operations with pattern matching
        >>> result = await cache.execute(
        ...     operation="get_pattern",
        ...     pattern="user:*",
        ...     limit=100
        ... )

        >>> # Cache with compression and JSON serialization
        >>> result = await cache.execute(
        ...     operation="set",
        ...     key="large_data",
        ...     value=large_dataset,
        ...     compression=True,
        ...     serialization="json",
        ...     ttl=86400  # 24 hours
        ... )
    """

    def __init__(self, **kwargs):
        """Initialize the cache node."""
        super().__init__(**kwargs)
        self._memory_cache = {}
        self._access_times = {}
        self._access_counts = {}
        self._redis_client = None
        self._cache_stats = {
            "hits": 0,
            "misses": 0,
            "sets": 0,
            "deletes": 0,
            "evictions": 0,
        }
        self.logger.info(f"Initialized CacheNode: {self.id}")

    def get_parameters(self) -> Dict[str, NodeParameter]:
        """Define the parameters this node accepts."""
        return {
            "operation": NodeParameter(
                name="operation",
                type=str,
                required=True,
                description="Cache operation (get, set, delete, exists, clear, stats, get_pattern)",
            ),
            "key": NodeParameter(
                name="key",
                type=str,
                required=False,
                description="Cache key for single operations",
            ),
            "value": NodeParameter(
                name="value",
                type=Any,
                required=False,
                description="Value to cache (for set operations)",
            ),
            "keys": NodeParameter(
                name="keys",
                type=list,
                required=False,
                description="Multiple keys for batch operations",
            ),
            "values": NodeParameter(
                name="values",
                type=dict,
                required=False,
                description="Key-value pairs for batch set operations",
            ),
            "pattern": NodeParameter(
                name="pattern",
                type=str,
                required=False,
                description="Pattern for pattern-based operations (supports wildcards)",
            ),
            "ttl": NodeParameter(
                name="ttl",
                type=int,
                required=False,
                default=3600,
                description="Time to live in seconds (0 = no expiration)",
            ),
            "backend": NodeParameter(
                name="backend",
                type=str,
                required=False,
                default="memory",
                description="Cache backend (memory, redis, file, hybrid)",
            ),
            "redis_url": NodeParameter(
                name="redis_url",
                type=str,
                required=False,
                default="redis://localhost:6379",
                description="Redis connection URL",
            ),
            "serialization": NodeParameter(
                name="serialization",
                type=str,
                required=False,
                default="json",
                description="Serialization format (json, pickle, string, bytes)",
            ),
            "compression": NodeParameter(
                name="compression",
                type=bool,
                required=False,
                default=False,
                description="Enable gzip compression for large values",
            ),
            "compression_threshold": NodeParameter(
                name="compression_threshold",
                type=int,
                required=False,
                default=1024,
                description="Minimum size in bytes to trigger compression",
            ),
            "eviction_policy": NodeParameter(
                name="eviction_policy",
                type=str,
                required=False,
                default="lru",
                description="Eviction policy (lru, lfu, ttl, fifo)",
            ),
            "max_memory_items": NodeParameter(
                name="max_memory_items",
                type=int,
                required=False,
                default=10000,
                description="Maximum items in memory cache",
            ),
            "namespace": NodeParameter(
                name="namespace",
                type=str,
                required=False,
                default="",
                description="Key namespace prefix",
            ),
        }

    def get_output_schema(self) -> Dict[str, NodeParameter]:
        """Define the output schema for this node."""
        return {
            "success": NodeParameter(
                name="success",
                type=bool,
                description="Whether the operation succeeded",
            ),
            "value": NodeParameter(
                name="value",
                type=Any,
                required=False,
                description="Retrieved value (for get operations)",
            ),
            "values": NodeParameter(
                name="values",
                type=dict,
                required=False,
                description="Multiple values (for batch operations)",
            ),
            "hit": NodeParameter(
                name="hit",
                type=bool,
                required=False,
                description="Cache hit status (for get operations)",
            ),
            "key": NodeParameter(
                name="key",
                type=str,
                required=False,
                description="The cache key used",
            ),
            "ttl_remaining": NodeParameter(
                name="ttl_remaining",
                type=int,
                required=False,
                description="Remaining TTL in seconds",
            ),
            "backend_used": NodeParameter(
                name="backend_used",
                type=str,
                description="Backend that handled the operation",
            ),
            "operation_time": NodeParameter(
                name="operation_time",
                type=float,
                description="Time taken for the operation",
            ),
            "stats": NodeParameter(
                name="stats",
                type=dict,
                required=False,
                description="Cache statistics (for stats operation)",
            ),
            "compressed": NodeParameter(
                name="compressed",
                type=bool,
                required=False,
                description="Whether the value was compressed",
            ),
        }

    async def async_run(self, **kwargs) -> Dict[str, Any]:
        """Execute cache operations."""
        operation = kwargs["operation"].lower()
        backend = CacheBackend(kwargs.get("backend", "memory"))

        start_time = time.time()

        try:
            # Initialize backend if needed
            await self._ensure_backend(backend, kwargs)

            # Execute operation
            if operation == "get":
                result = await self._get(kwargs)
            elif operation == "set":
                result = await self._set(kwargs)
            elif operation == "delete":
                result = await self._delete(kwargs)
            elif operation == "exists":
                result = await self._exists(kwargs)
            elif operation == "clear":
                result = await self._clear(kwargs)
            elif operation == "stats":
                result = await self._get_stats(kwargs)
            elif operation == "get_pattern":
                result = await self._get_pattern(kwargs)
            elif operation == "mget":
                result = await self._mget(kwargs)
            elif operation == "mset":
                result = await self._mset(kwargs)
            else:
                raise ValueError(f"Unsupported operation: {operation}")

            operation_time = time.time() - start_time
            result["operation_time"] = operation_time
            result["backend_used"] = backend.value

            return result

        except Exception as e:
            self.logger.error(f"Cache operation failed: {str(e)}")
            raise NodeExecutionError(f"Cache operation '{operation}' failed: {str(e)}")

        finally:
            # Clean up Redis connection after operation
            if self._redis_client:
                try:
                    await self._redis_client.aclose()
                except Exception:
                    pass  # Ignore cleanup errors
                self._redis_client = None

    async def _ensure_backend(self, backend: CacheBackend, kwargs: Dict[str, Any]):
        """Ensure the cache backend is initialized with proper connection management."""
        if backend in [CacheBackend.REDIS, CacheBackend.HYBRID]:
            if not REDIS_AVAILABLE:
                if backend == CacheBackend.REDIS:
                    raise NodeExecutionError(
                        "Redis is not available. Install with: pip install redis"
                    )
                else:
                    # Fall back to memory for hybrid mode
                    self.logger.warning("Redis not available, using memory cache only")
                    return

            # Create fresh Redis client for each operation to avoid event loop issues
            redis_url = kwargs.get("redis_url", "redis://localhost:6379")
            # Validate Redis URL scheme to prevent SSRF (H4)
            validate_redis_url(redis_url)
            try:
                # Create a new client for this operation
                redis_client = redis.from_url(redis_url, decode_responses=False)  # type: ignore[possibly-unbound]
                # Test connection
                await redis_client.ping()  # type: ignore[misc]

                # Store the client for this operation
                self._redis_client = redis_client

            except Exception as e:
                if backend == CacheBackend.REDIS:
                    raise NodeExecutionError(f"Failed to connect to Redis: {str(e)}")
                else:
                    # Fall back to memory for hybrid mode
                    self.logger.warning(
                        f"Redis connection failed, using memory cache: {str(e)}"
                    )
                    self._redis_client = None

    async def _close_redis_connection(self):
        """Close Redis connection if it exists."""
        if self._redis_client:
            try:
                await self._redis_client.aclose()
            except Exception:
                pass  # Ignore errors during cleanup
            self._redis_client = None

    def _build_key(self, key: str, namespace: str = "") -> str:
        """Build a namespaced cache key."""
        if namespace:
            return f"{namespace}:{key}"
        return key

    async def _get(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Get a value from cache."""
        key = kwargs["key"]
        namespace = kwargs.get("namespace", "")
        backend = CacheBackend(kwargs.get("backend", "memory"))
        serialization = SerializationFormat(kwargs.get("serialization", "json"))

        full_key = self._build_key(key, namespace)

        try:
            if backend == CacheBackend.REDIS and self._redis_client:
                value, hit = await self._redis_get(full_key, serialization, kwargs)
            elif backend == CacheBackend.HYBRID:
                # Try memory first, then Redis
                value, hit = await self._memory_get(full_key, serialization, kwargs)
                if hit and self._redis_client:
                    # Check if key still exists in Redis (might have been invalidated)
                    redis_exists = await self._redis_client.exists(full_key)
                    if not redis_exists:
                        # Key was invalidated in Redis, remove from memory cache
                        self._memory_cache.pop(full_key, None)
                        self._access_times.pop(full_key, None)
                        self._access_counts.pop(full_key, None)
                        value, hit = None, False
                elif not hit and self._redis_client:
                    value, hit = await self._redis_get(full_key, serialization, kwargs)
                    # Cache in memory for next access
                    if hit:
                        await self._memory_set(full_key, value, kwargs)
            else:
                value, hit = await self._memory_get(full_key, serialization, kwargs)

            if hit:
                self._cache_stats["hits"] += 1
            else:
                self._cache_stats["misses"] += 1

            return {
                "success": True,
                "value": value,
                "hit": hit,
                "key": full_key,
            }

        except Exception as e:
            self.logger.error(f"Cache get failed for key '{full_key}': {str(e)}")
            return {
                "success": False,
                "value": None,
                "hit": False,
                "key": full_key,
                "error": str(e),
            }

    async def _set(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Set a value in cache."""
        key = kwargs["key"]
        value = kwargs["value"]
        namespace = kwargs.get("namespace", "")
        backend = CacheBackend(kwargs.get("backend", "memory"))
        ttl = kwargs.get("ttl", 3600)

        full_key = self._build_key(key, namespace)

        try:
            if backend == CacheBackend.REDIS and self._redis_client:
                success = await self._redis_set(full_key, value, ttl, kwargs)
            elif backend == CacheBackend.HYBRID:
                # Set in both memory and Redis
                success1 = await self._memory_set(full_key, value, kwargs)
                success2 = True
                if self._redis_client:
                    success2 = await self._redis_set(full_key, value, ttl, kwargs)
                success = success1 and success2
            else:
                success = await self._memory_set(full_key, value, kwargs)

            if success:
                self._cache_stats["sets"] += 1

            return {
                "success": success,
                "key": full_key,
                "ttl_remaining": ttl if ttl > 0 else -1,
            }

        except Exception as e:
            self.logger.error(f"Cache set failed for key '{full_key}': {str(e)}")
            return {
                "success": False,
                "key": full_key,
                "error": str(e),
            }

    async def _delete(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Delete a value from cache."""
        key = kwargs["key"]
        namespace = kwargs.get("namespace", "")
        backend = CacheBackend(kwargs.get("backend", "memory"))

        full_key = self._build_key(key, namespace)

        try:
            deleted = False

            if backend == CacheBackend.REDIS and self._redis_client:
                deleted = bool(await self._redis_client.delete(full_key))
            elif backend == CacheBackend.HYBRID:
                # Delete from both
                mem_deleted = full_key in self._memory_cache
                if mem_deleted:
                    del self._memory_cache[full_key]
                    del self._access_times[full_key]
                    self._access_counts.pop(full_key, None)

                redis_deleted = False
                if self._redis_client:
                    redis_deleted = bool(await self._redis_client.delete(full_key))

                deleted = mem_deleted or redis_deleted
            else:
                if full_key in self._memory_cache:
                    del self._memory_cache[full_key]
                    del self._access_times[full_key]
                    self._access_counts.pop(full_key, None)
                    deleted = True

            if deleted:
                self._cache_stats["deletes"] += 1

            return {
                "success": True,
                "deleted": deleted,
                "key": full_key,
            }

        except Exception as e:
            self.logger.error(f"Cache delete failed for key '{full_key}': {str(e)}")
            return {
                "success": False,
                "deleted": False,
                "key": full_key,
                "error": str(e),
            }

    async def _exists(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Check if a key exists in cache."""
        key = kwargs["key"]
        namespace = kwargs.get("namespace", "")
        backend = CacheBackend(kwargs.get("backend", "memory"))

        full_key = self._build_key(key, namespace)

        try:
            exists = False

            if backend == CacheBackend.REDIS and self._redis_client:
                exists = bool(await self._redis_client.exists(full_key))
            elif backend == CacheBackend.HYBRID:
                exists = full_key in self._memory_cache or (
                    self._redis_client
                    and bool(await self._redis_client.exists(full_key))
                )
            else:
                exists = full_key in self._memory_cache

            return {
                "success": True,
                "exists": exists,
                "key": full_key,
            }

        except Exception as e:
            self.logger.error(
                f"Cache exists check failed for key '{full_key}': {str(e)}"
            )
            return {
                "success": False,
                "exists": False,
                "key": full_key,
                "error": str(e),
            }

    async def _clear(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Clear all cache entries."""
        backend = CacheBackend(kwargs.get("backend", "memory"))
        namespace = kwargs.get("namespace", "")

        try:
            cleared_count = 0

            if backend == CacheBackend.REDIS and self._redis_client:
                if namespace:
                    # Clear only namespaced keys
                    pattern = f"{namespace}:*"
                    keys = await self._redis_client.keys(pattern)
                    if keys:
                        cleared_count = await self._redis_client.delete(*keys)
                else:
                    # Clear all
                    await self._redis_client.flushdb()
                    cleared_count = -1  # Unknown count

            elif backend == CacheBackend.HYBRID:
                # Clear memory
                if namespace:
                    mem_keys = [
                        k
                        for k in self._memory_cache.keys()
                        if k.startswith(f"{namespace}:")
                    ]
                    for k in mem_keys:
                        del self._memory_cache[k]
                        del self._access_times[k]
                        self._access_counts.pop(k, None)
                    cleared_count += len(mem_keys)
                else:
                    cleared_count += len(self._memory_cache)
                    self._memory_cache.clear()
                    self._access_times.clear()
                    self._access_counts.clear()

                # Clear Redis
                if self._redis_client:
                    if namespace:
                        pattern = f"{namespace}:*"
                        keys = await self._redis_client.keys(pattern)
                        if keys:
                            cleared_count += await self._redis_client.delete(*keys)
                    else:
                        await self._redis_client.flushdb()

            else:
                # Memory cache
                if namespace:
                    mem_keys = [
                        k
                        for k in self._memory_cache.keys()
                        if k.startswith(f"{namespace}:")
                    ]
                    for k in mem_keys:
                        del self._memory_cache[k]
                        del self._access_times[k]
                        self._access_counts.pop(k, None)
                    cleared_count = len(mem_keys)
                else:
                    cleared_count = len(self._memory_cache)
                    self._memory_cache.clear()
                    self._access_times.clear()
                    self._access_counts.clear()

            return {
                "success": True,
                "cleared_count": cleared_count,
            }

        except Exception as e:
            self.logger.error(f"Cache clear failed: {str(e)}")
            return {
                "success": False,
                "cleared_count": 0,
                "error": str(e),
            }

    async def _get_stats(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Get cache statistics."""
        backend = CacheBackend(kwargs.get("backend", "memory"))

        try:
            stats = dict(self._cache_stats)

            if backend == CacheBackend.MEMORY or backend == CacheBackend.HYBRID:
                stats["memory_items"] = len(self._memory_cache)
                stats["memory_size_bytes"] = sum(
                    len(str(v).encode()) for v in self._memory_cache.values()
                )

            if (
                backend == CacheBackend.REDIS or backend == CacheBackend.HYBRID
            ) and self._redis_client:
                redis_info = await self._redis_client.info()
                stats["redis_connected"] = True
                stats["redis_memory"] = redis_info.get("used_memory", 0)
                stats["redis_keys"] = redis_info.get("db0", {}).get("keys", 0)
            else:
                stats["redis_connected"] = False

            # Calculate hit rate
            total_reads = stats["hits"] + stats["misses"]
            stats["hit_rate"] = stats["hits"] / total_reads if total_reads > 0 else 0  # type: ignore[assignment]

            return {
                "success": True,
                "stats": stats,
            }

        except Exception as e:
            self.logger.error(f"Failed to get cache stats: {str(e)}")
            return {
                "success": False,
                "stats": {},
                "error": str(e),
            }

    async def _get_pattern(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Get values matching a pattern."""
        pattern = kwargs["pattern"]
        namespace = kwargs.get("namespace", "")
        backend = CacheBackend(kwargs.get("backend", "memory"))
        limit = kwargs.get("limit", 1000)

        if namespace:
            pattern = f"{namespace}:{pattern}"

        try:
            values = {}

            if backend == CacheBackend.REDIS and self._redis_client:
                keys = await self._redis_client.keys(pattern)
                if keys:
                    keys = keys[:limit]  # Limit results
                    raw_values = await self._redis_client.mget(keys)
                    for key, raw_value in zip(keys, raw_values):
                        if raw_value:
                            try:
                                values[
                                    key.decode() if isinstance(key, bytes) else key
                                ] = json.loads(raw_value)
                            except json.JSONDecodeError:
                                values[
                                    key.decode() if isinstance(key, bytes) else key
                                ] = (
                                    raw_value.decode()
                                    if isinstance(raw_value, bytes)
                                    else raw_value
                                )

            elif backend == CacheBackend.HYBRID:
                # Get from memory
                import fnmatch

                mem_keys = [
                    k for k in self._memory_cache.keys() if fnmatch.fnmatch(k, pattern)
                ]
                for key in mem_keys[:limit]:
                    values[key] = self._memory_cache[key]

                # Get from Redis if not enough results
                if len(values) < limit and self._redis_client:
                    remaining_limit = limit - len(values)
                    redis_keys = await self._redis_client.keys(pattern)
                    redis_keys = [k for k in redis_keys if k not in values][
                        :remaining_limit
                    ]
                    if redis_keys:
                        raw_values = await self._redis_client.mget(redis_keys)
                        for key, raw_value in zip(redis_keys, raw_values):
                            if raw_value:
                                try:
                                    values[
                                        key.decode() if isinstance(key, bytes) else key
                                    ] = json.loads(raw_value)
                                except json.JSONDecodeError:
                                    values[
                                        key.decode() if isinstance(key, bytes) else key
                                    ] = (
                                        raw_value.decode()
                                        if isinstance(raw_value, bytes)
                                        else raw_value
                                    )

            else:
                # Memory cache with fnmatch
                import fnmatch

                mem_keys = [
                    k for k in self._memory_cache.keys() if fnmatch.fnmatch(k, pattern)
                ]
                for key in mem_keys[:limit]:
                    values[key] = self._memory_cache[key]

            return {
                "success": True,
                "values": values,
                "count": len(values),
                "pattern": pattern,
            }

        except Exception as e:
            self.logger.error(f"Pattern get failed for pattern '{pattern}': {str(e)}")
            return {
                "success": False,
                "values": {},
                "count": 0,
                "pattern": pattern,
                "error": str(e),
            }

    async def _mget(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Get multiple values by keys."""
        keys = kwargs["keys"]
        namespace = kwargs.get("namespace", "")
        backend = CacheBackend(kwargs.get("backend", "memory"))

        full_keys = [self._build_key(key, namespace) for key in keys]

        try:
            values = {}
            hits = 0

            if backend == CacheBackend.REDIS and self._redis_client:
                raw_values = await self._redis_client.mget(full_keys)
                for key, raw_value in zip(full_keys, raw_values):
                    if raw_value:
                        try:
                            values[key] = json.loads(raw_value)
                            hits += 1
                        except json.JSONDecodeError:
                            values[key] = (
                                raw_value.decode()
                                if isinstance(raw_value, bytes)
                                else raw_value
                            )
                            hits += 1

            elif backend == CacheBackend.HYBRID:
                # Try memory first
                for key in full_keys:
                    if key in self._memory_cache:
                        values[key] = self._memory_cache[key]
                        hits += 1

                # Get missing keys from Redis
                missing_keys = [k for k in full_keys if k not in values]
                if missing_keys and self._redis_client:
                    raw_values = await self._redis_client.mget(missing_keys)
                    for key, raw_value in zip(missing_keys, raw_values):
                        if raw_value:
                            try:
                                value = json.loads(raw_value)
                                values[key] = value
                                # Cache in memory
                                self._memory_cache[key] = value
                                self._access_times[key] = time.time()
                                hits += 1
                            except json.JSONDecodeError:
                                value = (
                                    raw_value.decode()
                                    if isinstance(raw_value, bytes)
                                    else raw_value
                                )
                                values[key] = value
                                self._memory_cache[key] = value
                                self._access_times[key] = time.time()
                                hits += 1

            else:
                # Memory cache
                for key in full_keys:
                    if key in self._memory_cache:
                        values[key] = self._memory_cache[key]
                        hits += 1

            self._cache_stats["hits"] += hits
            self._cache_stats["misses"] += len(full_keys) - hits

            return {
                "success": True,
                "values": values,
                "hits": hits,
                "total_keys": len(full_keys),
            }

        except Exception as e:
            self.logger.error(f"Batch get failed: {str(e)}")
            return {
                "success": False,
                "values": {},
                "hits": 0,
                "total_keys": len(full_keys),
                "error": str(e),
            }

    async def _mset(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """Set multiple key-value pairs."""
        values_dict = kwargs.get("values", {})
        namespace = kwargs.get("namespace", "")
        backend = CacheBackend(kwargs.get("backend", "memory"))
        ttl = kwargs.get("ttl", 3600)

        if not values_dict:
            return {"success": False, "error": "No values provided"}

        full_values = {
            self._build_key(key, namespace): value for key, value in values_dict.items()
        }

        try:
            success_count = 0

            if backend == CacheBackend.REDIS and self._redis_client:
                # Use Redis pipeline for efficiency
                pipe = self._redis_client.pipeline()
                for key, value in full_values.items():
                    serialized = (
                        json.dumps(value) if not isinstance(value, str) else value
                    )
                    if ttl > 0:
                        pipe.setex(key, ttl, serialized)
                    else:
                        pipe.set(key, serialized)

                results = await pipe.execute()
                success_count = sum(1 for r in results if r)

            elif backend == CacheBackend.HYBRID:
                # Set in memory
                for key, value in full_values.items():
                    self._memory_cache[key] = value
                    self._access_times[key] = time.time()
                    success_count += 1

                # Set in Redis
                if self._redis_client:
                    pipe = self._redis_client.pipeline()
                    for key, value in full_values.items():
                        serialized = (
                            json.dumps(value) if not isinstance(value, str) else value
                        )
                        if ttl > 0:
                            pipe.setex(key, ttl, serialized)
                        else:
                            pipe.set(key, serialized)
                    await pipe.execute()

            else:
                # Memory cache
                for key, value in full_values.items():
                    self._memory_cache[key] = value
                    self._access_times[key] = time.time()
                    success_count += 1

            self._cache_stats["sets"] += success_count

            return {
                "success": True,
                "set_count": success_count,
                "total_keys": len(full_values),
            }

        except Exception as e:
            self.logger.error(f"Batch set failed: {str(e)}")
            return {
                "success": False,
                "set_count": 0,
                "total_keys": len(full_values),
                "error": str(e),
            }

    async def _memory_get(
        self, key: str, serialization: SerializationFormat, kwargs: Dict[str, Any]
    ) -> tuple[Any, bool]:
        """Get value from memory cache."""
        if key not in self._memory_cache:
            return None, False

        # Check TTL if stored with timestamp
        value_data = self._memory_cache[key]
        if isinstance(value_data, dict) and "_cache_timestamp" in value_data:
            timestamp = value_data["_cache_timestamp"]
            ttl = value_data.get("_cache_ttl", 0)
            if ttl > 0 and time.time() - timestamp > ttl:
                # Expired
                del self._memory_cache[key]
                del self._access_times[key]
                self._access_counts.pop(key, None)
                return None, False
            value = value_data["_cache_value"]
        else:
            value = value_data

        # Update access statistics
        self._access_times[key] = time.time()
        self._access_counts[key] = self._access_counts.get(key, 0) + 1

        # Handle decompression if needed
        if kwargs.get("compression", False) and isinstance(value, bytes):
            try:
                value = gzip.decompress(value)
                if serialization == SerializationFormat.JSON:
                    value = json.loads(value.decode())
                elif serialization == SerializationFormat.PICKLE:
                    # SECURITY: pickle deserialization removed (RCE risk - H3)
                    raise _PickleDeserializationError(
                        "Pickle deserialization is disabled due to RCE risk. "
                        "Re-serialize your data using JSON format."
                    )
            except _PickleDeserializationError:
                raise
            except Exception:
                pass  # Use value as-is if decompression fails

        return value, True

    async def _memory_set(self, key: str, value: Any, kwargs: Dict[str, Any]) -> bool:
        """Set value in memory cache."""
        try:
            ttl = kwargs.get("ttl", 0)
            compression = kwargs.get("compression", False)
            compression_threshold = kwargs.get("compression_threshold", 1024)
            max_items = kwargs.get("max_memory_items", 10000)

            # Handle eviction if needed
            if len(self._memory_cache) >= max_items:
                await self._evict_memory_items(kwargs)

            # Prepare value for storage
            stored_value = value
            compressed = False

            # Apply compression if needed
            if compression:
                serialized = (
                    json.dumps(value) if not isinstance(value, (str, bytes)) else value
                )
                if isinstance(serialized, str):
                    serialized = serialized.encode()

                if len(serialized) >= compression_threshold:
                    stored_value = gzip.compress(serialized)
                    compressed = True

            # Store with TTL if specified
            if ttl > 0:
                self._memory_cache[key] = {
                    "_cache_value": stored_value,
                    "_cache_timestamp": time.time(),
                    "_cache_ttl": ttl,
                    "_cache_compressed": compressed,
                }
            else:
                self._memory_cache[key] = stored_value

            self._access_times[key] = time.time()
            return True

        except Exception as e:
            self.logger.error(f"Memory cache set failed: {str(e)}")
            return False

    async def _redis_get(
        self, key: str, serialization: SerializationFormat, kwargs: Dict[str, Any]
    ) -> tuple[Any, bool]:
        """Get value from Redis cache."""
        try:
            raw_value = await self._redis_client.get(key)  # type: ignore[union-attr]
            if raw_value is None:
                return None, False

            # Handle decompression
            if kwargs.get("compression", False):
                try:
                    raw_value = gzip.decompress(raw_value)
                except Exception:
                    pass  # Not compressed or failed decompression

            # Deserialize based on format
            if serialization == SerializationFormat.JSON:
                value = json.loads(raw_value)
            elif serialization == SerializationFormat.PICKLE:
                # SECURITY: pickle deserialization removed (RCE risk - H3)
                raise _PickleDeserializationError(
                    "Pickle deserialization is disabled due to RCE risk. "
                    "Re-serialize your data using JSON format."
                )
            elif serialization == SerializationFormat.STRING:
                value = (
                    raw_value.decode() if isinstance(raw_value, bytes) else raw_value
                )
            else:  # BYTES
                value = raw_value

            return value, True

        except _PickleDeserializationError:
            raise
        except Exception as e:
            self.logger.error(f"Redis get failed: {str(e)}")
            return None, False

    async def _redis_set(
        self, key: str, value: Any, ttl: int, kwargs: Dict[str, Any]
    ) -> bool:
        """Set value in Redis cache."""
        try:
            serialization = SerializationFormat(kwargs.get("serialization", "json"))
            compression = kwargs.get("compression", False)
            compression_threshold = kwargs.get("compression_threshold", 1024)

            # Serialize value
            if serialization == SerializationFormat.JSON:
                serialized = json.dumps(value)
            elif serialization == SerializationFormat.PICKLE:
                # SECURITY: pickle serialization replaced with JSON (RCE risk - H3)
                logger.warning(
                    "Pickle serialization requested but disabled for security. "
                    "Using JSON serialization instead."
                )
                serialized = json.dumps(value, default=str)
            elif serialization == SerializationFormat.STRING:
                serialized = str(value)
            else:  # BYTES
                serialized = value if isinstance(value, bytes) else str(value).encode()

            # Apply compression if needed
            if compression and len(serialized) >= compression_threshold:
                if isinstance(serialized, str):
                    serialized = serialized.encode()
                serialized = gzip.compress(serialized)

            # Store in Redis
            if ttl > 0:
                await self._redis_client.setex(key, ttl, serialized)  # type: ignore[union-attr]
            else:
                await self._redis_client.set(key, serialized)  # type: ignore[union-attr]

            return True

        except Exception as e:
            self.logger.error(f"Redis set failed: {str(e)}")
            return False

    async def _evict_memory_items(self, kwargs: Dict[str, Any]):
        """Evict items from memory cache based on policy."""
        eviction_policy = EvictionPolicy(kwargs.get("eviction_policy", "lru"))
        max_items = kwargs.get("max_memory_items", 10000)

        # Remove 10% of items to make room
        evict_count = max(1, len(self._memory_cache) // 10)

        if eviction_policy == EvictionPolicy.LRU:
            # Remove least recently used
            sorted_by_access = sorted(self._access_times.items(), key=lambda x: x[1])
            for key, _ in sorted_by_access[:evict_count]:
                del self._memory_cache[key]
                del self._access_times[key]
                self._access_counts.pop(key, None)
                self._cache_stats["evictions"] += 1

        elif eviction_policy == EvictionPolicy.LFU:
            # Remove least frequently used
            sorted_by_frequency = sorted(
                self._access_counts.items(), key=lambda x: x[1]
            )
            for key, _ in sorted_by_frequency[:evict_count]:
                del self._memory_cache[key]
                del self._access_times[key]
                del self._access_counts[key]
                self._cache_stats["evictions"] += 1

        elif eviction_policy == EvictionPolicy.TTL:
            # Remove expired items first
            now = time.time()
            expired_keys = []
            for key, value_data in self._memory_cache.items():
                if isinstance(value_data, dict) and "_cache_timestamp" in value_data:
                    timestamp = value_data["_cache_timestamp"]
                    ttl = value_data.get("_cache_ttl", 0)
                    if ttl > 0 and now - timestamp > ttl:
                        expired_keys.append(key)

            for key in expired_keys:
                del self._memory_cache[key]
                del self._access_times[key]
                self._access_counts.pop(key, None)
                self._cache_stats["evictions"] += 1

        elif eviction_policy == EvictionPolicy.FIFO:
            # Remove oldest inserted items
            sorted_keys = list(self._memory_cache.keys())[:evict_count]
            for key in sorted_keys:
                del self._memory_cache[key]
                del self._access_times[key]
                self._access_counts.pop(key, None)
                self._cache_stats["evictions"] += 1

    def run(self, **kwargs) -> Dict[str, Any]:
        """Synchronous wrapper for compatibility."""
        try:
            # Try to get current event loop
            loop = asyncio.get_running_loop()
        except RuntimeError:
            # No event loop running, safe to use asyncio.run()
            return asyncio.run(self.async_run(**kwargs))
        else:
            # Event loop is running, create a task
            import concurrent.futures

            with concurrent.futures.ThreadPoolExecutor() as executor:
                future = executor.submit(asyncio.run, self.async_run(**kwargs))
                return future.result()
