"""Thread-safe telemetry queue for buffering when collector is unreachable.

This module provides a bounded queue for storing telemetry data when the
OpenTelemetry collector is temporarily unavailable.
"""

import errno
import json
import logging
import os
import queue as queue_module
import shutil
import tempfile
import threading
import time
from collections import deque
from threading import Event, Lock, Thread
from typing import Any, Dict, List, Optional, TYPE_CHECKING

from ..utils.exceptions import BufferingError

if TYPE_CHECKING:
    from ..security.encryption import QueueEncryption

logger = logging.getLogger(__name__)


class TelemetryQueue:
    """Thread-safe bounded queue for telemetry buffering.

    Provides in-memory queueing with optional disk persistence for
    telemetry data when the collector is temporarily unreachable.

    Thread Safety:
        All operations are protected by a threading.Lock to ensure
        safe concurrent access from multiple threads.

    Examples:
        >>> queue = TelemetryQueue(max_size=1000)
        >>> queue.enqueue({"metric": "model.predictions_total", "value": 1})
        >>> batch = queue.dequeue_batch(size=10)
        >>> queue.size()
        0

        With disk persistence:
        >>> queue = TelemetryQueue(
        ...     max_size=1000,
        ...     persist=True,
        ...     persist_path="~/.mca_sdk/queue.pkl"
        ... )
    """

    def __init__(
        self,
        max_size: int = 1000,
        persist: bool = False,
        persist_path: Optional[str] = None,
        min_disk_space_mb: float = 10.0,
        shutdown_timeout: float = 5.0,
        encryption: Optional["QueueEncryption"] = None,
        encryption_enabled: bool = False,
        config: Optional[Any] = None,
    ):
        """Initialize telemetry queue.

        Args:
            max_size: Maximum number of items in queue (default: 1000)
            persist: Whether to persist queue to disk (default: False)
            persist_path: Path for queue persistence (required if persist=True)
                Security: Path is validated at initialization to prevent path traversal
                attacks and unauthorized writes. Only allowed in: ~/.mca_sdk/,
                /var/lib/mca/, /tmp/mca_sdk/ (Story 3.15 - HIPAA compliance).
            min_disk_space_mb: Minimum free disk space in MB
                required for persistence (default: 10.0)
            shutdown_timeout: Default timeout for shutdown in seconds (default: 5.0)
            encryption: QueueEncryption instance for encrypting disk persistence
                (default: None, loads from MCA_QUEUE_ENCRYPTION_KEY env var if encryption_enabled=True)
            encryption_enabled: Whether to encrypt persisted queue data (default: False)
            config: Optional MCAConfig for custom allowed directories (Story 3.15)

        Raises:
            BufferingError: If persist=True but persist_path not provided, or if
                path validation fails (path traversal, sensitive directory, unauthorized location)
        """
        if max_size <= 0:
            raise BufferingError(f"max_size must be positive, got {max_size}")

        if shutdown_timeout <= 0:
            raise BufferingError(f"shutdown_timeout must be positive, got {shutdown_timeout}")

        self._max_size = max_size
        self._queue: deque[Any] = deque(maxlen=max_size)
        self._lock = (
            threading.RLock()
        )  # Reentrant: _save_to_disk acquires inside callers that already hold
        self._persist = persist
        self._persist_path = persist_path
        self._min_disk_space_mb = min_disk_space_mb
        self._shutdown_timeout = shutdown_timeout
        self._encryption = encryption
        self._encryption_enabled = encryption_enabled

        # Async persistence
        self._write_queue: Optional[queue_module.Queue[Any]] = None
        self._writer_thread: Optional[Thread] = None
        self._shutdown_event: Optional[Event] = None

        # Persistence metrics (thread-safe counters)
        self._persist_writes = 0  # Attempted writes (overflow events)
        self._persist_success_count = 0  # Successful atomic writes
        self._persist_failure_count = 0  # Failed writes
        self._persist_disk_full_count = 0  # Skipped due to disk space
        self._corruption_count = 0  # Corrupted files encountered
        self._items_recovered = 0  # Items loaded from disk on startup

        # Disk space check caching (prevent I/O thrashing)
        self._last_disk_check_time = 0.0
        self._last_disk_check_result = True
        self._disk_check_interval = 5.0  # Cache disk check for 5 seconds
        self._disk_check_lock = Lock()  # Separate lock for disk check to prevent thundering herd

        # Validate persistence configuration
        if self._persist and not self._persist_path:
            raise BufferingError("persist_path required when persist=True")

        # Validate and expand persist path if provided
        if self._persist_path:
            self._persist_path = self._validate_persist_path(self._persist_path, config)

        # Load encryption from environment if enabled but not provided
        if self._encryption_enabled and self._encryption is None:
            env_key = os.environ.get("MCA_QUEUE_ENCRYPTION_KEY")
            if env_key:
                # Detect production environment
                is_production = self._is_production_environment()

                if is_production:
                    # ENFORCE: Refuse to load keys from environment variables in production
                    raise BufferingError(
                        "PRODUCTION SECURITY VIOLATION: Cannot load encryption keys from environment variables in production.\n"
                        "Environment variables expose keys to: process listings, core dumps, container logs, child processes.\n"
                        "\n"
                        "✅ REQUIRED FOR PRODUCTION:\n"
                        "Pass QueueEncryption instance with MULTIPLE key versions for rotation support:\n"
                        "    from mca_sdk.security.encryption import QueueEncryption\n"
                        "    # Fetch multiple key versions (newest first) from your secret manager\n"
                        "    keys = fetch_key_versions('queue-key', versions=['v2', 'v1'])\n"
                        "    encryption = QueueEncryption(encryption_keys=keys)\n"
                        "    queue = TelemetryQueue(..., encryption=encryption)\n"
                        "\n"
                        "⚠️  DO NOT USE from_secret_manager() in production yet - it only fetches\n"
                        "    latest key and will cause DATA LOSS after key rotation.\n"
                        "    Wait for Story 3.9 completion for full multi-version support.\n"
                        "\n"
                        "❌ BLOCKED: MCA_QUEUE_ENCRYPTION_KEY is only allowed in: development, testing, CI/CD.\n"
                        "To override (DEV/TEST ONLY): Set MCA_ALLOW_ENV_KEYS_IN_PROD=true"
                    )

                # Non-production: Allow with warning
                logger.warning(
                    "🔐 SECURITY WARNING 🔐\n"
                    "Loading encryption key from MCA_QUEUE_ENCRYPTION_KEY environment variable.\n"
                    "Environment variables expose keys to: process listings, core dumps, container logs, child processes.\n"
                    "✅ USE GCP Secret Manager (QueueEncryption.from_secret_manager) for production.\n"
                    "Current environment: development/testing (environment variable loading permitted)"
                )
                try:
                    from ..security.encryption import QueueEncryption

                    # FAIL LOUDLY if key is invalid - never silently disable encryption
                    self._encryption = QueueEncryption(encryption_keys=[env_key.encode("ascii")])
                    logger.info(
                        "Loaded queue encryption key from environment variable (DEV/TEST ONLY)"
                    )
                except Exception as e:
                    # CRITICAL: Do not silently disable encryption on invalid key
                    raise BufferingError(
                        f"Failed to load encryption key from MCA_QUEUE_ENCRYPTION_KEY: {e}. "
                        f"Encryption is enabled but key is invalid. "
                        f"Fix the encryption key or set encryption_enabled=False. "
                        f"NEVER silently fall back to unencrypted storage in production."
                    )
            else:
                # CRITICAL: Do not silently disable encryption if no key provided
                raise BufferingError(
                    "encryption_enabled=True but no encryption key provided. "
                    "Either provide 'encryption' parameter or set MCA_QUEUE_ENCRYPTION_KEY environment variable. "
                    "For production: Use QueueEncryption.from_secret_manager() to load keys from GCP Secret Manager. "
                    "For development: Generate a key with: from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
                )

        # Start background writer thread if persistence enabled
        if self._persist:
            # CRITICAL: Warn about unencrypted storage
            if not self._encryption_enabled or self._encryption is None:
                logger.warning(
                    "⚠️  HIPAA COMPLIANCE WARNING ⚠️\n"
                    "Queue persistence is enabled WITHOUT encryption at rest.\n"
                    "Persisted telemetry may contain PHI and is stored UNENCRYPTED.\n"
                    "Encryption at rest is required before enabling persistence in production.\n"
                    f"Unencrypted queue file: {self._persist_path}"
                )
            else:
                logger.info(
                    f"Queue persistence enabled with encryption at rest: {self._persist_path}"
                )

            self._write_queue = queue_module.Queue(maxsize=100)
            self._shutdown_event = Event()
            self._writer_thread = Thread(
                target=self._background_writer, name="TelemetryQueue-Writer", daemon=True
            )
            self._writer_thread.start()

            self._load_from_disk()

    def _validate_persist_path(self, path: str, config: Optional[Any] = None) -> str:
        """Validate persist_path with comprehensive security checks.

        Defense-in-depth: Runtime validation with security logging, symlink loop
        detection, and directory permission checks.

        Security Features:
        - Canonical path validation (Finding #6)
        - Symlink loop detection (Finding #3)
        - Directory permission validation (Findings #5, #10)
        - Security event logging (Finding #8)
        - Configurable whitelist support (Finding #2)

        Args:
            path: Path to validate (may be relative, unexpanded, or contain symlinks)
            config: Optional MCAConfig for custom allowed directories

        Returns:
            Validated absolute path (canonicalized and permission-checked)

        Raises:
            BufferingError: If validation fails

        Examples:
            >>> _validate_persist_path("~/.mca_sdk/queue.json")
            '/home/user/.mca_sdk/queue.json'

            >>> _validate_persist_path("../../../etc/passwd")
            BufferingError: Path traversal detected...
        """
        import os
        import stat
        import tempfile

        # Step 1: Expand user home directory
        expanded = os.path.expanduser(path)

        # Step 2: Canonical path resolution with symlink loop detection (Finding #3)
        try:
            # Manual symlink loop detection: track visited symlinks
            visited = set()
            current = expanded
            max_iterations = 40  # Linux kernel limit

            # Resolve symlinks manually to detect loops
            for _ in range(max_iterations):
                if os.path.islink(current):
                    if current in visited:
                        raise OSError("Symlink loop detected")
                    visited.add(current)
                    # Get symlink target
                    target = os.readlink(current)
                    if not os.path.isabs(target):
                        # Relative symlink - resolve relative to parent
                        current = os.path.join(os.path.dirname(current), target)
                    else:
                        current = target
                else:
                    break

            # Now use realpath for final resolution
            canonical = os.path.realpath(expanded, strict=False)

            # Additional check for path length (defense in depth)
            if len(canonical) > 4096:  # Linux PATH_MAX
                raise OSError("Path too long (possible symlink loop)")

        except (OSError, ValueError) as e:
            # Security logging (Finding #8)
            logger.warning(
                f"SECURITY: Path validation failed for persist_path. "
                f"Original: {path}, Expanded: {expanded}, Error: {type(e).__name__}"
            )
            raise BufferingError(
                f"Invalid persist_path: {path}. " f"Could not resolve to canonical path: {e}"
            )

        # Step 3: Check for path traversal in CANONICAL path (Finding #6)
        # This catches bypasses via encoding, symlinks, etc.
        expanded_normalized = os.path.normpath(expanded)

        # If canonical path "escapes" above the expanded path, it's traversal
        if not canonical.startswith(os.path.dirname(expanded_normalized)):
            # Check if ".." appears in normalized expanded path
            if ".." in expanded_normalized:
                logger.warning(
                    f"SECURITY: Path traversal attempt detected. "
                    f"Original: {path}, Canonical: {canonical}"
                )
                raise BufferingError(
                    f"Path traversal detected in persist_path: {path}. "
                    f"Resolved to: {canonical}. "
                    f"Use absolute paths within allowed directories."
                )

        # Step 4: Sensitive directory check (highest priority)
        sensitive_prefixes = ["/etc", "/sys", "/proc", "/boot", "/root"]
        if any(canonical.startswith(prefix) for prefix in sensitive_prefixes):
            logger.warning(
                f"SECURITY: Attempted write to sensitive directory. " f"Path: {canonical}"
            )
            raise BufferingError(
                f"Cannot persist queue to sensitive system directory: {canonical}. "
                f"Use ~/.mca_sdk/ or /var/lib/mca/ instead."
            )

        # Step 5: Build whitelist (static + custom from config)
        static_allowed = [
            os.path.expanduser("~/.mca_sdk"),
            "/var/lib/mca",  # nosec B108 - Production path with proper permissions
            "/tmp/mca_sdk",  # nosec B108 - Development fallback with security warnings
            "/tmp/.mca_sdk",  # nosec B108 - Testing path with proper validation
            tempfile.gettempdir(),  # pytest tmp_path
        ]

        # Add custom directories if provided (Finding #2)
        if config and hasattr(config, "persist_allowed_dirs") and config.persist_allowed_dirs:
            custom_dirs = [d.strip() for d in config.persist_allowed_dirs.split(",")]
            static_allowed.extend(custom_dirs)
            logger.debug(f"Using custom allowed directories: {custom_dirs}")

        # Step 6: Whitelist validation
        if not any(canonical.startswith(prefix) for prefix in static_allowed):
            # Find closest valid directory for helpful error message (Finding #4)
            def path_similarity(dir1: str, dir2: str) -> int:
                """Count matching path components"""
                parts1 = dir1.split(os.sep)
                parts2 = dir2.split(os.sep)
                return sum(1 for a, b in zip(parts1, parts2) if a == b)

            sorted_allowed = sorted(
                static_allowed, key=lambda d: path_similarity(canonical, d), reverse=True
            )
            suggestion = sorted_allowed[0] if sorted_allowed else "~/.mca_sdk/"

            logger.warning(f"SECURITY: Unauthorized persist_path. " f"Attempted: {canonical}")
            raise BufferingError(
                f"Unauthorized persist_path: {path}. "
                f"Resolved to: {canonical}. "
                f"Suggested: {suggestion}. "
                f"Set MCA_PERSIST_ALLOWED_DIRS to add custom directories."
            )

        # Step 7: Production /tmp warning (Finding #7)
        if canonical.startswith("/tmp"):  # nosec B108 - Intentional check with security warning
            env = os.getenv("ENVIRONMENT", "").lower()
            if env in ("production", "prod"):
                logger.warning(
                    f"SECURITY: Using /tmp for queue persistence in production. "
                    f"Path: {canonical}. /tmp may be world-writable and insecure."
                )

        # Step 8: Directory permission validation (Findings #5, #10)
        parent_dir = os.path.dirname(canonical) or "."
        if os.path.exists(parent_dir):
            try:
                dir_stat = os.stat(parent_dir)
                dir_perms = stat.S_IMODE(dir_stat.st_mode)

                # Check for world-writable (Finding #10)
                # Exception: /tmp with sticky bit (1777) is acceptable for testing
                has_sticky_bit = dir_perms & stat.S_ISVTX
                is_tmp_dir = parent_dir.startswith("/tmp") or parent_dir.startswith(
                    tempfile.gettempdir()
                )  # nosec B108

                if (dir_perms & stat.S_IWOTH) and not (has_sticky_bit and is_tmp_dir):
                    logger.error(
                        f"SECURITY: Directory is world-writable (777/other+w). "
                        f"Path: {parent_dir}, Permissions: {oct(dir_perms)}"
                    )
                    raise BufferingError(
                        f"Insecure directory permissions for: {parent_dir}. "
                        f"Directory is world-writable ({oct(dir_perms)}). "
                        f"Fix with: chmod 0700 {parent_dir}"
                    )

                # Warn on group-writable (Finding #10)
                if dir_perms & stat.S_IWGRP:
                    logger.warning(
                        f"SECURITY: Directory is group-writable. "
                        f"Path: {parent_dir}, Permissions: {oct(dir_perms)}. "
                        f"Consider restricting to owner-only (0700)."
                    )

                # Recommend 0700 for sensitive data
                recommended_perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR  # 0700
                if dir_perms != recommended_perms:
                    logger.info(
                        f"INFO: Directory permissions are {oct(dir_perms)}. "
                        f"Recommended: 0700 for PHI data. Path: {parent_dir}"
                    )

            except (OSError, PermissionError) as e:
                # Can't check permissions - log but don't fail
                logger.warning(f"Could not check directory permissions: {parent_dir}. Error: {e}")

        return canonical

    def enqueue(self, item: Any) -> bool:
        """Add item to queue.

        Thread-safe operation that adds an item to the queue. If the queue
        is full, the oldest item is automatically evicted (FIFO).

        Args:
            item: Telemetry data to enqueue (any serializable object)

        Returns:
            True if item was added, False if queue is full and item was dropped

        Example:
            >>> queue.enqueue({"metric": "model.latency_seconds", "value": 0.15})
            True

        Performance Note:
            Disk persistence trigger is executed OUTSIDE the lock to prevent
            blocking all telemetry logging if disk I/O is slow. The lock is
            held only for the in-memory queue operation (fast).
        """
        # Fast path: acquire lock only for in-memory queue operation
        with self._lock:
            was_full = len(self._queue) >= self._max_size

            # Log queue full event
            if was_full:
                logger.warning(
                    f"Telemetry queue full (size={self._max_size}), " "oldest item will be evicted"
                )

            # deque with maxlen automatically evicts oldest when full
            self._queue.append(item)
            should_persist = self._persist and was_full

        # Trigger persistence OUTSIDE lock to prevent blocking on disk I/O
        # _save_to_disk() is async (uses put_nowait()) so it won't block here
        if should_persist:
            self._save_to_disk()
            with self._lock:
                self._persist_writes += 1

        return not was_full

    def dequeue(self) -> Optional[Any]:
        """Remove and return oldest item from queue.

        Thread-safe operation that removes the oldest item (FIFO).

        Returns:
            The oldest item in queue, or None if queue is empty

        Example:
            >>> item = queue.dequeue()
        """
        with self._lock:
            if not self._queue:
                return None

            item = self._queue.popleft()

            # Note: No persistence on dequeue - only persist on overflow

            return item

    def dequeue_batch(self, size: int) -> List[Any]:
        """Remove and return multiple items from queue.

        Thread-safe operation that removes up to `size` oldest items (FIFO).

        Args:
            size: Maximum number of items to dequeue (must be positive)

        Returns:
            List of items (may be shorter than size if queue has fewer items)

        Raises:
            BufferingError: If size <= 0

        Example:
            >>> batch = queue.dequeue_batch(size=100)
            >>> len(batch)
            42  # Queue had only 42 items
        """
        if size <= 0:
            raise BufferingError(f"dequeue_batch size must be positive, got {size}")

        with self._lock:
            # Bulk dequeue using list comprehension - more efficient than individual popleft()
            actual_size = min(size, len(self._queue))
            batch = [self._queue.popleft() for _ in range(actual_size)]

            # Note: No persistence on dequeue_batch - only persist on overflow

            return batch

    def peek(self) -> Optional[Any]:
        """View oldest item without removing it.

        Thread-safe operation that returns the oldest item without modification.

        Returns:
            The oldest item in queue, or None if queue is empty
        """
        with self._lock:
            if not self._queue:
                return None
            return self._queue[0]

    def size(self) -> int:
        """Get current queue size.

        Thread-safe operation that returns the number of items in queue.

        Returns:
            Current number of items in queue
        """
        with self._lock:
            return len(self._queue)

    def is_empty(self) -> bool:
        """Check if queue is empty.

        Thread-safe operation.

        Returns:
            True if queue is empty, False otherwise
        """
        with self._lock:
            return len(self._queue) == 0

    def is_full(self) -> bool:
        """Check if queue is at maximum capacity.

        Thread-safe operation.

        Returns:
            True if queue is full, False otherwise
        """
        with self._lock:
            return len(self._queue) >= self._max_size

    def clear(self) -> int:
        """Remove all items from queue.

        Thread-safe operation that empties the queue.

        Returns:
            Number of items that were removed

        Example:
            >>> removed = queue.clear()
            >>> print(f"Removed {removed} items")
        """
        with self._lock:
            count = len(self._queue)
            self._queue.clear()

            # Persist to disk if enabled
            if self._persist:
                self._save_to_disk()

            if count > 0:
                logger.info(f"Queue cleared: {count} items removed")

            return count

    def buffer_stats(self) -> Dict[str, Any]:
        """Get buffer statistics for monitoring.

        Thread-safe operation that returns current queue metrics.

        Returns:
            Dictionary with buffer statistics:
                - size: Current number of items in queue
                - max_size: Maximum queue capacity
                - is_full: Whether queue is at maximum capacity
                - is_empty: Whether queue is empty
                - utilization: Queue utilization percentage (0-100)
                - persist_enabled: Whether disk persistence is enabled

        Example:
            >>> stats = queue.buffer_stats()
            >>> print(f"Queue {stats['utilization']:.1f}% full")
            Queue 42.5% full
        """
        with self._lock:
            current_size = len(self._queue)
            utilization = (current_size / self._max_size * 100) if self._max_size > 0 else 0

            stats: Dict[str, Any] = {
                "size": current_size,
                "max_size": self._max_size,
                "is_full": current_size >= self._max_size,
                "is_empty": current_size == 0,
                "utilization": round(utilization, 2),
                "persist_enabled": self._persist,
            }

            # Add persistence metrics if enabled
            if self._persist:
                stats["persistence"] = {
                    "writes_attempted": self._persist_writes,
                    "writes_succeeded": self._persist_success_count,
                    "writes_failed": self._persist_failure_count,
                    "disk_full_skipped": self._persist_disk_full_count,
                    "corruptions_detected": self._corruption_count,
                    "items_recovered_on_startup": self._items_recovered,
                }

            return stats

    def shutdown(self, timeout: Optional[float] = None) -> None:
        """Gracefully shutdown background writer and flush queue.

        Args:
            timeout: Maximum seconds to wait for writer thread (uses instance default if None)
        """
        if not self._persist or not self._writer_thread:
            return

        # Use instance default if not specified
        if timeout is None:
            timeout = self._shutdown_timeout

        try:
            # Signal shutdown via event only (simplified single-signal approach)
            if self._shutdown_event is None:
                raise RuntimeError(
                    "Shutdown event not initialized. Queue may not have been properly configured with persist=True."
                )
            self._shutdown_event.set()

            # Wait for thread to finish
            self._writer_thread.join(timeout=timeout)

            if self._writer_thread.is_alive():
                logger.warning(
                    f"Background writer did not stop within {timeout}s - "
                    "some queue data may not be persisted"
                )
            else:
                logger.debug("Background writer stopped cleanly")

        except Exception as e:
            logger.error(f"Error during queue shutdown: {e}")

    def _background_writer(self) -> None:
        """Background thread that performs disk writes asynchronously.

        Decouples disk I/O from enqueue/dequeue operations to prevent
        blocking the main thread while holding the queue lock.

        Shutdown mechanism: Single Event signal checked on each iteration.
        """
        logger.debug("Background queue writer thread started")

        if self._write_queue is None:
            raise RuntimeError(
                "Write queue not initialized. Background writer cannot start without a valid write queue."
            )
        if self._shutdown_event is None:
            raise RuntimeError(
                "Shutdown event not initialized. Background writer cannot start without a valid shutdown event."
            )

        while self._shutdown_event and not self._shutdown_event.is_set():
            try:
                # Wait for write request with timeout to check shutdown event
                write_request = self._write_queue.get(timeout=1.0)
                # Process write snapshot
                self._save_snapshot_to_disk(write_request)

            except queue_module.Empty:
                # Timeout - loop will check shutdown_event
                continue
            except Exception as e:
                logger.error(f"Background writer error: {e}", exc_info=True)

        # Shutdown requested - drain remaining writes before exit
        while not self._write_queue.empty():
            try:
                remaining = self._write_queue.get_nowait()
                self._save_snapshot_to_disk(remaining)
            except queue_module.Empty:
                break

        # Final flush
        self._save_to_disk_sync()
        logger.debug("Background queue writer thread stopped")

    @staticmethod
    def _str_to_bool(value: str) -> bool:
        """Convert string to boolean.

        Accepts: "true", "1", "yes", "on" (case-insensitive)

        Args:
            value: String value to convert

        Returns:
            True if value matches truthy string, False otherwise
        """
        return value.lower() in ("true", "1", "yes", "on")

    def _is_production_environment(self) -> bool:
        """Detect if running in production environment.

        Checks multiple indicators to determine if this is a production deployment.
        Production is detected by environment variables commonly set in prod systems.

        **IMPORTANT:** This is heuristic-based detection. For explicit control, set
        MCA_ENVIRONMENT=production or MCA_ENVIRONMENT=development to override detection.

        Returns:
            True if production environment detected, False otherwise

        Detection Logic (in priority order):
            1. MCA_ENVIRONMENT=production (explicit override, highest priority)
            2. ENV=production or ENVIRONMENT=production (exact match)
            3. KUBERNETES_NAMESPACE contains "prod" or "production" (K8s deployment)
            4. GCP_PROJECT contains "prod" or "production" (GCP deployment)
            5. Default: False (development)

        Override:
            - MCA_ALLOW_ENV_KEYS_IN_PROD=true bypasses detection (logs CRITICAL warning)
        """
        # Check for override first (log at CRITICAL level)
        if self._str_to_bool(os.environ.get("MCA_ALLOW_ENV_KEYS_IN_PROD", "")):
            logger.critical(
                "🔴 SECURITY OVERRIDE ACTIVE 🔴\n"
                "MCA_ALLOW_ENV_KEYS_IN_PROD=true detected.\n"
                "Environment variable key loading ALLOWED despite production detection.\n"
                "⚠️ THIS OVERRIDE MUST NEVER BE USED IN ACTUAL PRODUCTION DEPLOYMENTS ⚠️\n"
                "This is a security bypass for development/testing only."
            )
            return False

        # Explicit environment setting (highest priority)
        mca_env = os.environ.get("MCA_ENVIRONMENT", "").lower()
        if mca_env == "production":
            return True
        elif mca_env in ["development", "dev", "test", "testing"]:
            return False  # Explicitly non-production

        # Check explicit environment markers
        env_markers = ["ENV", "ENVIRONMENT", "DEPLOY_ENV", "APP_ENV"]
        for marker in env_markers:
            env_value = os.environ.get(marker, "").lower()
            if env_value in ["production", "prod", "prd"]:
                return True

        # Check Kubernetes namespace for production indicators
        if os.environ.get("KUBERNETES_SERVICE_HOST"):
            namespace = os.environ.get("KUBERNETES_NAMESPACE", "").lower()
            # Check for prod/production as separate component (handles -, _, .)
            # \W+ matches any non-word character (-, _, ., etc.)
            import re

            if re.search(r"(^|[\W_])(prod|production)([\W_]|$)", namespace):
                return True

        # Check GCP project name for production indicators
        gcp_project = os.environ.get("GCP_PROJECT", "").lower()
        # Check for prod/production as separate component (handles -, _, .)
        import re

        if re.search(r"(^|[\W_])(prod|production)([\W_]|$)", gcp_project):
            return True

        # Default to non-production (safe for development)
        return False

    def _check_disk_space(self) -> bool:
        """Check if sufficient disk space is available for persistence (with caching).

        Returns:
            True if sufficient space available, False otherwise

        Note:
            Uses fail-closed approach: returns False if check fails for any reason.
            Caches result for 5 seconds to prevent I/O thrashing at high TPS.
            Uses separate lock to prevent thundering herd when cache expires.
            Uses time.monotonic() to prevent NTP rollback issues.
            At 1000 TPS, this reduces disk checks from 1000/sec to ~0.2/sec.

        Performance Note:
            Disk check occurs on enqueue path every 5 seconds. If shutil.disk_usage()
            is slow (network mount, busy disk), this introduces periodic latency spikes.
            Trade-off accepted: 5-second interval minimizes impact (<0.02% of requests).
            Alternative: Move disk check to background thread (future optimization).
        """
        if not self._persist_path:
            return True

        # Quick check without lock - return cached result if still valid
        # CRITICAL: Use monotonic time to prevent NTP rollback breaking cache
        current_time = time.monotonic()
        if current_time - self._last_disk_check_time < self._disk_check_interval:
            return self._last_disk_check_result

        # Cache expired - acquire lock to ensure only ONE thread does expensive I/O
        # Other threads will wait for the first thread to complete and use its result
        with self._disk_check_lock:
            # Double-check pattern: another thread may have updated cache while we waited
            if current_time - self._last_disk_check_time < self._disk_check_interval:
                return self._last_disk_check_result

            # Still expired - perform actual disk check
            try:
                parent_dir = os.path.dirname(self._persist_path) or "."
                usage = shutil.disk_usage(parent_dir)
                available_mb = usage.free / (1024 * 1024)

                result = available_mb >= self._min_disk_space_mb

                if not result:
                    logger.warning(
                        f"Low disk space: {available_mb:.1f}MB available, "
                        f"{self._min_disk_space_mb:.1f}MB required. Skipping persistence."
                    )

                # Update cache (protected by lock)
                self._last_disk_check_time = current_time
                self._last_disk_check_result = result
                return result

            except Exception as e:
                # Broad exception intentional: disk_usage() can raise OS-specific errors
                # Fail-closed approach: prevent writes to potentially broken filesystem.
                logger.warning(
                    f"Failed to check disk space: {e}. Skipping persistence (fail-closed)."
                )

                # Cache failure result as well (protected by lock)
                self._last_disk_check_time = current_time
                self._last_disk_check_result = False
                return False

    def _save_to_disk(self) -> None:
        """Schedule async disk write (non-blocking) with automatic debouncing.

        IMPORTANT: This method does NOT block. It queues a write request
        for the background thread to process.

        Debouncing: Checks if write queue is full BEFORE creating expensive snapshot.
        This prevents CPU/memory thrashing under load (1000+ TPS).
        """
        if not self._persist or not self._write_queue:
            return

        # CRITICAL: Check if write queue is full BEFORE expensive operations
        # This prevents CPU/memory DoS from creating snapshots we immediately discard
        if self._write_queue.full():
            logger.debug(
                "Disk write queue full - skipping persistence (backpressure). "
                "This indicates disk I/O is slower than queue operations."
            )
            return

        # Check disk space before queueing write
        if not self._check_disk_space():
            # Thread-safe counter increment (lock protects read-modify-write)
            with self._lock:
                self._persist_disk_full_count += 1
            return

        try:
            # Take snapshot of current queue (RLock allows re-entry from enqueue/clear)
            # This is expensive (O(N)), so we only do it if queue has space
            with self._lock:
                snapshot = list(self._queue)

            # Queue write request (non-blocking)
            # Should always succeed since we checked full() above
            # (Race: queue could fill between check and put, but that's acceptable)
            self._write_queue.put_nowait(snapshot)

        except queue_module.Full:
            # Write queue full - skip this write
            logger.warning(
                "Disk write queue full - skipping persistence. "
                "This indicates disk I/O is slower than queue operations."
            )
        except Exception as e:
            logger.error(f"Failed to queue disk write: {e}")

    def _save_to_disk_sync(self) -> None:
        """Synchronous disk write with lock (for shutdown/flush).

        Used only during shutdown to ensure final state is persisted.
        """
        with self._lock:
            snapshot = list(self._queue)

        self._save_snapshot_to_disk(snapshot)

    @staticmethod
    def _cleanup_temp_file(temp_path: Optional[str]) -> None:
        """Remove a temp file left behind by a failed atomic write."""
        if temp_path and os.path.exists(temp_path):
            try:
                os.unlink(temp_path)
            except (OSError, PermissionError) as cleanup_error:
                logger.warning(
                    f"Failed to cleanup temp file {temp_path}: {type(cleanup_error).__name__}"
                )

    def _save_snapshot_to_disk(self, snapshot: List[Any]) -> None:
        """Synchronous disk write using JSON with atomic writes (called from background thread).

        Uses temp file + atomic rename pattern to prevent corruption on crashes.

        Args:
            snapshot: Pre-captured list of queue items
        """
        if not self._persist_path:
            return

        temp_path = None
        try:
            # Create parent directory if needed
            parent_dir = os.path.dirname(self._persist_path)
            if parent_dir and not os.path.exists(parent_dir):
                os.makedirs(parent_dir, mode=0o700, exist_ok=True)

            # Serialize queue to JSON (NOT pickle - security)
            queue_data = {
                "_queue_format_version": "1.0",
                "items": [self._serialize_item(item) for item in snapshot],
                "max_size": self._max_size,
            }

            # Convert to bytes for potential encryption
            json_bytes = json.dumps(queue_data, indent=2).encode("utf-8")

            # Encrypt if enabled - NEVER fall back to plaintext on failure
            if self._encryption_enabled and self._encryption:
                try:
                    data_to_write = self._encryption.encrypt(json_bytes)
                    write_mode = "wb"  # Binary mode for encrypted data
                except Exception as e:
                    # CRITICAL: FAIL SECURE - never write plaintext if encryption fails
                    logger.error(
                        f"Failed to encrypt queue data: {e}. "
                        f"ABORTING write to prevent unencrypted PHI exposure. "
                        f"Queue data will NOT be persisted."
                    )
                    # Thread-safe counter increment
                    with self._lock:
                        self._persist_failure_count += 1
                    # Re-raise to propagate error - do not silently fail
                    raise BufferingError(f"Queue persistence failed: encryption error: {e}")
            else:
                data_to_write = json_bytes
                write_mode = "wb"

            # Atomic write: Create temp file in same directory
            temp_fd, temp_path = tempfile.mkstemp(
                dir=parent_dir or ".", prefix=".queue_", suffix=".tmp"
            )

            try:
                # Write to temp file
                with os.fdopen(temp_fd, write_mode) as f:
                    f.write(data_to_write)

                # SECURITY: Set restrictive permissions on temp file
                os.chmod(temp_path, 0o600)

                # Atomic rename (replaces target file atomically)
                os.replace(temp_path, self._persist_path)
                temp_path = None  # Successfully renamed, no cleanup needed

                logger.debug(f"Queue persisted atomically to disk: {len(snapshot)} items")
                # Thread-safe counter increment
                with self._lock:
                    self._persist_success_count += 1

            except (IOError, OSError) as e:
                self._cleanup_temp_file(temp_path)
                # Check if this is a disk full error (TOCTOU race)
                if hasattr(e, "errno") and e.errno == errno.ENOSPC:
                    logger.warning("Disk full detected during queue persistence (TOCTOU)")
                    # Thread-safe counter increment
                    with self._lock:
                        self._persist_disk_full_count += 1
                raise

            except Exception:
                self._cleanup_temp_file(temp_path)
                raise

        except Exception as e:
            # Don't raise - disk persistence is best-effort
            # SECURITY: Log exception type only to prevent PHI in stack traces
            logger.error(f"Failed to persist queue snapshot: {type(e).__name__}")
            # Thread-safe counter increment
            with self._lock:
                self._persist_failure_count += 1

    def _serialize_item(self, item: Any) -> Dict[str, Any]:
        """Serialize a telemetry item to JSON-safe dict.

        Handles:
        - Plain dicts (passthrough)
        - OpenTelemetry SDK objects (extract attributes)
        - Custom objects (fallback to __dict__)

        Returns a placeholder dict for non-serializable items.
        """
        try:
            # Already a dict - validate JSON serializability
            if isinstance(item, dict):
                # Test serialization to catch non-serializable values early
                json.dumps(item)
                return item

            # OpenTelemetry SDK objects - extract attributes
            if hasattr(item, "attributes"):
                # SECURITY: Don't use str(item) - could expose PHI
                result = {
                    "type": item.__class__.__name__,
                    "attributes": (
                        dict(item.attributes) if hasattr(item.attributes, "items") else {}
                    ),
                    "timestamp": getattr(item, "timestamp", None),
                }
                # Optionally include name/resource if safe
                if hasattr(item, "name"):
                    result["name"] = item.name
                if hasattr(item, "resource") and hasattr(item.resource, "attributes"):
                    result["resource"] = dict(item.resource.attributes)
                # Validate serializability
                json.dumps(result)
                return result

            # Generic object - use __dict__ if available
            if hasattr(item, "__dict__"):
                candidate = {"type": item.__class__.__name__, **item.__dict__}
                # Validate serializability
                json.dumps(candidate)
                return candidate

            # Primitive type - wrap in dict
            result = {"value": item}
            json.dumps(result)
            return result

        except (TypeError, ValueError) as e:
            # Non-serializable object - return placeholder
            logger.warning(
                f"Failed to serialize telemetry item of type {type(item).__name__}: {e}. "
                "Item will be dropped from persistence."
            )
            return {"_serialization_error": True, "type": type(item).__name__, "error": str(e)}

    def _validate_item(self, item: Any) -> bool:
        """Validate a recovered item has expected structure.

        Args:
            item: Item to validate

        Returns:
            True if item is valid, False otherwise
        """
        # Must be a dict
        if not isinstance(item, dict):
            return False

        # Must not be empty
        if not item:
            return False

        # If it has a type field, it should be a string
        if "type" in item and not isinstance(item["type"], str):
            return False

        return True

    def _load_from_disk(self) -> None:
        """Load persisted queue from disk (JSON format only).

        Called during initialization if persist=True and file exists.
        Corrupted or non-JSON files are renamed with .corrupted timestamp suffix.
        Handles both encrypted and unencrypted queue files.
        """
        if not self._persist_path or not os.path.exists(self._persist_path):
            return

        try:
            # Check file size before loading (prevent OOM/DoS)
            file_size = os.path.getsize(self._persist_path)
            max_file_size = self._max_size * 10 * 1024  # 10KB per item estimate

            if file_size > max_file_size:
                logger.error(
                    f"Queue file too large: {file_size / (1024*1024):.1f}MB "
                    f"(max: {max_file_size / (1024*1024):.1f}MB). "
                    f"File may be corrupted or contains excessive data. "
                    f"Renaming to .corrupted to prevent OOM."
                )
                raise ValueError(f"Queue file exceeds maximum size: {file_size} bytes")

            # Read file as bytes (handles both encrypted and plaintext)
            with open(self._persist_path, "rb") as f:
                raw_data = f.read()

            # Decrypt if encryption is enabled
            if self._encryption_enabled and self._encryption:
                try:
                    from cryptography.fernet import InvalidToken

                    decrypted_data = self._encryption.decrypt(raw_data)
                    json_str = decrypted_data.decode("utf-8")
                except InvalidToken as e:
                    # WRONG KEY - Don't delete file, raise config error
                    logger.error(
                        f"Failed to decrypt queue file: Invalid encryption key. {e}. "
                        f"This usually means the encryption key has changed or is incorrect. "
                        f"The queue file will NOT be deleted - it contains valid encrypted data. "
                        f"Verify MCA_QUEUE_ENCRYPTION_KEY matches the key used to encrypt this file."
                    )
                    # Don't treat as corruption - keep file intact
                    raise BufferingError(
                        f"Cannot decrypt queue file: wrong encryption key. "
                        f"File: {self._persist_path}. "
                        f"Verify encryption key is correct."
                    )
                except Exception as e:
                    # OTHER ERRORS - might be corruption
                    logger.error(f"Failed to decrypt queue file (possible corruption): {e}")
                    raise ValueError(f"Failed to decrypt queue file: {e}")
            else:
                # Try to decode as UTF-8 (plaintext JSON)
                try:
                    json_str = raw_data.decode("utf-8")
                except UnicodeDecodeError:
                    # Might be encrypted but encryption not enabled
                    raise ValueError(
                        "Queue file appears to be encrypted but encryption is not enabled. "
                        "Set encryption_enabled=True and provide encryption key."
                    )

            # Parse JSON
            data = json.loads(json_str)

            # Validate JSON structure
            if not isinstance(data, dict) or "items" not in data:
                raise ValueError("Invalid JSON queue format")

            items = data["items"]

            # Validate loaded data
            if not isinstance(items, list):
                raise BufferingError(f"Persisted queue items must be a list, got {type(items)}")

            # Validate and filter items
            valid_items = []
            invalid_count = 0
            for item in items[-self._max_size :]:  # Only consider most recent items
                if self._validate_item(item):
                    valid_items.append(item)
                else:
                    invalid_count += 1
                    logger.warning(f"Skipping invalid item from persisted queue: {type(item)}")

            # Restore queue with validated items only
            self._queue = deque(valid_items, maxlen=self._max_size)
            self._items_recovered = len(self._queue)

            if invalid_count > 0:
                logger.warning(f"Skipped {invalid_count} invalid items during queue recovery")

            logger.info(f"Loaded {self._items_recovered} valid items from persisted queue")

        except Exception as e:
            # Corrupt or invalid file - rename and start fresh
            logger.warning(f"Failed to load persisted queue: {e}")

            # Rename corrupted file for forensics
            if os.path.exists(self._persist_path):
                corrupted_path = f"{self._persist_path}.corrupted.{int(time.time())}"
                try:
                    os.rename(self._persist_path, corrupted_path)
                    logger.info(f"Renamed corrupted queue file to: {corrupted_path}")
                except (OSError, PermissionError) as rename_error:
                    logger.error(
                        f"Failed to rename corrupted file {self._persist_path}: {rename_error}. "
                        f"Manual cleanup may be required."
                    )

            # Start with fresh queue
            self._queue = deque(maxlen=self._max_size)
            self._corruption_count += 1
