"""Async camera worker process for handling multiple cameras concurrently.

This module implements an async event loop worker that handles multiple cameras
in a single process using asyncio for efficient I/O-bound operations.
"""
import asyncio
import logging
import time
import multiprocessing
import os
import psutil
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from typing import Dict, Any, Optional, List, Union
from collections import deque
import cv2
from pathlib import Path

from .video_capture_manager import VideoCaptureManager
from .frame_processor import FrameProcessor
from .message_builder import StreamMessageBuilder
from .stream_statistics import StreamStatistics


class AsyncCameraWorker:
    """Async worker process that handles multiple cameras concurrently.

    This worker runs an async event loop to handle I/O-bound operations
    (video capture, Redis writes) for multiple cameras efficiently.
    """

    def __init__(
        self,
        worker_id: int,
        camera_configs: List[Dict[str, Any]],
        stream_config: Dict[str, Any],
        stop_event: multiprocessing.Event,
        health_queue: multiprocessing.Queue,
        command_queue: Optional[multiprocessing.Queue] = None,
        response_queue: Optional[multiprocessing.Queue] = None
    ):
        """Initialize async camera worker.

        Args:
            worker_id: Unique identifier for this worker
            camera_configs: List of camera configurations to handle
            stream_config: Streaming configuration (Redis, Kafka, etc.)
            stop_event: Event to signal worker shutdown
            health_queue: Queue for reporting health status
            command_queue: Queue for receiving dynamic camera commands (add/remove/update)
            response_queue: Queue for sending command responses back to manager
        """
        self.worker_id = worker_id
        self.camera_configs = camera_configs
        self.stream_config = stream_config
        self.stop_event = stop_event
        self.health_queue = health_queue
        self.command_queue = command_queue
        self.response_queue = response_queue

        # Setup logging with worker ID
        self.logger = logging.getLogger(f"AsyncWorker-{worker_id}")
        self.logger.info(f"Initializing worker {worker_id} with {len(camera_configs)} cameras")

        # Initialize components
        self.capture_manager = VideoCaptureManager()
        self.message_builder = StreamMessageBuilder(
            service_id=stream_config.get('service_id', 'streaming_gateway'),
            strip_input_content=False
        )
        self.statistics = StreamStatistics()

        # Track camera tasks
        self.camera_tasks: Dict[str, asyncio.Task] = {}
        self.captures: Dict[str, cv2.VideoCapture] = {}

        # Setup async Redis client
        self.redis_client = None

        # Create dedicated executors for parallel processing
        # ProcessPoolExecutor for CPU-intensive JPEG encoding (bypasses GIL)
        # Use 2 processes per worker for encoding parallelism
        self.num_encoding_processes = max(2, len(camera_configs) // 8)  # ~2 processes per worker
        self.encoding_executor = None
        self._encoding_pool_broken = False
        self._encoding_pool_lock = asyncio.Lock()  # Lock for thread-safe pool recreation
        self._pool_recreation_count = 0
        self._max_pool_recreations = 3  # Max times to recreate pool before falling back
        self._use_thread_fallback = False  # Fall back to thread-based encoding
        self._create_encoding_pool()

        # ThreadPoolExecutor for I/O-bound frame capture
        # Use 1 thread per camera for independent capture
        num_capture_threads = max(4, len(camera_configs))  # Minimum 4 threads for dynamic cameras
        self.capture_executor = ThreadPoolExecutor(max_workers=num_capture_threads)
        self.num_capture_threads = num_capture_threads

        # ========================================================================
        # Performance Metrics Tracking
        # ========================================================================
        self._encoding_times = deque(maxlen=100)  # Last 100 encoding times
        self._frame_times = deque(maxlen=100)  # Last 100 total frame times
        self._frames_encoded = 0
        self._encoding_errors = 0
        self._last_metrics_log = time.time()
        self._metrics_log_interval = 30.0  # Log metrics every 30 seconds
        self._process_info = psutil.Process(os.getpid())

        self.logger.info(
            f"Worker {worker_id}: Created encoding pool ({self.num_encoding_processes} processes) "
            f"and capture pool ({num_capture_threads} threads)"
        )
        self._log_system_resources("INIT")

    def _log_system_resources(self, context: str = ""):
        """Log detailed system resource usage for debugging.

        Args:
            context: Optional context string describing when this was called
        """
        try:
            # Process-level metrics
            proc_cpu = self._process_info.cpu_percent(interval=None)
            proc_memory = self._process_info.memory_info()
            proc_threads = self._process_info.num_threads()

            # Get child processes (encoding pool workers)
            children = self._process_info.children(recursive=True)
            child_count = len(children)
            child_cpu_total = sum(c.cpu_percent(interval=None) for c in children)

            # System-level metrics
            sys_cpu = psutil.cpu_percent(interval=None)
            sys_memory = psutil.virtual_memory()
            cpu_count = psutil.cpu_count()

            # Encoding metrics
            avg_encoding_time = 0
            if self._encoding_times:
                avg_encoding_time = sum(self._encoding_times) / len(self._encoding_times) * 1000  # ms

            avg_frame_time = 0
            if self._frame_times:
                avg_frame_time = sum(self._frame_times) / len(self._frame_times) * 1000  # ms

            self.logger.info(
                f"Worker {self.worker_id} METRICS [{context}]\n"
                f"  === Process Info ===\n"
                f"    PID: {os.getpid()}\n"
                f"    CPU%: {proc_cpu:.1f}%\n"
                f"    Memory RSS: {proc_memory.rss / 1024 / 1024:.1f} MB\n"
                f"    Threads: {proc_threads}\n"
                f"    Child Processes: {child_count}\n"
                f"    Child CPU Total: {child_cpu_total:.1f}%\n"
                f"  === Pool Config ===\n"
                f"    Encoding Processes: {self.num_encoding_processes}\n"
                f"    Capture Threads: {self.num_capture_threads}\n"
                f"    Pool Broken: {self._encoding_pool_broken}\n"
                f"    Pool Recreations: {self._pool_recreation_count}/{self._max_pool_recreations}\n"
                f"    Using Thread Fallback: {self._use_thread_fallback}\n"
                f"  === Encoding Stats ===\n"
                f"    Frames Encoded: {self._frames_encoded}\n"
                f"    Encoding Errors: {self._encoding_errors}\n"
                f"    Avg Encoding Time: {avg_encoding_time:.2f} ms\n"
                f"    Avg Frame Time: {avg_frame_time:.2f} ms\n"
                f"  === Camera Stats ===\n"
                f"    Active Cameras: {len(self.camera_tasks)}\n"
                f"    Active Captures: {len(self.captures)}\n"
                f"  === System Info ===\n"
                f"    System CPU%: {sys_cpu:.1f}%\n"
                f"    CPU Cores: {cpu_count}\n"
                f"    System Memory: {sys_memory.percent:.1f}% used "
                f"({sys_memory.used / 1024 / 1024 / 1024:.1f}/{sys_memory.total / 1024 / 1024 / 1024:.1f} GB)"
            )
        except Exception as exc:
            self.logger.warning(f"Worker {self.worker_id}: Failed to log system resources: {exc}")

    def _create_encoding_pool(self):
        """Create or recreate the encoding process pool."""
        try:
            # Shutdown old pool if exists
            if self.encoding_executor is not None:
                try:
                    self.encoding_executor.shutdown(wait=False, cancel_futures=True)
                except Exception:
                    pass

            self.encoding_executor = ProcessPoolExecutor(
                max_workers=self.num_encoding_processes,
                mp_context=multiprocessing.get_context('spawn')  # Use spawn for cross-platform
            )
            self._encoding_pool_broken = False
            self.logger.info(f"Worker {self.worker_id}: Created/recreated encoding pool with {self.num_encoding_processes} processes")
        except Exception as exc:
            self.logger.error(f"Worker {self.worker_id}: Failed to create encoding pool: {exc}")
            self._encoding_pool_broken = True

    async def initialize(self):
        """Initialize async resources (Redis client, etc.)."""
        try:
            # Import and initialize async Redis client
            from matrice_common.stream import MatriceStream, StreamType

            # Create MatriceStream with async support
            # Unpack stream_config as keyword arguments (MatriceStream expects **config)
            self.stream = MatriceStream(
                stream_type=StreamType.REDIS,
                **self.stream_config
            )

            # Use async client
            self.redis_client = self.stream.async_client
            await self.redis_client.setup_client()

            self.logger.info(f"Worker {self.worker_id}: Initialized async Redis client")

        except Exception as exc:
            self.logger.error(f"Worker {self.worker_id}: Failed to initialize: {exc}")
            raise

    async def run(self):
        """Main worker loop - starts async tasks for all cameras and handles commands."""
        try:
            # Initialize async resources
            await self.initialize()

            # Start initial camera tasks using internal method
            for camera_config in self.camera_configs:
                await self._add_camera_internal(camera_config)

            # Report initial health
            self._report_health("running", len(self.camera_tasks))

            # Start command handler task if command queue is provided
            command_task = None
            if self.command_queue:
                command_task = asyncio.create_task(
                    self._command_handler(),
                    name="command-handler"
                )
                self.logger.info(f"Worker {self.worker_id}: Command handler started")

            # Monitor tasks and stop event
            while not self.stop_event.is_set():
                # Check for completed/failed tasks
                for stream_key, task in list(self.camera_tasks.items()):
                    if task.done():
                        try:
                            # Check if task raised exception
                            task.result()
                            self.logger.warning(f"Worker {self.worker_id}: Camera {stream_key} task completed")
                        except Exception as exc:
                            self.logger.error(f"Worker {self.worker_id}: Camera {stream_key} task failed: {exc}")

                        # Remove completed task
                        del self.camera_tasks[stream_key]

                # Report health periodically
                self._report_health("running", len(self.camera_tasks))

                # Sleep briefly
                await asyncio.sleep(1.0)

            # Stop event set - graceful shutdown
            self.logger.info(f"Worker {self.worker_id}: Stop event detected, shutting down...")

            # Cancel command handler if running
            if command_task and not command_task.done():
                command_task.cancel()
                try:
                    await command_task
                except asyncio.CancelledError:
                    pass

            await self._shutdown()

        except Exception as exc:
            self.logger.error(f"Worker {self.worker_id}: Fatal error in run loop: {exc}")
            self._report_health("error", error=str(exc))
            raise

    async def _camera_handler(self, camera_config: Dict[str, Any]):
        """Handle a single camera with async I/O.

        Args:
            camera_config: Camera configuration dictionary
        """
        stream_key = camera_config['stream_key']
        stream_group_key = camera_config.get('stream_group_key', 'default')
        source = camera_config['source']
        topic = camera_config['topic']
        fps = camera_config.get('fps', 30)
        quality = camera_config.get('quality', 90)
        width = camera_config.get('width')
        height = camera_config.get('height')
        camera_location = camera_config.get('camera_location', 'Unknown')

        cap = None
        consecutive_failures = 0
        max_failures = 10

        try:
            # Prepare source (download if URL)
            prepared_source = self.capture_manager.prepare_source(source, stream_key)

            # Open capture in thread pool (blocking operation)
            cap, source_type = await asyncio.to_thread(
                self.capture_manager.open_capture,
                prepared_source, width, height
            )
            self.captures[stream_key] = cap

            # Get video properties
            video_props = self.capture_manager.get_video_properties(cap)
            original_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
            original_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

            actual_width, actual_height = FrameProcessor.calculate_actual_dimensions(
                original_width, original_height, width, height
            )

            self.logger.info(
                f"Worker {self.worker_id}: Camera {stream_key} opened - "
                f"{actual_width}x{actual_height} @ {fps} FPS"
            )

            frame_counter = 0

            # Main camera loop
            while not self.stop_event.is_set():
                try:
                    # Read frame (blocking - run in dedicated capture thread pool)
                    read_start = time.time()
                    loop = asyncio.get_event_loop()
                    ret, frame = await loop.run_in_executor(
                        self.capture_executor,  # Dedicated thread pool for I/O
                        cap.read
                    )
                    read_time = time.time() - read_start

                    if not ret:
                        consecutive_failures += 1
                        if consecutive_failures >= max_failures:
                            self.logger.error(
                                f"Worker {self.worker_id}: Camera {stream_key} - "
                                f"Max consecutive failures ({max_failures}) reached"
                            )
                            break
                        await asyncio.sleep(0.1)
                        continue

                    # Reset failure counter
                    consecutive_failures = 0
                    frame_counter += 1

                    # Resize if needed
                    if width or height:
                        frame = FrameProcessor.resize_frame(frame, width, height)

                    # Process and send frame
                    await self._process_and_send_frame(
                        frame, stream_key, stream_group_key, topic,
                        source, video_props, fps, quality,
                        actual_width, actual_height, source_type,
                        frame_counter, camera_location, read_time
                    )

                    # Maintain FPS for video files
                    if source_type == "video_file":
                        await asyncio.sleep(1.0 / fps)

                except asyncio.CancelledError:
                    self.logger.info(f"Worker {self.worker_id}: Camera {stream_key} task cancelled")
                    break
                except Exception as exc:
                    self.logger.error(f"Worker {self.worker_id}: Error in camera {stream_key}: {exc}")
                    consecutive_failures += 1
                    if consecutive_failures >= max_failures:
                        break
                    await asyncio.sleep(1.0)

        finally:
            # Cleanup
            if cap and stream_key in self.captures:
                cap.release()
                del self.captures[stream_key]
                self.logger.info(f"Worker {self.worker_id}: Released camera {stream_key}")

    async def _process_and_send_frame(
        self,
        frame,
        stream_key: str,
        stream_group_key: str,
        topic: str,
        source: Union[str, int],
        video_props: Dict[str, Any],
        fps: int,
        quality: int,
        actual_width: int,
        actual_height: int,
        source_type: str,
        frame_counter: int,
        camera_location: str,
        read_time: float
    ):
        """Process frame and send to Redis asynchronously.

        Args:
            frame: Frame data
            stream_key: Stream identifier
            stream_group_key: Stream group identifier
            topic: Topic name
            source: Video source
            video_props: Video properties
            fps: Target FPS
            quality: JPEG quality
            actual_width: Frame width
            actual_height: Frame height
            source_type: Type of source
            frame_counter: Current frame number
            camera_location: Camera location
            read_time: Time taken to read frame
        """
        frame_start = time.time()

        # Build metadata
        metadata = self.message_builder.build_frame_metadata(
            source, video_props, fps, quality, actual_width, actual_height,
            source_type, frame_counter, False, None, None, camera_location
        )
        metadata["feed_type"] = "disk" if source_type == "video_file" else "camera"
        metadata["frame_count"] = 1
        metadata["stream_unit"] = "frame"

        # Encode frame in process pool (CPU-bound)
        encoding_start = time.time()
        frame_data, codec = await self._encode_frame_async(frame, quality)
        encoding_time = time.time() - encoding_start
        metadata["encoding_type"] = "jpeg"

        # Get timing stats
        last_read, last_write, last_process = self.statistics.get_timing(stream_key)
        input_order = self.statistics.get_next_input_order(stream_key)

        # Build message
        message = self.message_builder.build_message(
            frame_data, stream_key, stream_group_key, codec, metadata, topic,
            self.stream_config.get('bootstrap_servers', 'localhost:9092'),
            input_order, last_read, last_write, last_process
        )

        # Send to Redis asynchronously
        write_start = time.time()
        await self.redis_client.add_message(topic, message)
        write_time = time.time() - write_start

        # Update statistics
        self.statistics.increment_frames_sent()
        process_time = read_time + write_time
        frame_size = len(frame_data) if frame_data else 0
        self.statistics.update_timing(stream_key, read_time, write_time, process_time, frame_size)

        # Track total frame time for metrics
        total_frame_time = time.time() - frame_start
        self._frame_times.append(total_frame_time)

    async def _encode_frame_async(self, frame, quality: int) -> tuple:
        """Encode frame using process pool for true parallelism.

        Uses ProcessPoolExecutor to bypass GIL and achieve parallel encoding
        across multiple CPU cores. Falls back to thread-based encoding if the
        process pool becomes unusable.

        Args:
            frame: Frame data (numpy array)
            quality: JPEG quality

        Returns:
            Tuple of (encoded_data, codec)
        """
        loop = asyncio.get_event_loop()
        encode_start = time.time()

        # Use thread fallback if process pool has failed too many times
        if self._use_thread_fallback:
            return await self._encode_frame_thread_fallback(frame, quality)

        # Check if pool needs recreation before attempting encoding
        if self._encoding_pool_broken:
            await self._try_recreate_pool()

        try:
            # Use process pool for encoding (bypasses GIL, enables true parallelism)
            encode_success, jpeg_buffer = await loop.run_in_executor(
                self.encoding_executor,  # Process pool for CPU-bound encoding
                _encode_frame_worker,
                frame,
                quality
            )

            encoding_time = time.time() - encode_start
            self._encoding_times.append(encoding_time)
            self._frames_encoded += 1

            # Periodically log metrics
            if time.time() - self._last_metrics_log > self._metrics_log_interval:
                self._last_metrics_log = time.time()
                self._log_system_resources("PERIODIC")

            if encode_success:
                # TRUE ZERO-COPY: Return memoryview directly without copying to bytes
                frame_data = memoryview(jpeg_buffer)
                return frame_data, "h264"
            else:
                # Encoding failed - return raw frame
                self.logger.warning(f"Worker {self.worker_id}: JPEG encoding returned False, using raw frame")
                frame_data = memoryview(frame).cast('B')
                return frame_data, "raw"

        except Exception as exc:
            self._encoding_errors += 1
            encoding_time = time.time() - encode_start

            # Check if this is a BrokenProcessPool error
            is_broken_pool = "BrokenProcessPool" in type(exc).__name__ or "process pool" in str(exc).lower()

            if is_broken_pool:
                self._encoding_pool_broken = True
                self.logger.warning(
                    f"Worker {self.worker_id}: Process pool broken, attempting recovery "
                    f"(recreation count: {self._pool_recreation_count}/{self._max_pool_recreations})"
                )

                # Try to recreate pool and retry encoding
                if await self._try_recreate_pool():
                    try:
                        # Retry with new pool
                        return await self._encode_frame_with_pool(frame, quality, loop)
                    except Exception as retry_exc:
                        self.logger.error(f"Worker {self.worker_id}: Retry after pool recreation failed: {retry_exc}")
                        # Fall through to thread fallback

                # If we've exceeded max recreations, switch to thread fallback permanently
                if self._pool_recreation_count >= self._max_pool_recreations:
                    self.logger.warning(
                        f"Worker {self.worker_id}: Max pool recreations exceeded, switching to thread-based encoding"
                    )
                    self._use_thread_fallback = True
                    return await self._encode_frame_thread_fallback(frame, quality)

                # Otherwise use thread fallback for this frame only
                return await self._encode_frame_thread_fallback(frame, quality)

            # For non-pool errors, log and re-raise
            import traceback
            self.logger.error(
                f"Worker {self.worker_id}: ENCODING ERROR\n"
                f"  Error Type: {type(exc).__name__}\n"
                f"  Error Message: {exc}\n"
                f"  Frame Shape: {frame.shape if hasattr(frame, 'shape') else 'N/A'}\n"
                f"  Frame Dtype: {frame.dtype if hasattr(frame, 'dtype') else 'N/A'}\n"
                f"  Quality: {quality}\n"
                f"  Encoding Time Before Error: {encoding_time*1000:.2f} ms\n"
                f"  Traceback:\n{traceback.format_exc()}"
            )
            raise

    async def _encode_frame_with_pool(self, frame, quality: int, loop) -> tuple:
        """Encode frame using the process pool.

        Args:
            frame: Frame data
            quality: JPEG quality
            loop: Event loop

        Returns:
            Tuple of (encoded_data, codec)
        """
        encode_success, jpeg_buffer = await loop.run_in_executor(
            self.encoding_executor,
            _encode_frame_worker,
            frame,
            quality
        )

        self._frames_encoded += 1

        if encode_success:
            frame_data = memoryview(jpeg_buffer)
            return frame_data, "h264"
        else:
            frame_data = memoryview(frame).cast('B')
            return frame_data, "raw"

    async def _encode_frame_thread_fallback(self, frame, quality: int) -> tuple:
        """Fallback encoding using thread pool instead of process pool.

        This is less efficient but more reliable when process pool fails.

        Args:
            frame: Frame data
            quality: JPEG quality

        Returns:
            Tuple of (encoded_data, codec)
        """
        encode_start = time.time()

        try:
            # Use asyncio.to_thread for thread-based encoding
            encode_success, jpeg_buffer = await asyncio.to_thread(
                _encode_frame_worker,
                frame,
                quality
            )

            encoding_time = time.time() - encode_start
            self._encoding_times.append(encoding_time)
            self._frames_encoded += 1

            if encode_success:
                frame_data = memoryview(jpeg_buffer)
                return frame_data, "h264"
            else:
                frame_data = memoryview(frame).cast('B')
                return frame_data, "raw"

        except Exception as exc:
            self.logger.error(f"Worker {self.worker_id}: Thread fallback encoding failed: {exc}")
            # Return raw frame as last resort
            frame_data = memoryview(frame).cast('B')
            return frame_data, "raw"

    async def _try_recreate_pool(self) -> bool:
        """Try to recreate the encoding pool.

        Returns:
            bool: True if pool was successfully recreated
        """
        async with self._encoding_pool_lock:
            # Double-check if still broken (another task may have fixed it)
            if not self._encoding_pool_broken:
                return True

            if self._pool_recreation_count >= self._max_pool_recreations:
                return False

            try:
                self.logger.info(f"Worker {self.worker_id}: Recreating encoding pool...")

                # Shutdown old pool
                if self.encoding_executor is not None:
                    try:
                        self.encoding_executor.shutdown(wait=False, cancel_futures=True)
                    except Exception as e:
                        self.logger.warning(f"Worker {self.worker_id}: Error shutting down old pool: {e}")

                # Small delay to allow cleanup
                await asyncio.sleep(0.1)

                # Create new pool
                self.encoding_executor = ProcessPoolExecutor(
                    max_workers=self.num_encoding_processes,
                    mp_context=multiprocessing.get_context('spawn')
                )

                self._encoding_pool_broken = False
                self._pool_recreation_count += 1

                self.logger.info(
                    f"Worker {self.worker_id}: Encoding pool recreated successfully "
                    f"(recreation {self._pool_recreation_count}/{self._max_pool_recreations})"
                )
                return True

            except Exception as exc:
                self.logger.error(f"Worker {self.worker_id}: Failed to recreate encoding pool: {exc}")
                self._pool_recreation_count += 1
                return False

    # ========================================================================
    # Dynamic Camera Management Methods
    # ========================================================================

    async def _command_handler(self):
        """Process commands from the manager (runs in async loop)."""
        self.logger.info(f"Worker {self.worker_id}: Command handler started")

        while not self.stop_event.is_set():
            try:
                # Non-blocking check for commands using executor
                command = await asyncio.get_event_loop().run_in_executor(
                    None,
                    self._get_command_nonblocking
                )

                if command:
                    await self._process_command(command)
                else:
                    # Small sleep when no commands to avoid busy-waiting
                    await asyncio.sleep(0.1)

            except asyncio.CancelledError:
                self.logger.info(f"Worker {self.worker_id}: Command handler cancelled")
                break
            except Exception as exc:
                self.logger.error(f"Worker {self.worker_id}: Error in command handler: {exc}")
                await asyncio.sleep(1.0)

        self.logger.info(f"Worker {self.worker_id}: Command handler stopped")

    def _get_command_nonblocking(self):
        """Get command from queue without blocking."""
        try:
            return self.command_queue.get_nowait()
        except Exception:
            return None

    async def _process_command(self, command: Dict[str, Any]):
        """Process a single command.

        Args:
            command: Command dictionary with 'type' and payload
        """
        cmd_type = command.get('type')
        self.logger.info(f"Worker {self.worker_id}: Processing command: {cmd_type}")

        try:
            if cmd_type == 'add_camera':
                camera_config = command.get('camera_config')
                success = await self._add_camera_internal(camera_config)
                self._send_response(cmd_type, camera_config.get('stream_key'), success)

            elif cmd_type == 'remove_camera':
                stream_key = command.get('stream_key')
                success = await self._remove_camera_internal(stream_key)
                self._send_response(cmd_type, stream_key, success)

            elif cmd_type == 'update_camera':
                camera_config = command.get('camera_config')
                stream_key = command.get('stream_key')
                # Update = remove + add with new config
                await self._remove_camera_internal(stream_key)
                success = await self._add_camera_internal(camera_config)
                self._send_response(cmd_type, stream_key, success)

            else:
                self.logger.warning(f"Worker {self.worker_id}: Unknown command type: {cmd_type}")

        except Exception as exc:
            self.logger.error(f"Worker {self.worker_id}: Error processing command {cmd_type}: {exc}")
            self._send_response(cmd_type, command.get('stream_key'), False, str(exc))

    async def _add_camera_internal(self, camera_config: Dict[str, Any]) -> bool:
        """Add a camera and start its streaming task.

        Args:
            camera_config: Camera configuration dictionary

        Returns:
            bool: True if camera was added successfully
        """
        stream_key = camera_config.get('stream_key')

        if not stream_key:
            self.logger.error(f"Worker {self.worker_id}: Camera config missing stream_key")
            return False

        if stream_key in self.camera_tasks:
            self.logger.warning(f"Worker {self.worker_id}: Camera {stream_key} already exists")
            return False

        try:
            # Create and start camera task
            task = asyncio.create_task(
                self._camera_handler(camera_config),
                name=f"camera-{stream_key}"
            )
            self.camera_tasks[stream_key] = task

            self.logger.info(f"Worker {self.worker_id}: Added camera {stream_key}")
            return True

        except Exception as exc:
            self.logger.error(f"Worker {self.worker_id}: Failed to add camera {stream_key}: {exc}")
            return False

    async def _remove_camera_internal(self, stream_key: str) -> bool:
        """Remove a camera and stop its streaming task.

        Args:
            stream_key: Unique identifier for the camera stream

        Returns:
            bool: True if camera was removed successfully
        """
        if stream_key not in self.camera_tasks:
            self.logger.warning(f"Worker {self.worker_id}: Camera {stream_key} not found")
            return False

        try:
            # Cancel the camera task
            task = self.camera_tasks[stream_key]
            if not task.done():
                task.cancel()
                try:
                    await asyncio.wait_for(task, timeout=5.0)
                except (asyncio.CancelledError, asyncio.TimeoutError):
                    pass

            # Remove from tracking
            del self.camera_tasks[stream_key]

            # Release capture if exists
            if stream_key in self.captures:
                self.captures[stream_key].release()
                del self.captures[stream_key]

            self.logger.info(f"Worker {self.worker_id}: Removed camera {stream_key}")
            return True

        except Exception as exc:
            self.logger.error(f"Worker {self.worker_id}: Error removing camera {stream_key}: {exc}")
            return False

    def _send_response(self, cmd_type: str, stream_key: str, success: bool, error: str = None):
        """Send response back to manager.

        Args:
            cmd_type: Type of command that was processed
            stream_key: Stream key the command was for
            success: Whether the command succeeded
            error: Error message if failed
        """
        if self.response_queue:
            try:
                self.response_queue.put_nowait({
                    'worker_id': self.worker_id,
                    'command_type': cmd_type,
                    'stream_key': stream_key,
                    'success': success,
                    'error': error,
                    'timestamp': time.time()
                })
            except Exception as exc:
                self.logger.warning(f"Worker {self.worker_id}: Failed to send response: {exc}")

    async def _shutdown(self):
        """Gracefully shutdown worker - cancel tasks and cleanup."""
        self.logger.info(f"Worker {self.worker_id}: Starting graceful shutdown")

        # Cancel all camera tasks
        for stream_key, task in self.camera_tasks.items():
            if not task.done():
                task.cancel()
                self.logger.info(f"Worker {self.worker_id}: Cancelled task for {stream_key}")

        # Wait for tasks to complete
        if self.camera_tasks:
            await asyncio.gather(*self.camera_tasks.values(), return_exceptions=True)

        # Release all captures
        for stream_key, cap in list(self.captures.items()):
            cap.release()
            self.logger.info(f"Worker {self.worker_id}: Released capture {stream_key}")
        self.captures.clear()

        # Close Redis client
        if self.redis_client:
            await self.redis_client.close()
            self.logger.info(f"Worker {self.worker_id}: Closed Redis client")

        # Shutdown executors
        self.logger.info(f"Worker {self.worker_id}: Shutting down executors...")
        try:
            if self.encoding_executor is not None:
                self.encoding_executor.shutdown(wait=True, cancel_futures=True)
        except Exception as exc:
            self.logger.warning(f"Worker {self.worker_id}: Error shutting down encoding pool: {exc}")
        try:
            self.capture_executor.shutdown(wait=True, cancel_futures=False)
        except Exception as exc:
            self.logger.warning(f"Worker {self.worker_id}: Error shutting down capture pool: {exc}")
        self.logger.info(f"Worker {self.worker_id}: Executors shut down")

        # Report final health
        self._report_health("stopped")

        self.logger.info(f"Worker {self.worker_id}: Shutdown complete")

    def _report_health(self, status: str, active_cameras: int = 0, error: Optional[str] = None):
        """Report health status to main process.

        Args:
            status: Worker status (running, stopped, error)
            active_cameras: Number of active camera tasks
            error: Error message if status is error
        """
        try:
            # Get CPU/memory metrics for this process
            proc_cpu = 0
            proc_memory_mb = 0
            try:
                proc_cpu = self._process_info.cpu_percent(interval=None)
                proc_memory_mb = self._process_info.memory_info().rss / 1024 / 1024
            except Exception:
                pass

            # Calculate average encoding time
            avg_encoding_ms = 0
            if self._encoding_times:
                avg_encoding_ms = sum(self._encoding_times) / len(self._encoding_times) * 1000

            health_report = {
                'worker_id': self.worker_id,
                'status': status,
                'active_cameras': active_cameras,
                'timestamp': time.time(),
                'error': error,
                # Extended metrics for debugging
                'metrics': {
                    'cpu_percent': proc_cpu,
                    'memory_mb': proc_memory_mb,
                    'frames_encoded': self._frames_encoded,
                    'encoding_errors': self._encoding_errors,
                    'avg_encoding_ms': avg_encoding_ms,
                    'pool_broken': self._encoding_pool_broken,
                    'encoding_processes': self.num_encoding_processes,
                    'capture_threads': self.num_capture_threads,
                    'pool_recreations': self._pool_recreation_count,
                    'using_thread_fallback': self._use_thread_fallback,
                }
            }
            self.health_queue.put_nowait(health_report)
        except Exception as exc:
            self.logger.warning(f"Worker {self.worker_id}: Failed to report health: {exc}")


def _encode_frame_worker(frame, quality: int):
    """Worker function for encoding frames in process pool.

    This runs in a separate process for true parallel execution.

    Args:
        frame: Frame data (numpy array)
        quality: JPEG quality

    Returns:
        Tuple of (success, encoded_buffer)
    """
    import cv2
    return cv2.imencode('.jpg', frame, [int(cv2.IMWRITE_JPEG_QUALITY), quality])


def run_async_worker(
    worker_id: int,
    camera_configs: List[Dict[str, Any]],
    stream_config: Dict[str, Any],
    stop_event: multiprocessing.Event,
    health_queue: multiprocessing.Queue,
    command_queue: multiprocessing.Queue = None,
    response_queue: multiprocessing.Queue = None
):
    """Entry point for async worker process.

    This function is called by multiprocessing.Process to start a worker.

    Args:
        worker_id: Worker identifier
        camera_configs: List of camera configurations
        stream_config: Streaming configuration
        stop_event: Shutdown event
        health_queue: Health reporting queue
        command_queue: Queue for receiving dynamic camera commands
        response_queue: Queue for sending command responses
    """
    # Setup logging for this process
    logging.basicConfig(
        level=logging.INFO,
        format=f'%(asctime)s - Worker-{worker_id} - %(name)s - %(levelname)s - %(message)s'
    )

    logger = logging.getLogger(f"AsyncWorker-{worker_id}")
    logger.info(f"Starting async worker {worker_id}")

    try:
        # Create worker
        worker = AsyncCameraWorker(
            worker_id=worker_id,
            camera_configs=camera_configs,
            stream_config=stream_config,
            stop_event=stop_event,
            health_queue=health_queue,
            command_queue=command_queue,
            response_queue=response_queue
        )

        # Run event loop
        asyncio.run(worker.run())

    except Exception as exc:
        logger.error(f"Worker {worker_id} failed: {exc}", exc_info=True)
        raise
