"""
High-throughput async streaming pipeline optimized for 10K+ FPS inference at scale.

Architecture (Async + Multiprocessing):
Consumer (async event loop + ProcessPool for decode) -> asyncio.Queue ->
Inference (async workers with dynamic batching) -> asyncio.Queue ->
Post-processing (multiprocess with per-camera routing) -> asyncio.Queue ->
Producer (async output with batching)

Key Optimizations:
- Async event loop for 1000+ camera streams (no GIL contention)
- ProcessPoolExecutor for CPU-bound H.265 decode (true parallelism)
- 8 async inference workers with dynamic batching (80%+ GPU utilization)
- Multiprocess post-processing with camera-based routing (stateful tracking isolation)
- asyncio.Queue with backpressure (prevents OOM under load)
- Shared Triton client on pipeline event loop (eliminates event loop conflicts)

Performance Targets:
- Throughput: 10,000 FPS
- Latency: <500ms (P50), <800ms (P99)
- GPU Utilization: 85%+
- Memory: Stable under sustained load
"""

import asyncio
import logging
import multiprocessing as mp
import os
import threading
from typing import Any, Dict, List, Optional

from matrice_inference.server.inference_interface import InferenceInterface
from matrice_inference.server.stream.consumer_manager import AsyncConsumerManager
from matrice_inference.server.stream.inference_worker import MultiprocessInferencePool
from matrice_inference.server.stream.post_processing_manager import MultiprocessPostProcessingPool
from matrice_inference.server.stream.producer_worker import ProducerWorker
from matrice_inference.server.stream.analytics_publisher import AnalyticsPublisher
from matrice_inference.server.stream.utils import CameraConfig
from matrice_inference.server.stream.inference_metric_logger import InferenceMetricLogger

# Set multiprocessing start method to 'spawn' for CUDA compatibility
# This must be done before any multiprocessing.Process is created
# 'spawn' is required for CUDA because 'fork' doesn't work with CUDA contexts
try:
    mp.set_start_method('spawn', force=True)
except RuntimeError:
    # Already set, ignore
    pass


class StreamingPipeline:
    """Optimized streaming pipeline with dynamic camera configuration and clean resource management."""

    DEFAULT_QUEUE_SIZE = 5000
    DEFAULT_MESSAGE_TIMEOUT = 10.0
    DEFAULT_INFERENCE_TIMEOUT = 30.0
    DEFAULT_SHUTDOWN_TIMEOUT = 30.0
    DEFAULT_METRIC_INTERVAL = 60.0  # 1 minutes (consistent with inference_metric_logger default)  

    def __init__(
        self,
        inference_interface: InferenceInterface,
        inference_queue_maxsize: int = DEFAULT_QUEUE_SIZE,
        postproc_queue_maxsize: int = DEFAULT_QUEUE_SIZE,
        output_queue_maxsize: int = DEFAULT_QUEUE_SIZE,
        message_timeout: float = DEFAULT_MESSAGE_TIMEOUT,
        inference_timeout: float = DEFAULT_INFERENCE_TIMEOUT,
        shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT,
        camera_configs: Optional[Dict[str, CameraConfig]] = None,
        app_deployment_id: Optional[str] = None,
        inference_pipeline_id: Optional[str] = None,
        enable_analytics_publisher: bool = True,
        deployment_id: Optional[str] = None,
        deployment_instance_id: Optional[str] = None,
        action_id: Optional[str] = None,
        app_id: Optional[str] = None,
        app_name: Optional[str] = None,
        app_version: Optional[str] = None,
        use_shared_metrics: Optional[bool] = True,
        enable_metric_logging: bool = True,
        metric_logging_interval: float = DEFAULT_METRIC_INTERVAL,
        frame_cache_worker_threads: int = 20,
        frame_cache_max_queue: int = 50000,
        frame_cache_max_connections: int = 200,
        # Predict functions from MatriceDeployServer (passed to inference workers)
        load_model: Optional[Any] = None,
        predict: Optional[Any] = None,
        async_predict: Optional[Any] = None,
        async_batch_predict: Optional[Any] = None,
        async_load_model: Optional[Any] = None,
        batch_predict: Optional[Any] = None,
        # Post-processing configuration (passed as dict instead of extracted from post_processor)
        post_processing_config: Optional[Dict[str, Any]] = None,
        index_to_category: Optional[Any] = None,
        target_categories: Optional[Any] = None,

    ):
        self.inference_interface = inference_interface
        self.message_timeout = message_timeout
        self.inference_timeout = inference_timeout
        self.shutdown_timeout = shutdown_timeout
        self.app_deployment_id = app_deployment_id
        self.inference_pipeline_id = inference_pipeline_id
        self.enable_analytics_publisher = enable_analytics_publisher

        self.deployment_id = deployment_id
        self.deployment_instance_id = deployment_instance_id
        self.action_id = action_id
        self.app_id = app_id
        self.app_name = app_name
        self.app_version = app_version

        # Store post-processing configuration (passed as dict, not extracted from post_processor)
        self.post_processing_config = post_processing_config or {}
        self.index_to_category = index_to_category
        self.target_categories = target_categories

        # Metric logging configuration
        self.enable_metric_logging = enable_metric_logging
        self.metric_logging_interval = metric_logging_interval
        self.use_shared_metrics = use_shared_metrics

        # Frame cache configuration
        self.frame_cache_worker_threads = frame_cache_worker_threads
        self.frame_cache_max_queue = frame_cache_max_queue
        self.frame_cache_max_connections = frame_cache_max_connections

        # Store predict functions from MatriceDeployServer (for passing to workers)
        self.load_model = load_model
        self.predict = predict
        self.async_predict = async_predict
        self.async_batch_predict = async_batch_predict
        self.async_load_model = async_load_model
        self.batch_predict = batch_predict

        self.camera_configs: Dict[str, CameraConfig] = camera_configs or {}
        self.running = False
        self.logger = logging.getLogger(__name__)

        # Event loop reference for async operations (set when pipeline starts)
        self._event_loop: Optional[asyncio.AbstractEventLoop] = None
        self._event_loop_thread: Optional[threading.Thread] = None
        self._loop_thread_running = False

        self._setup_queues(inference_queue_maxsize, postproc_queue_maxsize, output_queue_maxsize)
        self._setup_executors()
        self._setup_workers()
        # Frame cache instance (initialized lazily at start)
        self.frame_cache = None
        # Analytics publisher instance (initialized lazily at start)
        self.analytics_publisher = None
        # Metric logger instance (initialized lazily at start)
        self.metric_logger = None

    def _setup_queues(self, inference_size: int, postproc_size: int, output_size: int) -> None:
        """Initialize multiprocessing queues for pipeline stages.

        Uses PER-WORKER queues for inference and post-processing stages:
        - Consumer routes frames to specific worker queue based on hash(camera_id)
        - Each worker only reads from its assigned queue
        - No re-queuing → No race conditions → Order preserved per camera
        - True parallelism across different cameras

        Output queue remains single (producer is single-threaded).

        Note: Using Manager().Queue() instead of mp.Queue() for Windows compatibility.
        On Windows with spawn mode, mp.Queue cannot be pickled, but Manager queues can.
        """
        import multiprocessing as mp

        # Create a Manager for creating shared queues that can be pickled
        self.queue_manager = mp.Manager()

        # Store maxsize for per-worker queue creation (queues created in _setup_executors)
        self.inference_queue_maxsize = inference_size
        self.postproc_queue_maxsize = postproc_size
        self.output_queue_maxsize = output_size

        # Per-worker queues will be created in _setup_executors() after worker counts are known
        self.inference_queues = []  # List[Manager.Queue] - one per inference worker
        self.postproc_queues = []   # List[Manager.Queue] - one per postproc worker

        # Per-worker output queues (one per postproc worker) for reduced lock contention
        # Producer polls from all queues using round-robin
        # Note: output_queues created in _setup_executors() after worker count is known
        self.output_queues = []  # List[Manager.Queue] - one per postproc worker

        # Response queue for direct API calls (identity images, etc.)
        # Direct API requests are routed through the same inference workers
        # to avoid greenlet context switching issues
        self.direct_api_response_queue = self.queue_manager.Queue(maxsize=1000)

        # Shared metrics queue for multiprocessing workers (inference + post_processing)
        # Workers in separate processes send MetricUpdate messages to this queue
        # The InferenceMetricLogger drains this queue and aggregates metrics
        self.multiprocess_metrics_queue = self.queue_manager.Queue(maxsize=10000)

    def _setup_executors(self) -> None:
        """Initialize per-worker queues and worker counts for ordering-preserving parallelism.

        Uses PER-WORKER QUEUES with consumer-side routing:
        - Consumer routes frames to worker queue based on hash(camera_id) % num_workers
        - Each worker reads from its dedicated queue only
        - No re-queuing → No race conditions → 100% order preserved per camera
        - True parallelism across different cameras

        Worker count is dynamic based on CPU cores:
        - 25% of CPU count (int)
        - Minimum: 2 workers
        - Maximum: 8 workers
        """
        import multiprocessing as mp

        cpu_count = os.cpu_count() or 4

        # Calculate dynamic worker count: 25% of CPU count, min 2, max 8
        dynamic_worker_count = max(2, min(8, int(cpu_count * 0.25)))

        # Inference workers: Use dynamic count for true parallelism (no GIL)
        # Each worker process has its own model instance
        self.use_async_inference = self.inference_interface.has_async_predict()
        self.num_inference_workers = dynamic_worker_count

        # Post-processing workers: Match inference workers for balanced pipeline
        self.num_postproc_workers = dynamic_worker_count

        # Create per-worker inference queues (one queue per worker)
        # Using Manager().Queue() for Windows compatibility with spawn mode
        self.inference_queues = [
            self.queue_manager.Queue(maxsize=self.inference_queue_maxsize)
            for _ in range(self.num_inference_workers)
        ]
        self.logger.info(
            f"Created {len(self.inference_queues)} inference queues "
            f"(one per worker, maxsize={self.inference_queue_maxsize})"
        )

        # Create per-worker post-processing queues (one queue per worker)
        # Using Manager().Queue() for Windows compatibility with spawn mode
        self.postproc_queues = [
            self.queue_manager.Queue(maxsize=self.postproc_queue_maxsize)
            for _ in range(self.num_postproc_workers)
        ]
        self.logger.info(
            f"Created {len(self.postproc_queues)} post-processing queues "
            f"(one per worker, maxsize={self.postproc_queue_maxsize})"
        )

        # Create per-worker OUTPUT queues (one per postproc worker)
        # This eliminates lock contention when multiple workers write to output
        # Producer polls from all queues using round-robin
        self.output_queues = [
            self.queue_manager.Queue(maxsize=self.output_queue_maxsize)
            for _ in range(self.num_postproc_workers)
        ]
        self.logger.info(
            f"Created {len(self.output_queues)} output queues "
            f"(one per postproc worker, maxsize={self.output_queue_maxsize})"
        )

        # Note: No ThreadPoolExecutor - workers manage their own async tasks
        self.inference_executor = None
        self.postprocessing_executor = None

        self.logger.info(
            f"Initialized per-worker queue architecture: "
            f"inference_workers={self.num_inference_workers}, "
            f"postproc_workers={self.num_postproc_workers} "
            f"(cpu_count={cpu_count}, 25%={int(cpu_count * 0.25)}, min=2, max=8, ordering=PRESERVED)"
        )

    def _setup_workers(self) -> None:
        """Initialize worker containers."""
        self.consumer_manager: Optional[AsyncConsumerManager] = None  # Async consumer manager
        self.inference_pool: Optional[MultiprocessInferencePool] = None  # Multiprocess inference pool
        self.postproc_pool: Optional[MultiprocessPostProcessingPool] = None  # Multiprocess post-processing pool
        self.producer_workers: List = []
        self.worker_threads: List = []

    def _run_event_loop(self) -> None:
        """Run event loop in dedicated thread for the lifetime of the pipeline."""
        self._event_loop = asyncio.new_event_loop()
        asyncio.set_event_loop(self._event_loop)
        self._loop_thread_running = True

        self.logger.info("Event loop thread started")

        try:
            # Run the event loop forever until stop() is called
            self._event_loop.run_forever()
        finally:
            self.logger.info("Event loop thread stopping")
            # Clean up pending tasks
            pending = asyncio.all_tasks(self._event_loop)
            for task in pending:
                task.cancel()
            # Allow tasks to complete cancellation
            self._event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
            self._event_loop.close()
            self._loop_thread_running = False
            self.logger.info("Event loop thread stopped")

    def start(self) -> None:
        """Start the pipeline with proper error handling."""
        if self.running:
            self.logger.warning("Pipeline already running")
            return

        self.running = True

        # Start dedicated event loop thread
        self._event_loop_thread = threading.Thread(
            target=self._run_event_loop,
            daemon=True,
            name="PipelineEventLoop"
        )
        self._event_loop_thread.start()

        # Wait for event loop to be ready
        import time
        max_wait = 5.0
        wait_interval = 0.1
        elapsed = 0.0
        while not self._loop_thread_running and elapsed < max_wait:
            time.sleep(wait_interval)
            elapsed += wait_interval

        if not self._loop_thread_running:
            self.logger.error("Event loop thread failed to start")
            self.running = False
            raise RuntimeError("Event loop thread failed to start")

        self.logger.info(f"Event loop thread ready (waited {elapsed:.2f}s)")
        self.logger.info("Starting streaming pipeline...")

        try:
            # Initialize frame cache and analytics publisher ONLY if we have cameras
            # Otherwise, they'll be lazy-initialized when first camera arrives via refresh event
            if len(self.camera_configs) > 0:
                self._initialize_frame_cache()
                self._initialize_analytics_publisher()
                self.logger.info(
                    f"Initialized frame cache and analytics publisher with {len(self.camera_configs)} cameras at startup"
                )
            else:
                self.logger.info(
                    "No cameras at startup - frame cache and analytics publisher will be "
                    "lazy-initialized when first camera arrives via refresh event"
                )

            # Register event loop with InferenceInterface for thread-safe inference
            # Note: Models are NOT loaded here - inference workers load models independently
            # in each process for true parallelism (see inference_worker.py lines 101-118)
            self.inference_interface.set_pipeline_event_loop(self._event_loop)
            
            # Register worker queues with InferenceInterface for routing direct API calls
            # This ensures ALL inference (streaming + direct API) goes through worker processes
            # avoiding greenlet context switching issues
            self.inference_interface.set_worker_queues(
                input_queues=self.inference_queues,
                response_queue=self.direct_api_response_queue,
            )
            self.logger.info("✓ Registered pipeline event loop and worker queues with InferenceInterface")

            # Create workers (producer needs analytics publisher reference)
            # Note: Inference workers load models independently in each process (multiprocessing)
            # This is INTENTIONAL for process isolation - each worker needs its own model instance
            # The duplicate loading ensures no shared state between processes (GIL-free parallelism)
            # Schedule the async operation on the event loop
            future = asyncio.run_coroutine_threadsafe(
                self._create_workers(),
                self._event_loop
            )
            future.result(timeout=30.0)  # Wait for completion
            # Start all workers including analytics publisher
            self._start_workers()
            # Initialize and start metric logger after workers are started
            self._initialize_metric_logger()
            self._log_startup_info()
        except Exception as e:
            self.logger.error(f"Failed to start pipeline: {e}")
            self.stop()
            raise

    def _initialize_metric_logger(self) -> None:
        """
        Initialize and start InferenceMetricLogger for periodic metric reporting.
        
        The metric logger:
        - Runs on a background thread with configurable interval
        - Collects metrics from all active workers
        - Aggregates by worker type (consumer/inference/post_processing/producer)
        - Uses MultiprocessMetricsCollector for inference/post_processing workers (cross-process)
        - Uses WorkerMetrics for consumer/producer workers (same process)
        - Publishes to Kafka via RPC
        - Gracefully handles initialization failures (logs warning, continues)
        """
        if not self.enable_metric_logging:
            self.logger.info("Metric logging disabled")
            return

        try:
            from matrice_inference.server.stream.inference_metric_logger import (
                InferenceMetricLogger,
                KafkaMetricPublisher
            )

            self.metric_logger = InferenceMetricLogger(
                streaming_pipeline=self,
                interval_seconds=self.metric_logging_interval,
                # Will auto-initialize KafkaMetricPublisher
                publisher=None,  
                
                deployment_id=self.deployment_id,
                deployment_instance_id=self.deployment_instance_id,
                app_deploy_id=self.app_deployment_id,
                action_id=self.action_id,
                app_id=self.app_id,
                # Pass the shared metrics queue for collecting from multiprocessing workers
                multiprocess_metrics_queue=self.multiprocess_metrics_queue,
            )

            # Start background collection
            self.metric_logger.start()

            self.logger.info(
                f"Initialized metric logger: interval={self.metric_logging_interval}s, "
                f"deployment_id={self.deployment_id}, "
                f"deployment_instance_id={self.deployment_instance_id}, "
                f"multiprocess_metrics_queue=enabled"
            )

        except ImportError as e:
            self.logger.warning(
                f"Metric logging dependencies not available: {e}. "
                f"Continuing without metric logging."
            )
            self.metric_logger = None
        except Exception as e:
            self.logger.warning(
                f"Failed to initialize metric logger: {e}. "
                f"Continuing without metric logging."
            )
            self.metric_logger = None

    def _log_startup_info(self) -> None:
        """Log pipeline startup information."""
        inference_info = f"InferencePool({self.num_inference_workers} workers)" if self.inference_pool else "None"
        postproc_info = f"PostProcPool({self.num_postproc_workers} workers)" if self.postproc_pool else "None"

        if len(self.camera_configs) == 0:
            self.logger.info(
                f"Pipeline started with NO cameras - ready to accept dynamic camera configurations. "
                f"Inference: {inference_info}, PostProc: {postproc_info}, Producers: {len(self.producer_workers)}"
            )
        else:
            self.logger.info(
                f"Pipeline started - Cameras: {len(self.camera_configs)}, "
                f"Consumer: AsyncConsumerManager, Inference: {inference_info}, "
                f"PostProc: {postproc_info}, Producers: {len(self.producer_workers)}"
            )

    def stop(self) -> None:
        """Stop the pipeline gracefully with proper cleanup."""
        if not self.running:
            return

        self.logger.info("Stopping pipeline...")
        self.running = False

        # Disable worker queue routing in InferenceInterface
        # This allows direct API calls to use fallback path during shutdown
        try:
            self.inference_interface.disable_worker_queue_routing()
            self.logger.info("Disabled worker queue routing in InferenceInterface")
        except Exception as e:
            self.logger.warning(f"Error disabling worker queue routing: {e}")

        # Stop metric logger first (before stopping workers)
        if self.metric_logger:
            try:
                self.logger.info("Stopping metric logger...")
                self.metric_logger.stop(timeout=10.0)
                self.logger.info("Metric logger stopped")
            except Exception as e:
                self.logger.error(f"Error stopping metric logger: {e}")

        self._stop_workers()
        self._wait_for_threads()
        self._shutdown_executors()

        # Stop frame cache if running
        try:
            if self.frame_cache:
                self.frame_cache.stop()
                self.logger.info("Frame cache stopped")
        except Exception as e:
            self.logger.error(f"Error stopping frame cache: {e}")

        # Note: analytics_publisher.stop() is called in _stop_workers() before _wait_for_threads()
        # to ensure the thread is signaled to stop before we wait for it to complete

        # Stop event loop thread
        if self._event_loop and self._loop_thread_running:
            try:
                self.logger.info("Stopping event loop thread...")
                self._event_loop.call_soon_threadsafe(self._event_loop.stop)
                if self._event_loop_thread:
                    self._event_loop_thread.join(timeout=5.0)
                self.logger.info("Event loop thread stopped")
            except Exception as e:
                self.logger.error(f"Error stopping event loop thread: {e}")

        # Shutdown queue manager
        if hasattr(self, 'queue_manager'):
            try:
                self.logger.info("Shutting down queue manager...")
                self.queue_manager.shutdown()
                self.logger.info("Queue manager shutdown complete")
            except Exception as e:
                self.logger.error(f"Error shutting down queue manager: {e}")

        self.logger.info("Pipeline stopped")

    def _wait_for_threads(self) -> None:
        """Wait for all worker threads to complete."""
        for thread in self.worker_threads:
            if thread.is_alive():
                thread.join(timeout=self.shutdown_timeout)

    def _shutdown_executors(self) -> None:
        """Shutdown thread pool executors (if they exist)."""
        if self.inference_executor is not None:
            self.inference_executor.shutdown(wait=False)
        if self.postprocessing_executor is not None:
            self.postprocessing_executor.shutdown(wait=False)

    async def add_camera_config(self, camera_config: CameraConfig) -> bool:
        """
        Add a camera configuration dynamically while pipeline is running.
        
        Args:
            camera_config: Camera configuration to add
            
        Returns:
            bool: True if successfully added, False otherwise
        """
        try:
            camera_id = camera_config.camera_id

            if camera_id in self.camera_configs:
                self.logger.warning(f"Camera {camera_id} already exists, updating configuration")
                # Stop existing workers for this camera
                await self._stop_camera_workers(camera_id)

            # Add camera config
            self.camera_configs[camera_id] = camera_config

            # Lazy-initialize frame cache if this is first Redis camera
            self._ensure_frame_cache_initialized()

            # Lazy-initialize analytics publisher if this is first Redis camera
            self._ensure_analytics_publisher_initialized()

            # Add camera to consumer manager if pipeline is running
            if self.running:
                # Lazy-create consumer manager if this is first camera added after startup
                if not self.consumer_manager:
                    self.logger.info("First camera added - initializing consumer manager")
                    await self._create_consumer_workers()
                    # Start the newly created consumer manager
                    if self.consumer_manager:
                        await self.consumer_manager.start()
                        self.logger.info("✓ Consumer manager started")
                elif self.consumer_manager:
                    # Consumer manager exists, add camera to it
                    await self.consumer_manager.add_camera(camera_id, camera_config)

            self.logger.info(f"Successfully added camera configuration for {camera_id}")
            return True

        except Exception as e:
            self.logger.error(f"Failed to add camera config for {camera_config.camera_id}: {str(e)}")
            return False

    async def remove_camera_config(self, camera_id: str) -> bool:
        """
        Remove a camera configuration dynamically.

        Args:
            camera_id: ID of camera to remove

        Returns:
            bool: True if successfully removed, False otherwise
        """
        try:
            if camera_id not in self.camera_configs:
                # Camera already removed - return True since desired state is achieved
                self.logger.debug(f"Camera {camera_id} not found in configs, already removed")
                return True

            # Stop workers for this camera
            await self._stop_camera_workers(camera_id)

            # Remove camera config
            del self.camera_configs[camera_id]

            self.logger.info(f"Successfully removed camera configuration for {camera_id}")
            return True

        except Exception as e:
            self.logger.error(f"Failed to remove camera config for {camera_id}: {str(e)}")
            return False

    async def update_camera_config(self, camera_config: CameraConfig) -> bool:
        """
        Update an existing camera configuration.

        Args:
            camera_config: Updated camera configuration

        Returns:
            bool: True if successfully updated, False otherwise
        """
        return await self.add_camera_config(camera_config)

    async def reconcile_camera_configs(self, new_camera_configs: Dict[str, CameraConfig]) -> Dict[str, Any]:
        """
        Perform full reconciliation of camera configurations.

        This method replaces the current camera configurations with the provided
        snapshot, performing adds, updates, and removals as needed.

        Args:
            new_camera_configs: Complete snapshot of camera configurations

        Returns:
            Dict with reconciliation results:
                {
                    "success": bool,
                    "added": int,
                    "updated": int,
                    "removed": int,
                    "total_cameras": int,
                    "errors": List[str]
                }
        """
        result = {
            "success": True,
            "added": 0,
            "updated": 0,
            "removed": 0,
            "total_cameras": 0,
            "errors": []
        }

        try:
            # Validate input
            if new_camera_configs is None:
                error_msg = "new_camera_configs cannot be None"
                result["errors"].append(error_msg)
                result["success"] = False
                self.logger.error(error_msg)
                return result

            if not isinstance(new_camera_configs, dict):
                error_msg = f"new_camera_configs must be a dict, got {type(new_camera_configs)}"
                result["errors"].append(error_msg)
                result["success"] = False
                self.logger.error(error_msg)
                return result

            # Get current camera IDs
            current_ids = set(self.camera_configs.keys())
            new_ids = set(new_camera_configs.keys())

            # Determine operations
            cameras_to_remove = current_ids - new_ids
            cameras_to_add = new_ids - current_ids
            cameras_to_check_update = new_ids & current_ids

            self.logger.info(
                f"Reconciliation plan: "
                f"remove={len(cameras_to_remove)}, "
                f"add={len(cameras_to_add)}, "
                f"check_update={len(cameras_to_check_update)}"
            )

            # Step 1: Remove cameras no longer in config
            for camera_id in cameras_to_remove:
                try:
                    success = await self.remove_camera_config(camera_id)
                    if success:
                        result["removed"] += 1
                        self.logger.info(f"Removed camera {camera_id}")
                    else:
                        error_msg = f"Failed to remove camera {camera_id}"
                        result["errors"].append(error_msg)
                        self.logger.warning(error_msg)
                except Exception as e:
                    error_msg = f"Error removing camera {camera_id}: {e}"
                    result["errors"].append(error_msg)
                    self.logger.error(error_msg, exc_info=True)

            # Step 2: Check and update existing cameras
            for camera_id in cameras_to_check_update:
                try:
                    new_config = new_camera_configs[camera_id]
                    current_config = self.camera_configs.get(camera_id)

                    # Check if config has actually changed
                    config_changed = (
                        not current_config or
                        current_config.input_topic != new_config.input_topic or
                        current_config.output_topic != new_config.output_topic or
                        current_config.stream_config != new_config.stream_config or
                        current_config.enabled != new_config.enabled
                    )

                    if config_changed:
                        success = await self.update_camera_config(new_config)
                        if success:
                            result["updated"] += 1
                            self.logger.info(f"Updated camera {camera_id}")
                        else:
                            error_msg = f"Failed to update camera {camera_id}"
                            result["errors"].append(error_msg)
                            self.logger.warning(error_msg)
                    else:
                        self.logger.debug(f"Camera {camera_id} config unchanged, skipping update")

                except Exception as e:
                    error_msg = f"Error updating camera {camera_id}: {e}"
                    result["errors"].append(error_msg)
                    self.logger.error(error_msg, exc_info=True)

            # Step 3: Add new cameras
            for camera_id in cameras_to_add:
                try:
                    new_config = new_camera_configs[camera_id]
                    success = await self.add_camera_config(new_config)
                    if success:
                        result["added"] += 1
                        self.logger.info(f"Added camera {camera_id}")
                    else:
                        error_msg = f"Failed to add camera {camera_id}"
                        result["errors"].append(error_msg)
                        self.logger.warning(error_msg)
                except Exception as e:
                    error_msg = f"Error adding camera {camera_id}: {e}"
                    result["errors"].append(error_msg)
                    self.logger.error(error_msg, exc_info=True)

            # Update result
            result["total_cameras"] = len(self.camera_configs)
            result["success"] = len(result["errors"]) == 0

            if result["success"]:
                self.logger.info(
                    f"Reconciliation completed successfully: "
                    f"{result['total_cameras']} cameras active "
                    f"(+{result['added']}, ~{result['updated']}, -{result['removed']})"
                )
            else:
                self.logger.warning(
                    f"Reconciliation completed with {len(result['errors'])} errors: "
                    f"{result['total_cameras']} cameras active "
                    f"(+{result['added']}, ~{result['updated']}, -{result['removed']})"
                )

            return result

        except Exception as e:
            error_msg = f"Critical error during reconciliation: {e}"
            result["errors"].append(error_msg)
            result["success"] = False
            self.logger.error(error_msg, exc_info=True)
            return result

    def enable_camera(self, camera_id: str) -> bool:
        """Enable a camera configuration."""
        return self._set_camera_state(camera_id, True, "enabled")

    def disable_camera(self, camera_id: str) -> bool:
        """Disable a camera configuration."""
        return self._set_camera_state(camera_id, False, "disabled")

    def _set_camera_state(self, camera_id: str, enabled: bool, state_name: str) -> bool:
        """Set camera enabled state."""
        if camera_id in self.camera_configs:
            self.camera_configs[camera_id].enabled = enabled
            self.logger.info(f"Camera {camera_id} {state_name}")
            return True
        return False

    async def _create_workers(self) -> None:
        """Create all worker instances for the pipeline."""
        await self._create_consumer_workers()
        self._create_inference_worker()
        self._create_postprocessing_worker()
        self._create_producer_worker()

    async def _create_consumer_workers(self) -> None:
        """Create consumer manager for all cameras.

        Uses AsyncConsumerManager (single async event loop for all cameras).
        """
        if len(self.camera_configs) > 0:
            # Extract stream config from first camera (assuming all use same stream config)
            first_camera = next(iter(self.camera_configs.values()))
            stream_config = first_camera.stream_config
            input_topic = first_camera.input_topic

            self.consumer_manager = AsyncConsumerManager(
                camera_configs=self.camera_configs,
                stream_config=stream_config,
                app_deployment_id=self.app_deployment_id,
                pipeline=self,
                message_timeout=self.message_timeout,
            )

            self.logger.info(
                f"Created async consumer manager for {len(self.camera_configs)} cameras"
            )

    def _create_inference_worker(self) -> None:
        """Create multiprocess inference pool with async event loops.

        Architecture:
        - Multiple separate processes (one per GPU/core)
        - Each process recreates InferenceInterface → ModelManagerWrapper → ModelManager
        - Uses normal ModelManager with predict functions from MatriceDeployServer (NOT Triton)
        - Each process runs its own async event loop
        - Multiple concurrent inference requests per process
        - True parallelism (bypasses Python GIL)

        Note: Workers receive predict functions via model_config (passed from MatriceDeployServer).
        Functions are module-level and can be pickled by reference for multiprocessing.
        """
        # Extract configuration for ModelManager (NOT Triton)
        # Workers will recreate InferenceInterface with ModelManager using predict functions
        model_manager_wrapper = getattr(self.inference_interface, 'model_manager_wrapper', None)

        if model_manager_wrapper:
            # Extract action_id from action_tracker
            action_tracker = getattr(model_manager_wrapper, 'action_tracker', None)
            action_id = getattr(action_tracker, 'action_id', None) if action_tracker else None

            # Extract model manager config
            model_manager = getattr(model_manager_wrapper, 'model_manager', None)
            num_instances = len(getattr(model_manager, 'model_instances', [])) if model_manager else 1
            model_path = getattr(model_manager, 'model_path', None) if model_manager else None

            model_config = {
                "action_id": action_id,
                "model_path": model_path,
                "num_model_instances": num_instances,
                # Predict functions from MatriceDeployServer (module-level, picklable by reference)
                "load_model": self.load_model,
                "predict": self.predict,
                "async_predict": self.async_predict,
                "async_batch_predict": self.async_batch_predict,
                "async_load_model": self.async_load_model,
                "batch_predict": self.batch_predict,
            }

            if not action_id:
                self.logger.warning("No action_id found in action_tracker, workers may fail to initialize")
        else:
            # Fallback defaults if config not available
            self.logger.error("Could not extract ModelManager config from inference_interface")
            model_config = {
                "action_id": None,
                "model_path": None,
                "num_model_instances": 1,
                # Predict functions from MatriceDeployServer
                "load_model": self.load_model,
                "predict": self.predict,
                "async_predict": self.async_predict,
                "async_batch_predict": self.async_batch_predict,
                "async_load_model": self.async_load_model,
                "batch_predict": self.batch_predict,
            }

        self.inference_pool = MultiprocessInferencePool(
            num_workers=self.num_inference_workers,
            model_config=model_config,
            input_queues=self.inference_queues,  # Per-worker queues (one per worker)
            output_queues=self.postproc_queues,  # Per-worker output queues for routing to postproc
            use_async_inference=self.use_async_inference,  # Determines sync vs async behavior
            direct_api_response_queue=self.direct_api_response_queue,  # For identity images
            metrics_queue=self.multiprocess_metrics_queue,  # For sending metrics to main process
        )

        mode = "ASYNC+FEEDER (batched, no executor hops)" if self.use_async_inference else f"SYNC (blocking loop, {8} threads)"
        self.logger.info(
            f"Created multiprocess inference pool with {self.num_inference_workers} workers "
            f"- {mode} - PER-WORKER QUEUES for ordering"
        )

    def _create_postprocessing_worker(self) -> None:
        """Create multiprocess post-processing pool with camera routing.

        Architecture:
        - ProcessPoolExecutor with 4 workers (true parallelism, no GIL)
        - Camera-based hash routing (preserves per-camera ordering)
        - Each process maintains isolated tracker state
        - Process camera subsets (e.g., 250 cameras per process)
        """
        # Use post-processing configuration passed during initialization (as dict)
        # instead of extracting from post_processor object
        post_processor_config = {
            "post_processing_config": self.post_processing_config,
            "app_name": self.app_name,
            "index_to_category": self.index_to_category,
            "target_categories": self.target_categories,
        }

        self.postproc_pool = MultiprocessPostProcessingPool(
            pipeline=self,
            post_processor_config=post_processor_config,
            input_queues=self.postproc_queues,  # Per-worker queues (one per worker)
            output_queues=self.output_queues,  # Per-worker output queues (eliminates lock contention)
            num_processes=self.num_postproc_workers,
            metrics_queue=self.multiprocess_metrics_queue,  # For sending metrics to main process
        )

        self.logger.info(
            f"Created multiprocess post-processing pool with {self.num_postproc_workers} workers "
            f"- PER-WORKER QUEUES for ordering"
        )

    def _create_producer_worker(self) -> None:
        """Create single producer worker that handles all output queues.

        Now also handles frame caching (moved from consumer to avoid blocking inference flow).
        Producer polls from multiple output queues (one per postproc worker) using round-robin.
        """
        worker = ProducerWorker(
            worker_id=0,
            output_queues=self.output_queues,  # Per-worker output queues (list)
            pipeline=self,
            camera_configs=self.camera_configs,
            message_timeout=self.message_timeout,
            analytics_publisher=self.analytics_publisher,
            frame_cache=self.frame_cache,
            use_shared_metrics=self.use_shared_metrics,
            app_deployment_id=self.app_deployment_id,
        )
        self.producer_workers.append(worker)

    def _start_workers(self) -> None:
        """Start all worker instances and track their threads."""
        # Start consumer manager (async, single event loop for all cameras)
        if self.consumer_manager:
            asyncio.run_coroutine_threadsafe(self.consumer_manager.start(), self._event_loop)
            self.logger.info("Started async consumer manager")

        # Start inference pool (multiprocessing)
        if self.inference_pool:
            self.inference_pool.start()
            self.logger.info("Started multiprocess inference pool")

        # Start post-processing pool (multiprocessing)
        if self.postproc_pool:
            self.postproc_pool.start()
            self.logger.info("Started multiprocess post-processing pool")

        # Start producer workers
        self._start_worker_group(self.producer_workers)

        # Start analytics publisher if initialized
        if self.analytics_publisher:
            try:
                analytics_thread = self.analytics_publisher.start()
                self.worker_threads.append(analytics_thread)
                self.logger.info("Started analytics publisher thread")
            except Exception as e:
                self.logger.error(f"Failed to start analytics publisher: {e}")

    def _start_worker_group(self, workers: List) -> None:
        """Start a group of workers (handles both threading and async workers)."""
        for worker in workers:
            # Check if worker has async start method
            if hasattr(worker, 'start') and asyncio.iscoroutinefunction(worker.start):
                # Async worker - schedule on event loop
                asyncio.run_coroutine_threadsafe(worker.start(), self._event_loop)
            else:
                # Threading worker - traditional start
                thread = worker.start()
                if thread:
                    self.worker_threads.append(thread)

    def _stop_workers(self) -> None:
        """Stop all worker instances gracefully."""
        # Stop consumer manager (async) - MUST wait for completion before stopping event loop
        if self.consumer_manager and self._event_loop:
            try:
                future = asyncio.run_coroutine_threadsafe(self.consumer_manager.stop(), self._event_loop)
                # Wait for consumer manager to fully stop (with timeout)
                future.result(timeout=10.0)
                self.logger.info("Stopped async consumer manager")
            except TimeoutError:
                self.logger.warning("Consumer manager stop timed out after 10 seconds")
            except Exception as e:
                self.logger.error(f"Error stopping consumer manager: {e}")

        # Stop inference pool (multiprocessing)
        if self.inference_pool:
            self.inference_pool.stop()
            self.logger.info("Stopped multiprocess inference pool")

        # Stop post-processing pool (multiprocessing)
        if self.postproc_pool:
            self.postproc_pool.stop()
            self.logger.info("Stopped multiprocess post-processing pool")

        # Stop producer workers
        self._stop_worker_group(self.producer_workers)

        # Stop analytics publisher BEFORE waiting for threads
        # (analytics thread is in worker_threads, must signal stop before join)
        if self.analytics_publisher:
            try:
                self.analytics_publisher.stop()
                self.logger.info("Signaled analytics publisher to stop")
            except Exception as e:
                self.logger.error(f"Error stopping analytics publisher: {e}")

    def _stop_worker_group(self, workers: List) -> None:
        """Stop a group of workers."""
        for worker in workers:
            worker.stop()

    async def _stop_camera_workers(self, camera_id: str) -> None:
        """Clean up resources for a specific camera."""
        # Remove camera from consumer manager
        if self.consumer_manager:
            await self.consumer_manager.remove_camera(camera_id)

        # Clean up producer streams for this camera
        for producer_worker in self.producer_workers:
            try:
                # Check if producer event loop is available before cleanup
                if hasattr(producer_worker, '_event_loop') and producer_worker._event_loop:
                    if not producer_worker._event_loop.is_running():
                        self.logger.debug(
                            f"Producer event loop not running for camera {camera_id}, "
                            f"skipping graceful cleanup (expected during shutdown)"
                        )
                        continue

                producer_worker.remove_camera_stream(camera_id)
            except Exception as e:
                # Downgrade to debug during shutdown scenarios
                self.logger.debug(
                    f"Producer cleanup for camera {camera_id} failed (expected during concurrent operations): {e}"
                )

    def get_metrics(self) -> Dict[str, Any]:
        """Get pipeline metrics including frame cache statistics."""
        # Get per-worker queue sizes
        inference_qsizes = [q.qsize() for q in self.inference_queues] if hasattr(self, 'inference_queues') and self.inference_queues else []
        postproc_qsizes = [q.qsize() for q in self.postproc_queues] if hasattr(self, 'postproc_queues') and self.postproc_queues else []
        output_qsizes = [q.qsize() for q in self.output_queues] if hasattr(self, 'output_queues') and self.output_queues else []

        metrics = {
            "running": self.running,
            "camera_count": len(self.camera_configs),
            "enabled_cameras": sum(1 for config in self.camera_configs.values() if config.enabled),
            "queue_sizes": {
                "inference_queues": inference_qsizes,
                "inference_queue_total": sum(inference_qsizes),
                "postproc_queues": postproc_qsizes,
                "postproc_queue_total": sum(postproc_qsizes),
                "output_queues": output_qsizes,
                "output_queue_total": sum(output_qsizes),
                "queue_architecture": "per_worker_queues_for_ordering",
            },
            "worker_counts": {
                "consumer_manager": "AsyncConsumerManager" if self.consumer_manager else None,
                "inference_pool_workers": self.num_inference_workers if self.inference_pool else 0,
                "postproc_pool_workers": self.num_postproc_workers if self.postproc_pool else 0,
                "producers": len(self.producer_workers),
            },
            "thread_counts": {
                "total_threads": len(self.worker_threads),
                "active_threads": len([t for t in self.worker_threads if t.is_alive()]),
            },
            "camera_configs": {
                camera_id: {
                    "input_topic": config.input_topic,
                    "output_topic": config.output_topic,
                    "enabled": config.enabled,
                    "stream_type": config.stream_config.get("stream_type", "kafka")
                }
                for camera_id, config in self.camera_configs.items()
            }
        }

        # Add frame cache metrics if available
        if self.frame_cache:
            try:
                metrics["frame_cache"] = self.frame_cache.get_metrics()
            except Exception as e:
                self.logger.warning(f"Failed to get frame cache metrics: {e}")
                metrics["frame_cache"] = {"error": str(e)}
        else:
            metrics["frame_cache"] = {"enabled": False}

        # Add analytics publisher metrics if available
        if self.analytics_publisher:
            try:
                metrics["analytics_publisher"] = self.analytics_publisher.get_metrics()
            except Exception as e:
                self.logger.warning(f"Failed to get analytics publisher metrics: {e}")
                metrics["analytics_publisher"] = {"error": str(e)}
        else:
            metrics["analytics_publisher"] = {"enabled": False}

        # Add Metric logger statistics
        if self.metric_logger:
            try:
                metrics["metric_logger"] = self.metric_logger.get_stats()
            except Exception as e:
                self.logger.warning(f"Failed to get metric logger stats: {e}")
                metrics["metric_logger"] = {"error": str(e)}
        else:
            metrics["metric_logger"] = {"enabled": False}

        return metrics

    def _initialize_frame_cache(self) -> None:
        """Initialize RedisFrameCache with TTL 10 minutes, deriving connection from Redis cameras if available."""
        try:
            # Check if we have any Redis cameras - if not, skip initialization
            has_redis_cameras = False
            redis_camera_id = None
            for cfg in self.camera_configs.values():
                sc = cfg.stream_config or {}
                st = sc.get("stream_type", "redis").lower()
                if st == "redis":
                    has_redis_cameras = True
                    redis_camera_id = cfg.camera_id
                    break

            if not has_redis_cameras:
                self.logger.info(
                    "No Redis cameras found - skipping frame cache initialization. "
                    "Frame cache will be initialized when first Redis camera arrives."
                )
                return

            # Find Redis camera config for connection params
            host = "localhost"
            port = 6379
            password = None
            username = None
            db = 0

            for cfg in self.camera_configs.values():
                sc = cfg.stream_config or {}
                st = sc.get("stream_type", "redis").lower()
                self.logger.info(f"Frame cache init - Camera {cfg.camera_id}: stream_type={st}, config_keys={list(sc.keys())}")
                self.logger.info(f"Camera Config {cfg.camera_id} {st}")
                if st == "redis":
                    host = sc.get("host") or host
                    port = sc.get("port") or port
                    password = sc.get("password", password)
                    username = sc.get("username", username)
                    db = sc.get("db", db)
                    self.logger.info(f"Using Redis config from camera {cfg.camera_id}: {host}:{port}")
                    break

            # Lazy import to avoid dependency issues if not used
            from matrice_inference.server.stream.frame_cache import RedisFrameCache
            self.frame_cache = RedisFrameCache(
                host=host,
                port=port,
                db=db,
                password=password,
                username=username,
                ttl_seconds=600,  # 10 minutes
                worker_threads=self.frame_cache_worker_threads,
                max_queue=self.frame_cache_max_queue,
                max_connections=self.frame_cache_max_connections,
            )
            self.frame_cache.start()
            self.logger.info(
                f"Initialized RedisFrameCache with TTL=600s, workers={self.frame_cache_worker_threads}, "
                f"queue={self.frame_cache_max_queue}, max_connections={self.frame_cache_max_connections}"
            )
        except Exception as e:
            self.frame_cache = None
            self.logger.warning(f"Frame cache initialization failed; proceeding without cache: {e}")

    def _initialize_analytics_publisher(self) -> None:
        """Initialize AnalyticsPublisher to send aggregated stats to Redis only."""
        if not self.enable_analytics_publisher:
            self.logger.warning(
                "[ANALYTICS_INIT_SKIP] Analytics publisher disabled via enable_analytics_publisher=False"
            )
            return

        try:
            # Check if we have any Redis cameras - if not, skip initialization
            has_redis_cameras = False
            redis_camera_count = 0
            non_redis_camera_count = 0

            for cfg in self.camera_configs.values():
                sc = cfg.stream_config or {}
                st = sc.get("stream_type", "redis").lower()
                if st == "redis":
                    has_redis_cameras = True
                    redis_camera_count += 1
                else:
                    non_redis_camera_count += 1

            if not has_redis_cameras:
                self.logger.warning(
                    f"[ANALYTICS_INIT_SKIP] No Redis cameras found. "
                    f"Total cameras: {len(self.camera_configs)}, "
                    f"Non-Redis cameras: {non_redis_camera_count}. "
                    "AnalyticsPublisher requires at least one Redis camera."
                )
                return

            self.logger.info(
                f"[ANALYTICS_INIT] Found {redis_camera_count} Redis cameras, "
                f"{non_redis_camera_count} non-Redis cameras. Proceeding with initialization."
            )

            # Find connection params from camera configs
            redis_host = "localhost"
            redis_port = 6379
            redis_password = None
            redis_username = None
            redis_db = 0

            # Extract Redis connection info from camera configs
            redis_found = False

            for cfg in self.camera_configs.values():
                sc = cfg.stream_config or {}
                st = sc.get("stream_type", "redis").lower()

                # Log for debugging
                self.logger.info(
                    f"Analytics init - Camera {cfg.camera_id}: stream_type={st}, "
                    f"config_keys={list(sc.keys())}"
                )

                if st == "redis":
                    redis_host = sc.get("host") or redis_host
                    redis_port = sc.get("port") or redis_port
                    redis_password = sc.get("password", redis_password)
                    redis_username = sc.get("username", redis_username)
                    redis_db = sc.get("db", redis_db)
                    if not redis_found:
                        self.logger.info(f"Found Redis config from camera {cfg.camera_id}: {redis_host}:{redis_port}")
                        redis_found = True
                        break  # Use first Redis config found

            # Initialize analytics publisher (don't start yet) - Redis only
            self.analytics_publisher = AnalyticsPublisher(
                camera_configs=self.camera_configs,
                aggregation_interval=300,  # 5 minutes
                publish_interval=60,  # Publish every 60 seconds
                app_deployment_id=self.app_deployment_id,
                inference_pipeline_id=self.inference_pipeline_id,
                deployment_instance_id=self.deployment_instance_id,
                app_id=self.app_id,
                app_name=self.app_name,
                app_version=self.app_version,
                redis_host=redis_host,
                redis_port=redis_port,
                redis_password=redis_password,
                redis_username=redis_username,
                redis_db=redis_db,
                kafka_bootstrap_servers=None,
                enable_kafka=False,  # Disable Kafka publishing
            )

            self.logger.info(
                f"[ANALYTICS_INIT_SUCCESS] Initialized AnalyticsPublisher "
                f"(Redis: {redis_host}:{redis_port}, aggregation: 5min, publish: 60s)"
            )
        except Exception as e:
            self.analytics_publisher = None
            self.logger.warning(
                f"[ANALYTICS_INIT_FAIL] Analytics publisher initialization failed: {e}. "
                "Proceeding without analytics publishing."
            )

    def _ensure_frame_cache_initialized(self) -> None:
        """
        Ensure frame cache is initialized if we have Redis cameras.
        This is called when cameras are added dynamically to enable lazy initialization.
        """
        if self.frame_cache is not None:
            # Already initialized
            return

        # Check if we have Redis cameras now
        has_redis = any(
            cfg.stream_config.get("stream_type", "").lower() == "redis"
            for cfg in self.camera_configs.values()
        )

        if has_redis:
            self.logger.info(
                "Redis camera detected - lazy initializing frame cache..."
            )
            try:
                self._initialize_frame_cache()
                if self.frame_cache:
                    self.logger.info("✓ Frame cache lazy initialization successful")
                    
                    # CRITICAL: Update ProducerWorker's reference to the newly initialized frame_cache
                    # Without this, ProducerWorker would still have frame_cache=None
                    for producer in self.producer_workers:
                        producer.frame_cache = self.frame_cache
                        self.logger.info(
                            f"✓ Updated ProducerWorker-{producer.worker_id} with frame_cache reference"
                        )
                else:
                    self.logger.warning("✗ Frame cache initialization returned None")
            except Exception as e:
                self.logger.error(f"✗ Frame cache lazy initialization failed: {e}", exc_info=True)

    def _ensure_analytics_publisher_initialized(self) -> None:
        """
        Ensure analytics publisher is initialized and started if we have Redis cameras.
        This is called when cameras are added dynamically to enable lazy initialization.
        """
        if self.analytics_publisher is not None:
            # Already initialized
            return

        if not self.enable_analytics_publisher:
            # Disabled
            return

        # Check if we have Redis cameras now
        has_redis = any(
            cfg.stream_config.get("stream_type", "").lower() == "redis"
            for cfg in self.camera_configs.values()
        )

        if has_redis:
            self.logger.info(
                "Redis camera detected - lazy initializing analytics publisher..."
            )
            try:
                self._initialize_analytics_publisher()

                # Start the analytics publisher thread if pipeline is running and publisher was initialized
                if self.running and self.analytics_publisher:
                    try:
                        analytics_thread = self.analytics_publisher.start()
                        self.worker_threads.append(analytics_thread)
                        self.logger.info("✓ Analytics publisher lazy initialization successful and thread started")

                        # Update ProducerWorker's reference to the newly initialized analytics_publisher
                        for producer in self.producer_workers:
                            producer.analytics_publisher = self.analytics_publisher
                            self.logger.info(
                                f"✓ Updated ProducerWorker-{producer.worker_id} with analytics_publisher reference"
                            )
                    except Exception as e:
                        self.logger.error(f"✗ Failed to start analytics publisher thread: {e}", exc_info=True)
                elif self.analytics_publisher:
                    self.logger.info("✓ Analytics publisher initialized but pipeline not running yet")
                    # Still update ProducerWorker references for when pipeline starts
                    for producer in self.producer_workers:
                        producer.analytics_publisher = self.analytics_publisher
                else:
                    self.logger.warning("✗ Analytics publisher initialization returned None")
            except Exception as e:
                self.logger.error(f"✗ Analytics publisher lazy initialization failed: {e}", exc_info=True)
