"""
InferenceInterface: Thread-safe inference with worker queue routing.

THREAD SAFETY & CONCURRENT REQUEST HANDLING:
============================================

This module solves the greenlet thread context switching problem that occurs when:
1. Streaming frames are being processed continuously through the StreamingPipeline
2. Direct API calls (e.g., identity images for face recognition) arrive simultaneously

The Problem:
-----------
- Streaming frames are processed by inference worker processes with their own models
- Direct API calls attempt to use models in the main process from different thread contexts
- Models using gevent/greenlet internally cannot switch between different greenlet contexts
- This causes: "Cannot switch to a different thread" errors

The Solution (Worker Queue Routing):
-----------------------------------
1. StreamingPipeline creates inference worker processes that load their own models
2. When pipeline is active, ALL inference requests (streaming + direct API) are routed
   through the same worker queue (inference_queue)
3. Direct API calls (identity images) submit tasks to the worker queue and wait for
   responses via a dedicated response queue (direct_api_response_queue)
4. This ensures all inference uses the same greenlet context (worker process)
5. High-priority requests bypass the streaming queue backpressure with priority handling

Benefits:
--------
- No greenlet thread context errors (all inference in worker process context)
- Identity images work during streaming
- Natural frame skipping: Workers process identity images, streaming frames queue up
  and may be dropped if queue fills (acceptable for continuous video streams)
- Simple, robust architecture using multiprocessing queues

Usage:
-----
1. StreamingPipeline calls: inference_interface.set_worker_queues(input_queue, response_queue)
2. Direct API calls automatically route through worker queue when pipeline is active
3. High-priority requests (identity images) get dedicated handling
"""

from matrice_inference.server.model.model_manager_wrapper import ModelManagerWrapper
from typing import Dict, Any, List, Optional, Tuple, Union
from datetime import datetime, timezone
import logging
import time
import asyncio
import multiprocessing as mp
import uuid
import queue
from matrice_analytics.post_processing.post_processor import PostProcessor

class InferenceInterface:
    """Interface for proxying requests to model servers with optional post-processing."""

    def __init__(
        self,
        model_manager_wrapper: ModelManagerWrapper,
        post_processor: Optional[PostProcessor] = None,
    ):
        """
        Initialize the inference interface.

        Args:
            model_manager: Model manager for model inference
            post_processor: Post processor for post-processing
        """
        self.logger = logging.getLogger(__name__)
        self.model_manager_wrapper = model_manager_wrapper
        self.post_processor = post_processor
        self.latest_inference_time = datetime.now(timezone.utc)
        self.pipeline_event_loop: Optional[asyncio.AbstractEventLoop] = None

        # Worker queue routing for direct API calls
        # When set, ALL inference requests are routed through worker processes
        # to avoid greenlet context switching issues
        self._worker_input_queues: Optional[List[mp.Queue]] = None
        self._worker_response_queue: Optional[mp.Queue] = None
        self._use_worker_queue_routing = False
        self._direct_api_worker_counter = 0  # Round-robin counter for load balancing

        # Track concurrent inference requests for monitoring
        self._active_inference_count = 0
        self._inference_count_lock = asyncio.Lock() if asyncio else None

    def get_latest_inference_time(self) -> datetime:
        """Get the latest inference time."""
        return self.latest_inference_time

    def set_pipeline_event_loop(self, event_loop: asyncio.AbstractEventLoop) -> None:
        """Set the pipeline event loop for thread-safe async operations.

        Args:
            event_loop: Event loop from StreamingPipeline
        """
        self.pipeline_event_loop = event_loop
        self.logger.info("Pipeline event loop registered for thread-safe inference")

    def set_worker_queues(
        self,
        input_queues: List[mp.Queue],
        response_queue: mp.Queue,
    ) -> None:
        """Set worker queues for routing direct API calls through inference workers.

        When set, direct API calls (e.g., identity images for face recognition) are
        routed through the same inference worker processes that handle streaming frames.
        This avoids greenlet context switching issues by ensuring all model inference
        happens in the worker process context.

        Args:
            input_queues: List of multiprocessing queues (one per worker) for submitting tasks
            response_queue: Multiprocessing queue for receiving inference results
        """
        self._worker_input_queues = input_queues
        self._worker_response_queue = response_queue
        self._use_worker_queue_routing = True
        self._direct_api_worker_counter = 0  # Round-robin counter for load balancing
        self.logger.info(
            f"Worker queue routing enabled - direct API calls will use {len(input_queues)} inference workers"
        )

    def disable_worker_queue_routing(self) -> None:
        """Disable worker queue routing (used when pipeline stops)."""
        self._use_worker_queue_routing = False
        self._worker_input_queues = None
        self._worker_response_queue = None
        self._direct_api_worker_counter = 0
        self.logger.info("Worker queue routing disabled")

    def has_async_predict(self) -> bool:
        """Check if async_predict is available in the underlying model manager.

        Returns:
            bool: True if async_predict is available, False otherwise
        """
        try:
            # Check if model_manager_wrapper has model_manager attribute
            if not hasattr(self.model_manager_wrapper, 'model_manager'):
                return False

            model_manager = self.model_manager_wrapper.model_manager

            # Check if model_manager has async_predict and it's not None
            if hasattr(model_manager, 'async_predict') and model_manager.async_predict is not None:
                return True

            return False
        except Exception as e:
            self.logger.warning(f"Error checking async_predict availability: {e}")
            return False

    def _route_through_worker_queue(
        self,
        input: Any,
        extra_params: Optional[Dict[str, Any]] = None,
        stream_key: Optional[str] = None,
        stream_info: Optional[Dict[str, Any]] = None,
        timeout: float = 5.0,
    ) -> Tuple[Any, bool]:
        """Route inference through worker queue to avoid greenlet context issues.

        This method submits the inference task to the same queue used by streaming
        frames, ensuring the model is accessed in the worker process context where
        it was loaded. This avoids greenlet "Cannot switch to a different thread" errors.

        Args:
            input: Input data (image bytes)
            extra_params: Additional parameters for inference
            stream_key: Stream key identifier
            stream_info: Stream metadata
            timeout: Maximum time to wait for response (seconds)

        Returns:
            Tuple of (results, success_flag)

        Raises:
            RuntimeError: If worker queue routing fails
        """
        if not self._worker_input_queues:
            raise RuntimeError("Worker queues not configured for routing")

        # Generate unique request ID for correlation
        request_id = str(uuid.uuid4())

        # Create a dedicated response queue for this request to avoid cross-talk
        response_queue = mp.Queue(maxsize=1)

        # Create task for worker queue
        # Uses special "direct_api" type so workers know to send response back
        task = {
            "type": "direct_api",
            "request_id": request_id,
            "input_bytes": input if isinstance(input, bytes) else bytes(input),
            "extra_params": extra_params or {},
            "stream_key": stream_key or f"direct_api_{request_id}",
            "stream_info": stream_info,
            "response_queue": response_queue,
            # Required fields for worker validation (using placeholder values)
            "camera_id": f"direct_api_{request_id[:8]}",
            "frame_id": request_id,
            "message": {"type": "direct_api"},
            "camera_config": {"type": "direct_api"},
        }

        # Round-robin select a worker queue for load balancing
        num_workers = len(self._worker_input_queues)
        worker_id = self._direct_api_worker_counter % num_workers
        self._direct_api_worker_counter += 1
        target_queue = self._worker_input_queues[worker_id]

        self.logger.debug(f"Submitting direct API task {request_id} to worker {worker_id}")

        try:
            # Submit task to worker queue (non-blocking with short timeout)
            # This ensures we don't block forever if queue is full
            target_queue.put(task, timeout=5.0)
        except Exception as e:
            self.logger.error(f"Failed to submit task to worker queue {worker_id}: {e}")
            raise RuntimeError(f"Worker queue submission failed: {e}") from e

        # Wait for response on the dedicated response queue
        try:
            result = response_queue.get(timeout=timeout)
        except Exception:
            raise RuntimeError(
                f"Timeout waiting for worker response (request_id={request_id}, timeout={timeout}s)"
            )

        # Extract result
        if result.get("success"):
            self.logger.debug(f"Direct API task {request_id} completed successfully")
            return result.get("model_result"), True
        else:
            error_msg = result.get("error", "Unknown worker error")
            self.logger.error(f"Direct API task {request_id} failed: {error_msg}")
            return None, False

    async def _inference_via_worker_queue(
        self,
        input: Any,
        extra_params: Optional[Dict[str, Any]] = None,
        apply_post_processing: bool = False,
        post_processing_config: Optional[Union[Dict[str, Any], str]] = None,
        stream_key: Optional[str] = None,
        stream_info: Optional[Dict[str, Any]] = None,
    ) -> Tuple[Any, Optional[Dict[str, Any]]]:
        """Async wrapper for worker queue inference with optional post-processing.

        Routes inference through worker queue and handles post-processing if requested.
        This method is used for high-priority requests (e.g., identity images) when
        streaming is active to avoid greenlet context switching issues.

        Args:
            input: Input data (image bytes)
            extra_params: Additional parameters for inference
            apply_post_processing: Whether to apply post-processing
            post_processing_config: Configuration for post-processing
            stream_key: Stream key identifier
            stream_info: Stream metadata

        Returns:
            Tuple of (results, metadata)
        """
        model_start_time = time.time()

        # Update latest inference time
        self.latest_inference_time = datetime.now(timezone.utc)

        try:
            # Route through worker queue (synchronous call)
            # Run in thread pool to avoid blocking async event loop
            loop = asyncio.get_event_loop()
            raw_results, success = await loop.run_in_executor(
                None,  # Use default executor
                self._route_through_worker_queue,
                input,
                extra_params,
                stream_key,
                stream_info,
                6.0,  # timeout
            )

            model_inference_time = time.time() - model_start_time

            if not success:
                raise RuntimeError("Model inference via worker queue failed")

            self.logger.debug(
                f"Worker queue inference executed stream_key={stream_key} "
                f"time={model_inference_time:.4f}s"
            )

        except Exception as exc:
            error_msg = str(exc)
            if "greenlet" in error_msg.lower() or "cannot switch" in error_msg.lower():
                self.logger.error(
                    f"Greenlet error in worker queue routing. This is unexpected - "
                    f"worker queue routing should avoid greenlet issues. Error: {error_msg}",
                    exc_info=True
                )
            else:
                self.logger.error(f"Worker queue inference failed: {error_msg}", exc_info=True)
            raise RuntimeError(f"Worker queue inference failed: {error_msg}") from exc

        # If no post-processing requested, return raw results
        if not apply_post_processing or not self.post_processor:
            return raw_results, {
                "timing_metadata": {
                    "model_inference_time_sec": model_inference_time,
                    "post_processing_time_sec": 0.0,
                    "total_time_sec": model_inference_time,
                },
                "routing": "worker_queue",
            }

        # Apply post-processing using PostProcessor
        try:
            post_processing_start_time = time.time()

            result = await self.post_processor.process(
                data=raw_results,
                config=post_processing_config,
                input_bytes=input if isinstance(input, bytes) else None,
                stream_key=stream_key,
                stream_info=stream_info
            )

            post_processing_time = time.time() - post_processing_start_time

            if result.is_success():
                processed_raw_results = [] if (
                    hasattr(result, 'usecase') and result.usecase == 'face_recognition'
                ) else raw_results

                agg_summary = {}
                if hasattr(result, 'data') and isinstance(result.data, dict):
                    agg_summary = result.data.get("agg_summary", {})

                post_processing_result = {
                    "status": "success",
                    "processing_time": result.processing_time,
                    "usecase": getattr(result, 'usecase', ''),
                    "category": getattr(result, 'category', ''),
                    "summary": getattr(result, 'summary', ''),
                    "insights": getattr(result, 'insights', []),
                    "metrics": getattr(result, 'metrics', {}),
                    "predictions": getattr(result, 'predictions', []),
                    "agg_summary": agg_summary,
                    "stream_key": stream_key or "default_stream",
                    "timing_metadata": {
                        "model_inference_time_sec": model_inference_time,
                        "post_processing_time_sec": post_processing_time,
                        "total_time_sec": model_inference_time + post_processing_time,
                    },
                    "routing": "worker_queue",
                }

                return processed_raw_results, post_processing_result
            else:
                self.logger.error(f"Post-processing failed: {result.error_message}")
                return raw_results, {
                    "status": "post_processing_failed",
                    "error": result.error_message,
                    "error_type": getattr(result, 'error_type', 'ProcessingError'),
                    "processing_time": result.processing_time,
                    "processed_data": raw_results,
                    "stream_key": stream_key or "default_stream",
                    "timing_metadata": {
                        "model_inference_time_sec": model_inference_time,
                        "post_processing_time_sec": post_processing_time,
                        "total_time_sec": model_inference_time + post_processing_time,
                    },
                    "routing": "worker_queue",
                }

        except Exception as e:
            post_processing_time = time.time() - post_processing_start_time
            self.logger.error(f"Post-processing exception: {str(e)}", exc_info=True)

            return raw_results, {
                "status": "post_processing_failed",
                "error": str(e),
                "error_type": type(e).__name__,
                "processed_data": raw_results,
                "stream_key": stream_key or "default_stream",
                "timing_metadata": {
                    "model_inference_time_sec": model_inference_time,
                    "post_processing_time_sec": post_processing_time,
                    "total_time_sec": model_inference_time + post_processing_time,
                },
                "routing": "worker_queue",
            }
    
    async def inference(
        self,
        input: Any,
        extra_params: Optional[Dict[str, Any]] = None,
        apply_post_processing: bool = False,
        post_processing_config: Optional[Union[Dict[str, Any], str]] = None,
        stream_key: Optional[str] = None,
        stream_info: Optional[Dict[str, Any]] = None,
        camera_info: Optional[Dict[str, Any]] = None,
        pipeline_event_loop: Optional[asyncio.AbstractEventLoop] = None,
        is_high_priority: bool = False,
    ) -> Tuple[Any, Optional[Dict[str, Any]]]:
        """Perform inference using the appropriate client with optional post-processing.

        Args:
            input: Primary input data (e.g., image bytes, numpy array)
            extra_params: Additional parameters for inference (optional)
            apply_post_processing: Whether to apply post-processing
            post_processing_config: Configuration for post-processing
            stream_key: Unique identifier for the input stream
            stream_info: Additional metadata about the stream (optional)
            camera_info: Additional metadata about the camera/source (optional)
            pipeline_event_loop: Event loop from StreamingPipeline (if available)
            is_high_priority: If True, this is a high-priority request (e.g., identity image)

        Returns:
            A tuple containing:
                - The inference results (raw or post-processed)
                - Metadata about the inference and post-processing (if applicable)

        Note:
            High-priority requests (like identity images for face recognition) are routed
            through the worker queue when streaming is active. This avoids greenlet context
            switching issues by ensuring all model inference happens in the worker process.
            During their execution, streaming frames may be naturally skipped if the
            inference queue fills up, which is acceptable for continuous streaming scenarios.
        """
        if input is None:
            raise ValueError("Input cannot be None")

        # Log high-priority requests for monitoring
        if is_high_priority:
            self.logger.info(f"Processing high-priority inference request (stream_key={stream_key})")

        # CRITICAL: Route high-priority requests through worker queue when streaming is active
        # This avoids greenlet "Cannot switch to a different thread" errors
        # Only applies when: 1) high priority request AND 2) worker queue routing enabled AND 3) queues available
        if (
            is_high_priority
            and self._use_worker_queue_routing
            and self._worker_input_queues is not None
        ):
            self.logger.info(
                f"Routing high-priority request through worker queue to avoid greenlet issues "
                f"(stream_key={stream_key})"
            )
            try:
                return await self._inference_via_worker_queue(
                    input=input,
                    extra_params=extra_params,
                    apply_post_processing=apply_post_processing,
                    post_processing_config=post_processing_config,
                    stream_key=stream_key,
                    stream_info=stream_info,
                )
            except Exception as worker_exc:
                # If worker queue routing fails, log warning and fall back to direct inference
                # This ensures the request still has a chance to complete
                self.logger.warning(
                    f"Worker queue routing failed, falling back to direct inference: {worker_exc}"
                )
                # Continue to original inference path below

        # Measure model inference time
        model_start_time = time.time()

        # Update latest inference time
        self.latest_inference_time = datetime.now(timezone.utc)

        # Run model inference with proper thread-safety
        try:
            # Use provided event loop or fall back to stored pipeline event loop
            event_loop_to_use = pipeline_event_loop or self.pipeline_event_loop

            # If event loop is available and has async_predict, use thread-safe async inference
            if event_loop_to_use and self.has_async_predict():
                # Run async inference in pipeline's event loop from any thread
                # This ensures identity images and streaming frames use the same event loop
                # This prevents greenlet/gevent thread context switching errors
                self.logger.debug(
                    f"Using thread-safe async inference via pipeline event loop "
                    f"(priority={'high' if is_high_priority else 'normal'})"
                )
                future = asyncio.run_coroutine_threadsafe(
                    self.model_manager_wrapper.async_inference(
                        input=input,
                        extra_params=extra_params,
                        stream_key=stream_key,
                        stream_info=stream_info
                    ),
                    event_loop_to_use
                )
                # High-priority requests get longer timeout
                timeout = 10.0 if is_high_priority else 6.0
                raw_results, success = future.result(timeout=timeout)
            else:
                # Fall back to sync inference (no async support or no event loop)
                self.logger.debug("Using synchronous inference (no async support or event loop)")
                raw_results, success = self.model_manager_wrapper.inference(
                    input=input,
                    extra_params=extra_params,
                    stream_key=stream_key,
                    stream_info=stream_info
                )

            model_inference_time = time.time() - model_start_time

            if not success:
                raise RuntimeError("Model inference failed")

            self.logger.debug(
                f"Model inference executed stream_key={stream_key} "
                f"time={model_inference_time:.4f}s priority={'high' if is_high_priority else 'normal'}"
            )

        except Exception as exc:
            # Add context about greenlet errors
            error_msg = str(exc)
            if "greenlet" in error_msg.lower() or "cannot switch" in error_msg.lower():
                self.logger.error(
                    f"Greenlet thread context error detected. This typically means the model "
                    f"is being accessed from multiple threads without proper event loop coordination. "
                    f"Error: {error_msg}",
                    exc_info=True
                )
            else:
                self.logger.error(f"Model inference failed: {error_msg}", exc_info=True)
            raise RuntimeError(f"Model inference failed: {error_msg}") from exc
        
        # If no post-processing requested, return raw results
        if not apply_post_processing or not self.post_processor:
            return raw_results, {
                "timing_metadata": {
                    "model_inference_time_sec": model_inference_time,
                    "post_processing_time_sec": 0.0,
                    "total_time_sec": model_inference_time,
                }
            }

        # Apply post-processing using PostProcessor
        try:
            post_processing_start_time = time.time()
            
            # Use PostProcessor.process() method directly
            result = await self.post_processor.process(
                data=raw_results,
                config=post_processing_config,  # Use stream_key as fallback if no config
                input_bytes=input if isinstance(input, bytes) else None,
                stream_key=stream_key,
                stream_info=stream_info
            )
            
            post_processing_time = time.time() - post_processing_start_time
            
            # Format the response based on PostProcessor result
            if result.is_success():
                # For face recognition use case, return empty raw results
                processed_raw_results = [] if (
                    hasattr(result, 'usecase') and result.usecase == 'face_recognition'
                ) else raw_results
                
                # Extract agg_summary from result data if available
                agg_summary = {}
                if hasattr(result, 'data') and isinstance(result.data, dict):
                    agg_summary = result.data.get("agg_summary", {})
                
                post_processing_result = {
                    "status": "success",
                    "processing_time": result.processing_time,
                    "usecase": getattr(result, 'usecase', ''),
                    "category": getattr(result, 'category', ''),
                    "summary": getattr(result, 'summary', ''),
                    "insights": getattr(result, 'insights', []),
                    "metrics": getattr(result, 'metrics', {}),
                    "predictions": getattr(result, 'predictions', []),
                    "agg_summary": agg_summary,
                    "stream_key": stream_key or "default_stream",
                    "timing_metadata": {
                        "model_inference_time_sec": model_inference_time,
                        "post_processing_time_sec": post_processing_time,
                        "total_time_sec": model_inference_time + post_processing_time,
                    }
                }
                
                return processed_raw_results, post_processing_result
            else:
                # Post-processing failed
                self.logger.error(f"Post-processing failed: {result.error_message}")
                return raw_results, {
                    "status": "post_processing_failed",
                    "error": result.error_message,
                    "error_type": getattr(result, 'error_type', 'ProcessingError'),
                    "processing_time": result.processing_time,
                    "processed_data": raw_results,
                    "stream_key": stream_key or "default_stream",
                    "timing_metadata": {
                        "model_inference_time_sec": model_inference_time,
                        "post_processing_time_sec": post_processing_time,
                        "total_time_sec": model_inference_time + post_processing_time,
                    }
                }
                
        except Exception as e:
            post_processing_time = time.time() - post_processing_start_time
            self.logger.error(f"Post-processing exception: {str(e)}", exc_info=True)
            
            return raw_results, {
                "status": "post_processing_failed",
                "error": str(e),
                "error_type": type(e).__name__,
                "processed_data": raw_results,
                "stream_key": stream_key or "default_stream",
                "timing_metadata": {
                    "model_inference_time_sec": model_inference_time,
                    "post_processing_time_sec": post_processing_time,
                    "total_time_sec": model_inference_time + post_processing_time,
                }
            }
    
    async def async_inference(
        self,
        input: Any,
        extra_params: Optional[Dict[str, Any]] = None,
        apply_post_processing: bool = False,
        post_processing_config: Optional[Union[Dict[str, Any], str]] = None,
        stream_key: Optional[str] = None,
        stream_info: Optional[Dict[str, Any]] = None,
        camera_info: Optional[Dict[str, Any]] = None,
        pipeline_event_loop: Optional[asyncio.AbstractEventLoop] = None,
    ) -> Tuple[Any, Optional[Dict[str, Any]]]:
        """Perform asynchronous inference using async_predict when available.

        This method MUST be called within the pipeline's event loop.
        For calls from other threads, use the regular inference() method which
        handles thread-safety automatically.

        Args:
            input: Primary input data (e.g., image bytes, numpy array)
            extra_params: Additional parameters for inference (optional)
            apply_post_processing: Whether to apply post-processing
            post_processing_config: Configuration for post-processing
            stream_key: Unique identifier for the input stream
            stream_info: Additional metadata about the stream (optional)
            camera_info: Additional metadata about the camera/source (optional)
            pipeline_event_loop: Event loop from StreamingPipeline (optional, for validation)

        Returns:
            A tuple containing:
                - The inference results (raw or post-processed)
                - Metadata about the inference and post-processing (if applicable)
        """
        if input is None:
            raise ValueError("Input cannot be None")

        # Measure model inference time
        model_start_time = time.time()

        # Update latest inference time
        self.latest_inference_time = datetime.now(timezone.utc)

        # Run asynchronous model inference
        try:
            raw_results, success = await self.model_manager_wrapper.async_inference(
                input=input,
                extra_params=extra_params,
                stream_key=stream_key,
                stream_info=stream_info
            )
            model_inference_time = time.time() - model_start_time

            if not success:
                raise RuntimeError("Model inference failed")

            self.logger.debug(
                f"Async model inference executed stream_key={stream_key} time={model_inference_time:.4f}s"
            )

        except Exception as exc:
            self.logger.error(f"Async model inference failed: {str(exc)}", exc_info=True)
            raise RuntimeError(f"Async model inference failed: {str(exc)}") from exc
        
        # If no post-processing requested, return raw results
        if not apply_post_processing or not self.post_processor:
            return raw_results, {
                "timing_metadata": {
                    "model_inference_time_sec": model_inference_time,
                    "post_processing_time_sec": 0.0,
                    "total_time_sec": model_inference_time,
                }
            }

        # Apply post-processing using PostProcessor
        try:
            post_processing_start_time = time.time()
            
            # Use PostProcessor.process() method directly (async)
            result = await self.post_processor.process(
                data=raw_results,
                config=post_processing_config,
                input_bytes=input if isinstance(input, bytes) else None,
                stream_key=stream_key,
                stream_info=stream_info
            )
            
            post_processing_time = time.time() - post_processing_start_time
            
            # Format the response based on PostProcessor result
            if result.is_success():
                # For face recognition use case, return empty raw results
                processed_raw_results = [] if (
                    hasattr(result, 'usecase') and result.usecase == 'face_recognition'
                ) else raw_results
                
                # Extract agg_summary from result data if available
                agg_summary = {}
                if hasattr(result, 'data') and isinstance(result.data, dict):
                    agg_summary = result.data.get("agg_summary", {})
                
                post_processing_result = {
                    "status": "success",
                    "processing_time": result.processing_time,
                    "usecase": getattr(result, 'usecase', ''),
                    "category": getattr(result, 'category', ''),
                    "summary": getattr(result, 'summary', ''),
                    "insights": getattr(result, 'insights', []),
                    "metrics": getattr(result, 'metrics', {}),
                    "predictions": getattr(result, 'predictions', []),
                    "agg_summary": agg_summary,
                    "stream_key": stream_key or "default_stream",
                    "timing_metadata": {
                        "model_inference_time_sec": model_inference_time,
                        "post_processing_time_sec": post_processing_time,
                        "total_time_sec": model_inference_time + post_processing_time,
                    }
                }
                
                return processed_raw_results, post_processing_result
            else:
                # Post-processing failed
                self.logger.error(f"Post-processing failed: {result.error_message}")
                return raw_results, {
                    "status": "post_processing_failed",
                    "error": result.error_message,
                    "error_type": getattr(result, 'error_type', 'ProcessingError'),
                    "processing_time": result.processing_time,
                    "processed_data": raw_results,
                    "stream_key": stream_key or "default_stream",
                    "timing_metadata": {
                        "model_inference_time_sec": model_inference_time,
                        "post_processing_time_sec": post_processing_time,
                        "total_time_sec": model_inference_time + post_processing_time,
                    }
                }
                
        except Exception as e:
            post_processing_time = time.time() - post_processing_start_time
            self.logger.error(f"Post-processing exception: {str(e)}", exc_info=True)
            
            return raw_results, {
                "status": "post_processing_failed",
                "error": str(e),
                "error_type": type(e).__name__,
                "processed_data": raw_results,
                "stream_key": stream_key or "default_stream",
                "timing_metadata": {
                    "model_inference_time_sec": model_inference_time,
                    "post_processing_time_sec": post_processing_time,
                    "total_time_sec": model_inference_time + post_processing_time,
                }
            }