"""Scheduler for fair task queue."""

import asyncio
import contextlib
import logging
from contextlib import AbstractAsyncContextManager
from datetime import UTC, datetime, timedelta
from typing import Any, NoReturn, Self

from haolib.background.fair.abstract import (
    AbstractFairQueueStorage,
    AbstractTaskAdapter,
    ComponentHealthStatus,
    FairnessAlgorithm,
    FairTask,
    HealthDetails,
    RetryStrategy,
    SchedulerDetails,
    TaskMetadata,
    TaskStatus,
)
from haolib.background.fair.adapters import TaskIQAdapter
from haolib.background.fair.exceptions import FairQueueSerializationError
from haolib.background.fair.extensions import HookManager, MiddlewareManager
from haolib.background.fair.resilience import CircuitBreaker, ExponentialBackoffRetryStrategy
from haolib.background.fair.results import NoOpResultHandler, TaskResultHandler
from haolib.background.fair.storages.redis import RedisFairQueueStorage

logger = logging.getLogger(__name__)

# ============================================================================
# Constants
# ============================================================================

# Maximum number of pending results to track before cleanup
# Prevents memory leaks in long-running schedulers
DEFAULT_MAX_PENDING_RESULTS = 10_000

# Maximum number of result events to track before cleanup
# Prevents memory leaks from result event tracking
DEFAULT_MAX_RESULT_EVENTS = 10_000

# Status verification constants
STATUS_VERIFICATION_MAX_ATTEMPTS = 3  # Maximum number of verification attempts for status updates
STATUS_VERIFICATION_RETRY_DELAY_SECONDS = 0.01  # Delay between verification retries in seconds


class FairQueueScheduler(AbstractAsyncContextManager["FairQueueScheduler"]):
    """Scheduler that dispatches tasks from fair queue to TaskIQ.

    Can be used as an async context manager:

    Example:
        ```python
        async with scheduler:
            # Scheduler is running
            await asyncio.sleep(10)
        # Scheduler is stopped gracefully
        ```

    """

    def __init__(
        self,
        storage: AbstractFairQueueStorage,
        task_adapter: AbstractTaskAdapter,
        fairness_algorithm: FairnessAlgorithm,
        poll_interval: timedelta = timedelta(seconds=0.1),
        error_backoff_interval: timedelta = timedelta(seconds=1.0),
        max_concurrent_tasks: int = 10,
        default_max_retries: int = 3,
        retry_strategy: RetryStrategy | None = None,
        hook_manager: HookManager | None = None,
        middleware_manager: MiddlewareManager | None = None,
        result_handler: TaskResultHandler | None = None,
        max_pending_results: int = DEFAULT_MAX_PENDING_RESULTS,
        max_result_events: int = DEFAULT_MAX_RESULT_EVENTS,
    ) -> None:
        """Initialize fair queue scheduler.

        Args:
            storage: Storage backend
            task_adapter: Task adapter for dispatching tasks
            fairness_algorithm: Fairness algorithm to use
            poll_interval: Interval between scheduler iterations
            error_backoff_interval: Backoff interval on errors
            max_concurrent_tasks: Maximum concurrent tasks to process
            default_max_retries: Default maximum retry attempts
            retry_strategy: Retry strategy to use (defaults to ExponentialBackoffRetryStrategy)
            hook_manager: Optional hook manager for event hooks
            middleware_manager: Optional middleware manager for cross-cutting concerns
            result_handler: Optional result handler for storing task results
            max_pending_results: Maximum number of pending results to track (prevents memory leaks)
            max_result_events: Maximum number of result events to track (prevents memory leaks)

        """
        self._storage = storage
        self._task_adapter = task_adapter
        self._fairness_algorithm = fairness_algorithm
        self._poll_interval = poll_interval
        self._error_backoff_interval = error_backoff_interval
        self._max_concurrent_tasks = max_concurrent_tasks
        self._default_max_retries = default_max_retries
        self._retry_strategy = retry_strategy if retry_strategy is not None else ExponentialBackoffRetryStrategy()
        self._hook_manager = hook_manager or HookManager()
        self._middleware_manager = middleware_manager or MiddlewareManager()
        self._result_handler = result_handler
        self._circuit_breaker = CircuitBreaker()

        self._stopped = False
        self._paused = False
        self._scheduler_task: asyncio.Task | None = None
        self._active_tasks: set[asyncio.Task] = set()
        self._processing_tasks: dict[str, tuple[asyncio.Task, FairTask]] = {}  # task_id -> (task, fair_task)
        # Track tasks waiting for results: task_id -> (task, group_id, dispatch_time, poll_count)
        # Bounded to prevent memory leaks - oldest entries are removed when limit is reached
        self._pending_results: dict[str, tuple[FairTask, str, datetime, int]] = {}
        # Event-driven result notifications: task_id -> asyncio.Event
        # Bounded to prevent memory leaks - cleaned up when tasks complete
        self._result_events: dict[str, asyncio.Event] = {}
        # Locks for thread-safe dictionary access
        self._pending_results_lock = asyncio.Lock()
        self._result_events_lock = asyncio.Lock()
        # Configurable limits for memory management
        self._max_pending_results: int = max_pending_results
        self._max_result_events: int = max_result_events
        self._result_polling_task: asyncio.Task | None = None
        self._result_poll_interval = timedelta(seconds=1.0)  # Base poll interval
        self._result_poll_timeout = timedelta(minutes=10)  # Stop polling after 10 minutes (reduced from 30)
        self._result_poll_max_interval = timedelta(seconds=30)  # Maximum poll interval with backoff

    async def start(self) -> None:
        """Start the scheduler."""
        if self._scheduler_task is not None:
            return

        self._stopped = False
        self._scheduler_task = asyncio.create_task(self._run())
        # Start result polling if result handler is configured
        if self._result_handler:
            self._result_polling_task = asyncio.create_task(self._poll_results_loop())

    async def stop(self, graceful: bool = True, shutdown_timeout: timedelta | None = None) -> None:
        """Stop the scheduler.

        Args:
            graceful: If True, wait for active tasks to complete. If False, cancel immediately.
            shutdown_timeout: Maximum time to wait for graceful shutdown. None means wait indefinitely.

        """
        self._stopped = True

        # Stop result polling task
        if self._result_polling_task:
            self._result_polling_task.cancel()
            with contextlib.suppress(asyncio.CancelledError):
                await self._result_polling_task
            self._result_polling_task = None

        if self._scheduler_task:
            self._scheduler_task.cancel()
            with contextlib.suppress(asyncio.CancelledError):
                await self._scheduler_task
            self._scheduler_task = None

        # Wait for active tasks to complete
        if graceful and self._active_tasks:
            if shutdown_timeout:
                try:
                    await asyncio.wait_for(
                        asyncio.gather(*self._active_tasks, return_exceptions=True),
                        timeout=shutdown_timeout.total_seconds(),
                    )
                except TimeoutError:
                    # Cancel remaining tasks on timeout
                    for task in self._active_tasks:
                        task.cancel()
                    await asyncio.gather(*self._active_tasks, return_exceptions=True)
            else:
                await asyncio.gather(*self._active_tasks, return_exceptions=True)
            self._active_tasks.clear()
        elif not graceful:
            # Cancel all active tasks immediately
            for task in self._active_tasks:
                task.cancel()
            await asyncio.gather(*self._active_tasks, return_exceptions=True)
            self._active_tasks.clear()

    async def health_check(self) -> bool:
        """Check if scheduler is healthy.

        Verifies:
        1. Scheduler is not stopped
        2. Storage backend is healthy
        3. Scheduler task is running (if started)

        Returns:
            True if scheduler is healthy, False otherwise

        """
        status = await self.get_health_status()
        return status.healthy

    async def get_health_status(self) -> ComponentHealthStatus:
        """Get detailed health status for scheduler.

        Returns:
            ComponentHealthStatus model with health status details

        """
        error: str | None = None
        healthy = True

        # Check if scheduler is stopped
        stopped = self._stopped
        if stopped:
            healthy = False
            error = "Scheduler is stopped"

        # Check storage health
        storage_healthy = False
        try:
            storage_healthy = await self._storage.health_check()
            if not storage_healthy:
                healthy = False
                if not error:
                    error = "Storage backend is unhealthy"
        except Exception as e:
            healthy = False
            if not error:
                error = f"Storage health check failed: {e}"

        # Check scheduler task status
        scheduler_task_status = "not_started"
        active_tasks_count = 0
        max_concurrent_tasks = self._max_concurrent_tasks

        if self._scheduler_task is not None:
            if self._scheduler_task.done():
                healthy = False
                if not error:
                    error = "Scheduler task has stopped"
                scheduler_task_status = "stopped"
                with contextlib.suppress(Exception):
                    # Get exception if task failed
                    exc = self._scheduler_task.exception()
                    if exc:
                        error = f"Scheduler task failed: {exc}"
            else:
                scheduler_task_status = "running"
            active_tasks_count = len(self._active_tasks)

        scheduler_details = SchedulerDetails(
            stopped=stopped,
            storage_healthy=storage_healthy,
            scheduler_task=scheduler_task_status,
            active_tasks_count=active_tasks_count,
            max_concurrent_tasks=max_concurrent_tasks,
        )

        return ComponentHealthStatus(
            healthy=healthy,
            component="scheduler",
            details=HealthDetails(custom_data=scheduler_details.model_dump()),
            error=error,
            timestamp=datetime.now(UTC),
        )

    def is_task_processing(self, task_id: str) -> bool:
        """Check if a task is currently being processed by this scheduler.

        Args:
            task_id: Task ID to check

        Returns:
            True if task is being processed, False otherwise

        """
        return task_id in self._processing_tasks

    def get_result_event(self, task_id: str) -> asyncio.Event:
        """Get or create an event for task result notification.

        This event will be set when the task completes (successfully or with failure).
        Events are automatically cleaned up when tasks complete to prevent memory leaks.

        Args:
            task_id: Task ID

        Returns:
            asyncio.Event that will be set when task completes

        """
        if task_id not in self._result_events:
            # Check if we need to clean up old events to prevent memory leaks
            if len(self._result_events) >= self._max_result_events:
                # Remove oldest events (those not in processing_tasks)
                # This is a best-effort cleanup - events for completed tasks should
                # already be cleaned up, but we handle edge cases here
                events_to_remove = [tid for tid in self._result_events if tid not in self._processing_tasks]
                # Remove up to 10% of events to make room
                remove_count = max(1, len(events_to_remove) // 10)
                for tid in events_to_remove[:remove_count]:
                    self._result_events.pop(tid, None)

            self._result_events[task_id] = asyncio.Event()
        return self._result_events[task_id]

    def cleanup_result_event(self, task_id: str) -> None:
        """Clean up result event for a task.

        Args:
            task_id: Task ID

        """
        self._result_events.pop(task_id, None)

    @property
    def pending_results_count(self) -> int:
        """Get the number of pending results.

        Returns:
            Number of tasks waiting for results

        """
        return len(self._pending_results)

    @property
    def result_events_count(self) -> int:
        """Get the number of result events.

        Returns:
            Number of result events being tracked

        """
        return len(self._result_events)

    @property
    def is_stopped(self) -> bool:
        """Check if the scheduler is stopped.

        Returns:
            True if scheduler is stopped, False otherwise

        """
        return self._stopped

    @property
    def is_paused(self) -> bool:
        """Check if the scheduler is paused.

        Returns:
            True if scheduler is paused, False otherwise

        """
        return self._paused

    async def cancel_processing_task(self, task_id: str) -> bool:
        """Cancel a task that is currently being processed.

        Args:
            task_id: Task ID to cancel

        Returns:
            True if task was cancelled, False if not found or not processing

        """
        if task_id not in self._processing_tasks:
            return False

        # Remove from pending results if it was there (task cancelled, no result to wait for)
        self._pending_results.pop(task_id, None)

        task_processing, fair_task = self._processing_tasks[task_id]

        # Cancel the processing task
        task_processing.cancel()

        # Mark task as cancelled in storage
        with contextlib.suppress(Exception):
            await self._storage.update_task_status(fair_task.task_id, fair_task.group_id, TaskStatus.CANCELLED)

        # Remove from tracking
        self._processing_tasks.pop(task_id, None)
        self._active_tasks.discard(task_processing)

        return True

    async def __aenter__(self) -> Self:
        """Enter async context manager."""
        await self.start()

        return self

    async def __aexit__(
        self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: object
    ) -> None:
        """Exit async context manager."""
        await self.stop(graceful=True)

    async def pause(self) -> None:
        """Pause the scheduler.

        When paused, the scheduler will stop processing new tasks but will
        continue processing tasks that are already in progress.

        """
        self._paused = True
        logger.info("FairQueueScheduler paused")

    async def resume(self) -> None:
        """Resume the scheduler.

        Resumes processing of tasks after a pause.

        """
        self._paused = False
        logger.info("FairQueueScheduler resumed")

    async def _run(self) -> None:  # noqa: C901, PLR0912, PLR0915
        """Main scheduler loop with error handling and circuit breakers."""
        while not self._stopped:
            try:
                # Check if paused
                if self._paused:
                    await asyncio.sleep(self._poll_interval.total_seconds())
                    continue

                # Check circuit breaker
                if not self._circuit_breaker.can_proceed():
                    # Circuit breaker is open, wait before retrying
                    await asyncio.sleep(self._error_backoff_interval.total_seconds())
                    continue

                # Limit concurrent tasks
                if len(self._active_tasks) >= self._max_concurrent_tasks:
                    await asyncio.sleep(self._poll_interval.total_seconds())
                    continue

                # Get next group based on fairness algorithm
                try:
                    group_id = await self._storage.get_next_group(self._fairness_algorithm)
                    self._circuit_breaker.record_success()
                except Exception as e:
                    self._circuit_breaker.record_failure()
                    # Check if storage is healthy before backing off
                    try:
                        storage_healthy = await self._storage.health_check()
                        if not storage_healthy:
                            backoff_seconds = self._error_backoff_interval.total_seconds()
                            logger.exception(
                                f"Storage backend is unhealthy. Backing off for {backoff_seconds}s. "
                                f"Check storage connection and capacity.",
                                extra={"error_type": type(e).__name__},
                            )
                    except Exception:
                        # Health check failed, assume storage is down
                        backoff_seconds = self._error_backoff_interval.total_seconds()
                        logger.exception(
                            f"Storage health check failed. Backing off for {backoff_seconds}s.",
                            extra={"error_type": type(e).__name__},
                        )
                    logger.exception(
                        "Failed to get next group",
                        extra={"error_type": type(e).__name__},
                    )
                    await asyncio.sleep(self._error_backoff_interval.total_seconds())
                    continue

                if group_id is None:
                    await asyncio.sleep(self._poll_interval.total_seconds())
                    continue

                # Check rate limit
                if not await self._check_rate_limit(group_id):
                    await asyncio.sleep(self._poll_interval.total_seconds())
                    continue

                # Dequeue task (expiration check happens in storage)
                try:
                    task = await self._storage.dequeue_task(group_id)
                    self._circuit_breaker.record_success()
                except Exception as e:
                    self._circuit_breaker.record_failure()
                    # Provide better error context for debugging
                    error_msg = (
                        f"Failed to dequeue task from group '{group_id}': {e}. "
                        f"This may indicate: 1) Storage connection issues, 2) Corrupted task data, "
                        f"3) Storage capacity problems. Check storage health and task data integrity."
                    )
                    logger.exception(
                        error_msg,
                        extra={"group_id": group_id, "error_type": type(e).__name__},
                    )
                    await asyncio.sleep(self._error_backoff_interval.total_seconds())
                    continue

                if task is None:
                    continue

                # Check task dependencies before processing
                if task.depends_on:
                    dependencies_met, dep_error = await self._check_dependencies(task)
                    if not dependencies_met:
                        if dep_error:
                            # Dependency failed/cancelled/expired - fail this task
                            error = RuntimeError(dep_error)
                            await self._handle_task_failure(task, group_id, error)
                            continue
                        # Dependencies not ready yet - re-enqueue task to check again later
                        task.status = TaskStatus.PENDING
                        try:
                            await self._storage.enqueue_task(task)
                        except Exception as e:
                            # If re-enqueue fails, log with full context and fail the task
                            logger.exception(
                                f"Failed to re-enqueue task {task.task_id} with unmet dependencies",
                                extra={
                                    "task_id": task.task_id,
                                    "group_id": group_id,
                                    "depends_on": task.depends_on,
                                    "error_type": type(e).__name__,
                                },
                            )
                            # Fail the task since we can't re-enqueue it
                            await self._handle_task_failure(
                                task,
                                group_id,
                                RuntimeError(f"Failed to re-enqueue task with unmet dependencies: {e}"),
                            )
                        continue

                # Fire dequeued event
                await self._hook_manager.fire_task_dequeued(task, group_id)

                # Note: Expiration check is now done in storage.dequeue_task
                # If task is expired, it will be marked as expired and next task will be dequeued

                # Process task asynchronously
                current_task_id = task.task_id
                current_task = task
                task_processing = asyncio.create_task(self._process_task(task, group_id))
                self._active_tasks.add(task_processing)
                self._processing_tasks[current_task_id] = (task_processing, current_task)

                # Create closures with captured values
                def remove_from_active(t: asyncio.Task) -> None:
                    self._active_tasks.discard(t)

                def remove_from_processing(t: asyncio.Task, task_id: str = current_task_id) -> None:
                    # Use task_id from default argument to properly capture the value
                    self._processing_tasks.pop(task_id, None)

                task_processing.add_done_callback(remove_from_active)
                task_processing.add_done_callback(remove_from_processing)

            except Exception as e:
                self._circuit_breaker.record_failure()
                logger.exception(
                    "Unexpected error in scheduler loop",
                    extra={"error_type": type(e).__name__},
                )
                await asyncio.sleep(self._error_backoff_interval.total_seconds())

    async def _check_dependencies(self, task: FairTask) -> tuple[bool, str | None]:
        """Check if all task dependencies are completed.

        Args:
            task: Task to check dependencies for

        Returns:
            Tuple of (success, error_message):
            - success: True if all dependencies are completed, False otherwise
            - error_message: Error message if dependency failed/cancelled/expired, None otherwise

        """
        if not task.depends_on:
            return (True, None)

        for dep_task_id in task.depends_on:
            # Get dependency task status
            # Try to get group_id from dependency task
            dep_task = await self._storage.get_task(dep_task_id, "")
            if not dep_task:
                # Dependency task not found, consider it not met
                logger.debug(
                    f"Dependency task {dep_task_id} not found for task {task.task_id}",
                    extra={"task_id": task.task_id, "depends_on": task.depends_on},
                )
                return (False, None)

            dep_status = await self._storage.get_task_status(dep_task_id, dep_task.group_id)
            if dep_status == TaskStatus.COMPLETED:
                # Dependency completed successfully, continue checking others
                continue
            if dep_status in (TaskStatus.FAILED, TaskStatus.CANCELLED, TaskStatus.EXPIRED):
                # Dependency failed/cancelled/expired - this task should also fail
                dep_status_value = dep_status.value if dep_status else "unknown"
                error_msg = (
                    f"Dependency task {dep_task_id} has status {dep_status_value}, "
                    f"cannot proceed with task {task.task_id}"
                )
                logger.warning(
                    error_msg,
                    extra={"task_id": task.task_id, "depends_on": task.depends_on, "dep_status": dep_status_value},
                )
                return (False, error_msg)
            # Dependency not completed yet (PENDING or PROCESSING)
            dep_status_value = dep_status.value if dep_status else None
            logger.debug(
                f"Dependency task {dep_task_id} not completed (status: {dep_status_value}) for task {task.task_id}",
                extra={"task_id": task.task_id, "depends_on": task.depends_on, "dep_status": dep_status_value},
            )
            return (False, None)

        # All dependencies completed
        return (True, None)

    def _raise_status_update_error(self, task_id: str, verified_status: TaskStatus | None) -> NoReturn:
        """Raise error for failed status update.

        Args:
            task_id: Task ID
            verified_status: Status that was verified (unexpected value)

        Raises:
            RuntimeError: Always raises with status update failure message

        """
        status_str = str(verified_status) if verified_status else "None"
        msg = f"Failed to update task {task_id} status to COMPLETED after retry. Current status: {status_str}"
        raise RuntimeError(msg)

    async def _process_task(self, task: FairTask, group_id: str) -> None:  # noqa: C901, PLR0912, PLR0915
        """Process a single task.

        Args:
            task: Task to process
            group_id: Group ID of the task

        """
        start_time = datetime.now(UTC)
        result: Any = None

        try:
            # Update task status to processing
            task.status = TaskStatus.PROCESSING
            # Store processing start time in metadata for recovery
            if task.metadata is None:
                task.metadata = TaskMetadata()
            task.metadata.processing_started_at = start_time.isoformat()
            # Update storage with processing status and metadata
            await self._storage.update_task_status(
                task.task_id,
                group_id,
                TaskStatus.PROCESSING,
                metadata=task.metadata,
            )

            # Fire processing event
            await self._hook_manager.fire_task_processing(task, group_id)

            # Apply middleware before processing
            task = await self._middleware_manager.apply_before_process(task)

            # Deserialize TaskInvocation from storage
            try:
                invocation = self._task_adapter.deserialize_invocation(task.task_data)
            except Exception as e:
                error_msg = f"Failed to deserialize task invocation: {e}"
                logger.exception(
                    "Failed to deserialize task invocation",
                    extra={"task_id": task.task_id, "group_id": group_id},
                )
                raise FairQueueSerializationError(
                    detail=error_msg,
                    task_id=task.task_id,
                    serialization_error=e,
                ) from e

            # Dispatch TaskInvocation via adapter
            # Check if result backend exists and is functional (not a dummy)
            # TaskIQ may have a dummy result backend that always returns None
            # We need to check if it's actually functional
            has_result_backend = False
            try:
                if isinstance(self._task_adapter, TaskIQAdapter):
                    result_backend = None
                    with contextlib.suppress(AttributeError):
                        result_backend = self._task_adapter.broker.result_backend  # type: ignore[attr-defined]

                    # Check if result backend exists and is not a dummy
                    # TaskIQ's dummy backend has a specific type/class we can check
                    if result_backend is not None:
                        # Check if it's a real backend by checking its type name
                        # Dummy backends typically have "Dummy" or "NoOp" in their class name
                        backend_type_name = type(result_backend).__name__
                        # Check for common dummy backend patterns in the class name
                        # This is the most reliable way to detect dummy backends without accessing
                        # private attributes that may not exist on all backends
                        is_dummy = (
                            "Dummy" in backend_type_name
                            or "NoOp" in backend_type_name
                            or "Null" in backend_type_name
                            or "Empty" in backend_type_name
                        )
                        has_result_backend = not is_dummy
                        logger.debug(
                            f"Task {task.task_id} result backend check: "
                            f"backend_type={backend_type_name}, has_result_backend={has_result_backend}",
                            extra={
                                "task_id": task.task_id,
                                "group_id": group_id,
                                "has_result_backend": has_result_backend,
                                "backend_type": backend_type_name,
                            },
                        )
            except Exception as backend_check_error:
                has_result_backend = False
                logger.debug(
                    f"Task {task.task_id} result backend check failed: {backend_check_error}",
                    extra={"task_id": task.task_id, "group_id": group_id},
                )

            try:
                await self._task_adapter.dispatch(invocation)
                # Try to retrieve result from TaskIQ result backend if available
                # This will return None if no result backend is configured or result not ready
                # Note: TaskIQ is async, so results may not be immediately available
                # For best results, configure a result handler to poll and store results
                result = None
                result_obj = None
                try:
                    # Try to get result (may return None if not ready or no backend)
                    if has_result_backend:
                        result_obj = await self._task_adapter.get_result(task.task_id)
                        if result_obj is not None:
                            # Check if result indicates a failure
                            is_err = False
                            with contextlib.suppress(AttributeError):
                                is_err = result_obj.is_err  # type: ignore[attr-defined]
                            if is_err:
                                # Task failed - extract error
                                error_value = None
                                try:
                                    error_value = result_obj.return_value  # type: ignore[attr-defined]
                                except AttributeError:
                                    with contextlib.suppress(AttributeError):
                                        error_value = result_obj.error  # type: ignore[attr-defined]

                                if isinstance(error_value, Exception):
                                    error = error_value
                                else:
                                    error = RuntimeError(str(error_value) if error_value else "Task failed")

                                logger.warning(
                                    f"Task {task.task_id} failed immediately: {error}",
                                    extra={"task_id": task.task_id, "group_id": group_id},
                                )
                                await self._handle_task_failure(task, group_id, error)
                                return
                            # Extract successful result value
                            try:
                                result = result_obj.return_value  # type: ignore[attr-defined]
                            except AttributeError:
                                try:
                                    result = result_obj.result  # type: ignore[attr-defined]
                                except AttributeError:
                                    result = result_obj

                            result_status = "available" if result is not None else "not available"
                            logger.debug(
                                f"Task {task.task_id} immediate result check: result={result_status}",
                                extra={
                                    "task_id": task.task_id,
                                    "group_id": group_id,
                                    "has_result": result is not None,
                                },
                            )
                except Exception as result_check_error:
                    # Result not available yet or error retrieving - will be None
                    result = None
                    logger.debug(
                        f"Task {task.task_id} immediate result check failed: {result_check_error}",
                        extra={"task_id": task.task_id, "group_id": group_id},
                    )
            except Exception as dispatch_error:
                # Dispatch failed - apply error middleware
                await self._middleware_manager.apply_on_error(task, dispatch_error)
                # Re-enqueue task for retry
                # This prevents task loss on transient dispatch failures
                await self._handle_task_failure(task, group_id, dispatch_error)
                return

            # Apply middleware after processing (non-critical, don't fail task if this fails)
            try:
                await self._middleware_manager.apply_after_process(task, result)
            except Exception as middleware_error:
                logger.warning(
                    f"Middleware after_process failed for task {task.task_id}: {middleware_error}",
                    extra={"task_id": task.task_id, "group_id": group_id},
                    exc_info=True,
                )
                # Continue processing - middleware failure shouldn't prevent task completion

            # Handle result if result handler is configured and result is immediately available
            # Note: NoOpResultHandler should be treated as no result handler for completion logic
            is_noop_handler = isinstance(self._result_handler, NoOpResultHandler)
            has_effective_result_handler = self._result_handler is not None and not is_noop_handler

            logger.debug(
                f"Checking result handler conditions for task {task.task_id}: "
                f"result_handler={has_effective_result_handler}, "
                f"result={result is not None}, is_noop={is_noop_handler}",
                extra={
                    "task_id": task.task_id,
                    "group_id": group_id,
                    "has_result_handler": has_effective_result_handler,
                    "has_result": result is not None,
                    "is_noop": is_noop_handler,
                },
            )

            if has_effective_result_handler and result is not None:
                if self._result_handler is not None:
                    try:
                        await self._result_handler.handle_result(task, group_id, result)
                    except Exception as result_error:
                        # Log result handler errors but don't fail the task
                        logger.warning(
                            f"Result handler failed for task {task.task_id}: {result_error}",
                            extra={"task_id": task.task_id, "group_id": group_id},
                            exc_info=True,
                        )
                # Result handled, mark as completed
                await self._storage.mark_task_completed(task.task_id, group_id)
                task.status = TaskStatus.COMPLETED
            elif has_effective_result_handler and result is None:
                # Result not immediately available
                # Only add to pending results if there's a result backend to poll
                # If no result backend, mark as completed immediately (result will never be available)
                if has_result_backend:
                    # Result backend exists, add to pending results for polling
                    # Task will be marked as completed when result is found by polling loop
                    await self._add_to_pending_results(task.task_id, task, group_id, start_time, 0)
                    logger.debug(
                        f"Task {task.task_id} dispatched, waiting for result from backend",
                        extra={"task_id": task.task_id, "group_id": group_id},
                    )
                    # Don't mark as completed yet - polling loop will handle it
                    return
                # No result backend, result will never be available
                # Mark as completed immediately (similar to no result handler case)
                logger.info(
                    f"Task {task.task_id} dispatched, no result backend - marking as completed immediately",
                    extra={"task_id": task.task_id, "group_id": group_id},
                )
                # Explicitly mark as completed (don't fall through, handle it here)
                try:
                    await self._storage.update_task_status(
                        task.task_id,
                        group_id,
                        TaskStatus.COMPLETED,
                    )
                    task.status = TaskStatus.COMPLETED
                    logger.info(
                        f"Successfully marked task {task.task_id} as COMPLETED (no result backend)",
                        extra={"task_id": task.task_id, "group_id": group_id},
                    )
                except Exception:
                    logger.exception(
                        f"Failed to mark task {task.task_id} as COMPLETED",
                        extra={"task_id": task.task_id, "group_id": group_id},
                    )
                    raise
                # Update group state and fire events (non-critical)
                processing_time = datetime.now(UTC) - start_time
                processing_seconds = processing_time.total_seconds()
                with contextlib.suppress(Exception):
                    await self._update_group_state(group_id, processing_time)
                with contextlib.suppress(Exception):
                    await self._hook_manager.fire_task_completed(task, group_id, processing_seconds)
                # Signal result event
                if task.task_id in self._result_events:
                    self._result_events[task.task_id].set()
                return
            else:
                # No result handler or no result backend, mark as completed immediately
                # First update status to COMPLETED, then clean up
                logger.debug(
                    f"Marking task {task.task_id} as COMPLETED (no result handler, result={result})",
                    extra={
                        "task_id": task.task_id,
                        "group_id": group_id,
                        "has_result_handler": bool(self._result_handler),
                    },
                )
                try:
                    # Update status first (critical operation)
                    logger.info(
                        f"Attempting to update task {task.task_id} status to COMPLETED",
                        extra={"task_id": task.task_id, "group_id": group_id},
                    )
                    try:
                        await self._storage.update_task_status(
                            task.task_id,
                            group_id,
                            TaskStatus.COMPLETED,
                        )
                        task.status = TaskStatus.COMPLETED
                        logger.info(
                            f"Successfully called update_task_status for task {task.task_id}",
                            extra={"task_id": task.task_id, "group_id": group_id},
                        )
                        # Immediately check what's actually in Redis
                        immediate_check = await self._storage.get_task_status(task.task_id, group_id)
                        status_str = str(immediate_check)
                        logger.info(
                            f"Immediate status check after update_task_status for task {task.task_id}: {status_str}",
                            extra={
                                "task_id": task.task_id,
                                "group_id": group_id,
                                "status": status_str,
                            },
                        )
                    except Exception:
                        logger.exception(
                            f"update_task_status raised exception for task {task.task_id}",
                            extra={"task_id": task.task_id, "group_id": group_id},
                        )
                        raise

                    # Verify status was actually updated in storage (with retry for race conditions)
                    verified_status = None
                    for verification_attempt in range(STATUS_VERIFICATION_MAX_ATTEMPTS):
                        verified_status = await self._storage.get_task_status(task.task_id, group_id)
                        if verified_status == TaskStatus.COMPLETED:
                            break
                        # Wait a tiny bit and retry (handles race conditions)
                        if verification_attempt < STATUS_VERIFICATION_MAX_ATTEMPTS - 1:
                            await asyncio.sleep(STATUS_VERIFICATION_RETRY_DELAY_SECONDS)

                    if verified_status != TaskStatus.COMPLETED:
                        status_str = str(verified_status)
                        logger.error(
                            f"Status verification failed for task {task.task_id}: expected COMPLETED, got {status_str}",
                            extra={
                                "task_id": task.task_id,
                                "group_id": group_id,
                                "actual_status": status_str,
                            },
                        )
                        # Try to update again
                        await self._storage.update_task_status(
                            task.task_id,
                            group_id,
                            TaskStatus.COMPLETED,
                        )
                        # Final verification
                        await asyncio.sleep(0.01)
                        verified_status = await self._storage.get_task_status(task.task_id, group_id)
                        if verified_status != TaskStatus.COMPLETED:
                            self._raise_status_update_error(task.task_id, verified_status)

                    logger.debug(
                        f"Successfully updated and verified task {task.task_id} status to COMPLETED",
                        extra={"task_id": task.task_id, "group_id": group_id},
                    )
                    # Then clean up task data (non-critical) - use mark_task_completed for cleanup
                    # but status is already updated above
                    try:
                        # Decrement global queue size
                        await self._storage.decrement_global_queue_size()
                        # Try to delete task data if it exists (non-critical)
                        # Note: We already updated status, so this is just cleanup

                        if isinstance(self._storage, RedisFairQueueStorage):
                            # Use the storage's method to delete task data
                            task_data = await self._storage.get_task(task.task_id, group_id)
                            if task_data:
                                # Task data exists, try to delete it
                                await self._storage.delete_task_data(task.task_id)
                    except Exception as cleanup_error:
                        # Cleanup failed, but status is already updated
                        logger.debug(
                            f"Failed to cleanup task {task.task_id} data (non-critical): {cleanup_error}",
                            extra={"task_id": task.task_id, "group_id": group_id},
                        )
                except Exception:
                    # Status update failed - this is critical
                    logger.exception(
                        f"Failed to update task {task.task_id} status to COMPLETED",
                        extra={"task_id": task.task_id, "group_id": group_id},
                    )
                    # Try mark_task_completed as fallback
                    try:
                        await self._storage.mark_task_completed(task.task_id, group_id)
                        task.status = TaskStatus.COMPLETED
                    except Exception:
                        logger.exception(
                            f"Fallback mark_task_completed also failed for task {task.task_id}",
                            extra={"task_id": task.task_id, "group_id": group_id},
                        )
                        # Update locally at least
                        task.status = TaskStatus.COMPLETED

            # Update group state (non-critical, don't fail task if this fails)
            processing_time = datetime.now(UTC) - start_time
            processing_seconds = processing_time.total_seconds()
            try:
                await self._update_group_state(group_id, processing_time)
            except Exception as state_error:
                logger.warning(
                    f"Failed to update group state for task {task.task_id}: {state_error}",
                    extra={"task_id": task.task_id, "group_id": group_id},
                    exc_info=True,
                )

            # Fire completed event (non-critical, don't fail task if this fails)
            try:
                await self._hook_manager.fire_task_completed(task, group_id, processing_seconds)
            except Exception as event_error:
                logger.warning(
                    f"Failed to fire completed event for task {task.task_id}: {event_error}",
                    extra={"task_id": task.task_id, "group_id": group_id},
                    exc_info=True,
                )

            # Signal result event if anyone is waiting
            async with self._result_events_lock:
                if task.task_id in self._result_events:
                    self._result_events[task.task_id].set()
                    # Always cleanup result event after setting it
                    self.cleanup_result_event(task.task_id)

        except Exception as e:
            # Apply error middleware
            await self._middleware_manager.apply_on_error(task, e)
            # Handle other failures (deserialization, storage, etc.)
            await self._handle_task_failure(task, group_id, e)
        finally:
            # Remove from pending results if it was added (only if not waiting for result)
            # If task is waiting for result, keep it in pending_results for polling
            async with self._pending_results_lock:
                if task.status != TaskStatus.PROCESSING or task.task_id not in self._pending_results:
                    self._pending_results.pop(task.task_id, None)

    async def _handle_task_failure(self, task: FairTask, group_id: str, error: Exception) -> None:
        """Handle task dispatch failure with retry logic.

        Args:
            task: Failed task
            group_id: Group ID of the task
            error: Exception that occurred

        """
        # Remove from pending results if it was there (task failed, no result to wait for)
        self._pending_results.pop(task.task_id, None)

        task.retry_count += 1

        # Use retry strategy to determine if we should retry
        if self._retry_strategy.should_retry(task, error, task.retry_count, task.max_retries):
            # Get backoff delay from strategy
            backoff_seconds = self._retry_strategy.get_backoff_delay(task, error, task.retry_count)
            await asyncio.sleep(backoff_seconds)

            # Re-enqueue task
            task.status = TaskStatus.PENDING
            try:
                await self._storage.enqueue_task(task)
            except Exception:
                await self._move_to_dead_letter_queue(task, group_id, error)
        else:
            # Move to dead letter queue
            await self._move_to_dead_letter_queue(task, group_id, error)

    async def _handle_expired_task(self, task: FairTask) -> None:
        """Handle expired task.

        Args:
            task: Expired task

        """
        task.status = TaskStatus.EXPIRED
        await self._storage.update_task_status(task.task_id, task.group_id, TaskStatus.EXPIRED)

    async def _move_to_dead_letter_queue(self, task: FairTask, group_id: str, error: Exception) -> None:
        """Move task to dead letter queue.

        Args:
            task: Failed task
            group_id: Group ID of the task
            error: Exception that caused the failure

        """
        task.status = TaskStatus.FAILED
        if task.metadata is None:
            task.metadata = TaskMetadata()
        task.metadata.error_message = str(error)
        task.metadata.error_type = type(error).__name__
        task.metadata.custom_data["failed_at"] = datetime.now(UTC).isoformat()
        task.metadata.custom_data["group_id"] = group_id

        # Fire failed event
        await self._hook_manager.fire_task_failed(task, group_id, error, task.retry_count)

        # Signal result event if anyone is waiting (task failed)
        if task.task_id in self._result_events:
            self._result_events[task.task_id].set()

        # Mark as failed (preserves task data for DLQ)
        await self._storage.mark_task_failed(task.task_id, group_id)

        # Add to dead letter queue (separate from main queue)
        await self._storage.add_to_dead_letter_queue(task)

    async def _check_rate_limit(self, group_id: str) -> bool:
        """Check if group can process a task based on rate limit.

        Uses atomic operation to prevent race conditions.

        Args:
            group_id: Group ID to check

        Returns:
            True if rate limit allows processing, False otherwise

        Note:
            This method requires storage to support atomic rate limiting.
            Non-atomic fallback is not provided as it can lead to race conditions.

        """
        # Get group state to check if rate limit is configured
        state = await self._storage.get_group_state(group_id)
        if not state or not state.rate_limit:
            return True

        rate_limit = state.rate_limit

        # Use atomic rate limit check - required for correctness
        # All storage implementations must support this
        allowed, window_start, current_count = await self._storage.check_and_increment_rate_limit(
            group_id,
            rate_limit.max_tasks,
            rate_limit.per_time,
        )

        # Update state with new values
        rate_limit.window_start = window_start
        rate_limit.current_count = current_count
        await self._storage.update_group_state(state)

        return allowed

    async def _update_group_state(self, group_id: str, processing_time: timedelta) -> None:
        """Update group state after processing task.

        Args:
            group_id: Group ID
            processing_time: Time taken to process the task

        """
        state = await self._storage.get_group_state(group_id)
        if state:
            self._fairness_algorithm.update_after_processing(state, processing_time)
            await self._storage.update_group_state(state)

    async def _add_to_pending_results(
        self,
        task_id: str,
        task: FairTask,
        group_id: str,
        dispatch_time: datetime,
        poll_count: int,
    ) -> None:
        """Add task to pending results with bounded queue management.

        If the queue is full, removes the oldest entry (by dispatch_time) to make room.
        This prevents memory leaks but may result in tasks being marked as completed
        without results if the queue is consistently full.

        Args:
            task_id: Task ID
            task: Task object
            group_id: Group ID
            dispatch_time: When task was dispatched
            poll_count: Current poll count

        """
        async with self._pending_results_lock:
            # If queue is full, remove oldest entry (by dispatch_time)
            if len(self._pending_results) >= self._max_pending_results:
                if not self._pending_results:
                    # Should not happen, but be safe
                    return

                # Find oldest entry by dispatch_time
                oldest_task_id = min(
                    self._pending_results.keys(),
                    key=lambda tid: self._pending_results[tid][2],  # dispatch_time is at index 2
                )
                oldest_task, oldest_group_id, oldest_dispatch_time, _ = self._pending_results.pop(oldest_task_id)

                # Mark oldest task as completed without result (timeout)
                elapsed = (datetime.now(UTC) - oldest_dispatch_time).total_seconds()
                logger.warning(
                    f"Pending results queue full ({self._max_pending_results} tasks). "
                    f"Removing oldest task {oldest_task_id} (dispatched {elapsed:.1f}s ago). "
                    f"Consider: 1) Increasing _max_pending_results, 2) Reducing result_poll_timeout, "
                    f"3) Processing results faster.",
                    extra={
                        "task_id": oldest_task_id,
                        "group_id": oldest_group_id,
                        "dispatch_time": oldest_dispatch_time.isoformat(),
                        "elapsed_seconds": elapsed,
                        "queue_size": self._max_pending_results,
                    },
                )
                # Schedule cleanup of the removed task (outside lock to avoid deadlock)
                cleanup_task = asyncio.create_task(self._cleanup_removed_pending_task(oldest_task, oldest_group_id))
                # Store reference to prevent garbage collection
                self._active_tasks.add(cleanup_task)

            self._pending_results[task_id] = (task, group_id, dispatch_time, poll_count)

    async def _cleanup_removed_pending_task(self, task: FairTask, group_id: str) -> None:
        """Clean up a task that was removed from pending results due to queue limit.

        Args:
            task: Task to clean up
            group_id: Group ID

        """
        try:
            # Mark as completed without result (timeout scenario)
            await self._storage.mark_task_completed(task.task_id, group_id)
            task.status = TaskStatus.COMPLETED
            logger.info(
                f"Marked task {task.task_id} as completed (removed from pending results due to queue limit)",
                extra={"task_id": task.task_id, "group_id": group_id},
            )
        except Exception:
            logger.exception(
                f"Failed to cleanup removed pending task {task.task_id}",
                extra={"task_id": task.task_id, "group_id": group_id},
            )

    async def _poll_results_loop(self) -> None:
        """Background loop that polls TaskIQ result backend for pending tasks.

        This loop periodically checks for results of tasks that were dispatched
        but didn't have results available immediately.

        """
        while not self._stopped:
            try:
                await asyncio.sleep(self._result_poll_interval.total_seconds())
                await self._poll_pending_results()
            except asyncio.CancelledError:
                break
            except Exception as e:
                logger.exception(
                    "Error in result polling loop",
                    extra={"error_type": type(e).__name__},
                )
                # Continue polling even on errors

    def _calculate_poll_interval(self, poll_count: int) -> timedelta:
        """Calculate poll interval with exponential backoff.

        Args:
            poll_count: Number of times this task has been polled

        Returns:
            Poll interval for this task

        """
        # Exponential backoff: 1s, 2s, 4s, 8s, 16s, 30s (max)
        base_seconds = self._result_poll_interval.total_seconds()
        backoff_seconds = min(base_seconds * (2**poll_count), self._result_poll_max_interval.total_seconds())
        return timedelta(seconds=backoff_seconds)

    async def _should_poll_task(self, dispatch_time: datetime, poll_count: int) -> bool:
        """Determine if a task should be polled in this iteration.

        Uses simplified exponential backoff: poll immediately, then after 1s, 2s, 4s, 8s, etc.
        up to max_interval.

        Args:
            dispatch_time: When task was dispatched
            poll_count: Number of times task has been polled

        Returns:
            True if task should be polled, False otherwise

        """
        current_time = datetime.now(UTC)
        elapsed = current_time - dispatch_time

        # Check timeout - always poll one last time before giving up
        if elapsed > self._result_poll_timeout:
            return True

        # Simplified exponential backoff: poll at 0s, 1s, 2s, 4s, 8s, 16s, 30s (max)
        if poll_count == 0:
            return True  # Always poll immediately

        # Calculate next poll time using exponential backoff
        poll_interval = self._calculate_poll_interval(poll_count - 1)
        next_poll_time = dispatch_time + poll_interval

        # For subsequent polls, calculate cumulative time
        if poll_count > 1:
            # Sum all previous intervals
            total_interval = timedelta(seconds=0)
            for i in range(poll_count):
                total_interval += self._calculate_poll_interval(i)
            next_poll_time = dispatch_time + total_interval

        return current_time >= next_poll_time

    async def _check_task_status_changed(self, task_id: str, group_id: str) -> bool:
        """Check if task status has changed (no longer processing).

        Args:
            task_id: Task ID
            group_id: Group ID

        Returns:
            True if task status changed, False otherwise

        """
        task_status = await self._storage.get_task_status(task_id, group_id)
        match task_status:
            case TaskStatus.PROCESSING | TaskStatus.PENDING | None:
                return False
            case _:
                logger.debug(
                    f"Task {task_id} status changed to {task_status}, removing from pending results",
                    extra={"task_id": task_id, "group_id": group_id, "status": task_status.value},
                )
                return True

    async def _handle_polling_timeout(
        self, task: FairTask, task_id: str, group_id: str, dispatch_time: datetime, current_time: datetime
    ) -> bool:
        """Handle polling timeout for a task.

        Args:
            task: Task object
            task_id: Task ID
            group_id: Group ID
            dispatch_time: When task was dispatched
            current_time: Current time

        Returns:
            True if timeout occurred, False otherwise

        """
        elapsed = current_time - dispatch_time
        if elapsed > self._result_poll_timeout:
            logger.warning(
                f"Result polling timeout for task {task_id} after {elapsed.total_seconds():.1f}s. "
                f"Marking as completed. Consider: 1) Increasing result_poll_timeout, "
                f"2) Checking if task actually completes, 3) Using result callbacks if available.",
                extra={
                    "task_id": task_id,
                    "group_id": group_id,
                    "elapsed_seconds": elapsed.total_seconds(),
                },
            )
            await self._storage.mark_task_completed(task.task_id, group_id)
            task.status = TaskStatus.COMPLETED
            return True
        return False

    async def _process_polled_result(
        self, task: FairTask, task_id: str, group_id: str, result: Any, dispatch_time: datetime, current_time: datetime
    ) -> None:
        """Process a result that was found during polling.

        Args:
            task: Task object
            task_id: Task ID
            group_id: Group ID
            result: Task result
            dispatch_time: When task was dispatched
            current_time: Current time

        """
        if self._result_handler:
            try:
                await self._result_handler.handle_result(task, group_id, result)
                logger.debug(
                    f"Result found and handled for task {task_id}",
                    extra={"task_id": task_id, "group_id": group_id},
                )
            except Exception as result_error:
                logger.warning(
                    f"Result handler failed for task {task.task_id}: {result_error}",
                    extra={"task_id": task.task_id, "group_id": group_id},
                    exc_info=True,
                )

        # Mark task as completed
        await self._storage.mark_task_completed(task.task_id, group_id)
        task.status = TaskStatus.COMPLETED

        # Update group state
        processing_time = current_time - dispatch_time
        processing_seconds = processing_time.total_seconds()
        await self._update_group_state(group_id, processing_time)

        # Fire completed event
        await self._hook_manager.fire_task_completed(task, group_id, processing_seconds)

    async def _poll_single_task_result(  # noqa: C901, PLR0912
        self,
        task_id: str,
        task: FairTask,
        group_id: str,
        dispatch_time: datetime,
        poll_count: int,
        current_time: datetime,
    ) -> tuple[bool, bool]:  # (should_remove, should_update)
        """Poll result for a single task.

        Args:
            task_id: Task ID
            task: Task object
            group_id: Group ID
            dispatch_time: When task was dispatched
            poll_count: Current poll count
            current_time: Current time

        Returns:
            Tuple of (should_remove, should_update) where:
            - should_remove: True if task should be removed from pending results
            - should_update: True if poll count should be incremented

        """
        # Check if task status changed
        if await self._check_task_status_changed(task_id, group_id):
            return (True, False)

        # Check if polling timeout exceeded
        if await self._handle_polling_timeout(task, task_id, group_id, dispatch_time, current_time):
            return (True, False)

        # Check if we should poll this task (exponential backoff)
        if not await self._should_poll_task(dispatch_time, poll_count):
            return (False, False)  # Skip this task for now

        # Try to get result from TaskIQ result backend
        result = await self._task_adapter.get_result(task_id)
        if result is not None:
            # Check if result indicates a failure
            # TaskIQ result objects may have is_err, is_error, or error properties
            is_failed = False
            error: Exception | None = None

            # Check for TaskIQ result error indicators
            is_err = False
            with contextlib.suppress(AttributeError):
                is_err = result.is_err  # type: ignore[attr-defined]
            if is_err:
                is_failed = True
                # Extract error from result
                try:
                    error_value = result.return_value  # type: ignore[attr-defined]
                    error = error_value if isinstance(error_value, Exception) else RuntimeError(str(error_value))
                except AttributeError:
                    try:
                        error_value = result.error  # type: ignore[attr-defined]
                        error = error_value if isinstance(error_value, Exception) else RuntimeError(str(error_value))
                    except AttributeError:
                        error = RuntimeError("Task failed")
            else:
                is_error = False
                with contextlib.suppress(AttributeError):
                    is_error = result.is_error  # type: ignore[attr-defined]
                if is_error:
                    is_failed = True
                    try:
                        error_value = result.error  # type: ignore[attr-defined]
                        error = error_value if isinstance(error_value, Exception) else RuntimeError(str(error_value))
                    except AttributeError:
                        error = RuntimeError("Task failed")

            if is_failed and error:
                # Task failed - handle failure
                logger.warning(
                    f"Task {task_id} failed: {error}",
                    extra={"task_id": task_id, "group_id": group_id},
                )
                await self._handle_task_failure(task, group_id, error)
                return (True, False)

            # Result found and successful - extract return value for processing
            result_value = result
            try:
                result_value = result.return_value  # type: ignore[attr-defined]
            except AttributeError:
                with contextlib.suppress(AttributeError):
                    result_value = result.result  # type: ignore[attr-defined]

            # Process successful result
            if self._result_handler:
                await self._process_polled_result(task, task_id, group_id, result_value, dispatch_time, current_time)
                return (True, False)
        # No result yet, increment poll count for exponential backoff
        return (False, True)

    async def _poll_pending_results(self) -> None:
        """Poll TaskIQ result backend for pending tasks with exponential backoff.

        Checks all pending tasks and processes results when available.
        Uses exponential backoff to reduce polling frequency for tasks that
        haven't returned results yet.

        """
        async with self._pending_results_lock:
            if not self._pending_results:
                return

            # Create a snapshot of pending results to iterate over
            pending_snapshot = dict(self._pending_results)

        current_time = datetime.now(UTC)
        tasks_to_remove: list[str] = []
        tasks_to_update: dict[str, int] = {}  # task_id -> new_poll_count

        for task_id, (task, group_id, dispatch_time, poll_count) in pending_snapshot.items():
            try:
                should_remove, should_update = await self._poll_single_task_result(
                    task_id, task, group_id, dispatch_time, poll_count, current_time
                )
                if should_remove:
                    tasks_to_remove.append(task_id)
                elif should_update:
                    tasks_to_update[task_id] = poll_count + 1
            except Exception as e:
                logger.warning(
                    f"Error polling result for task {task_id}: {e}",
                    extra={"task_id": task_id, "group_id": group_id},
                    exc_info=True,
                )
                # Continue with other tasks even if one fails

        # Remove processed tasks (with lock protection)
        async with self._pending_results_lock:
            for task_id in tasks_to_remove:
                self._pending_results.pop(task_id, None)

            # Update poll counts for tasks that were polled but didn't have results
            for task_id, new_poll_count in tasks_to_update.items():
                try:
                    task, group_id, dispatch_time, _ = self._pending_results[task_id]
                    self._pending_results[task_id] = (task, group_id, dispatch_time, new_poll_count)
                except KeyError:
                    # Task was removed by another coroutine, skip
                    continue
