"""Main fair task queue implementation."""

import asyncio
import contextlib
import logging
from collections.abc import AsyncIterator, Callable
from contextlib import AbstractAsyncContextManager
from datetime import UTC, datetime, timedelta
from typing import Any, NoReturn, Self

from haolib.background.fair.abstract import (
    AbstractFairQueueStorage,
    AbstractTaskAdapter,
    ComponentHealthStatus,
    ComponentHealthStatusMap,
    EnqueueOptions,
    FairnessAlgorithm,
    FairTask,
    GroupKeyExtractor,
    GroupMetrics,
    GroupMetricsMap,
    GroupQueueLengths,
    GroupQueueState,
    GroupStates,
    HealthDetails,
    OverallHealthStatus,
    OverallHealthStatusDetails,
    QueueMetrics,
    QueueStats,
    RateLimit,
    RetryStrategy,
    SchedulerMetrics,
    TaskInfo,
    TaskInvocation,
    TaskRegistration,
    TaskResultUpdate,
    TaskStatus,
)
from haolib.background.fair.algorithms import WeightedFairQueuing
from haolib.background.fair.customization import (
    DefaultDependencyResolver,
    DefaultGroupWeightCalculator,
    DefaultTaskPrioritizer,
    DependencyResolver,
    GroupWeightCalculator,
    TaskPrioritizer,
)
from haolib.background.fair.distributed_lock import DistributedLock, DistributedLockContext
from haolib.background.fair.exceptions import (
    FairQueueIdempotencyConflictError,
    FairQueueInvalidTaskError,
    FairQueuePartialBatchError,
    FairQueueQueueFullError,
    FairQueueStorageError,
)
from haolib.background.fair.extensions import HookManager, LoggingHook, MetricsHook, MiddlewareManager
from haolib.background.fair.results import (
    NoOpResultHandler,
    StorageResultHandler,
    TaskResultHandler,
    stream_task_result,
)
from haolib.background.fair.scheduler import FairQueueScheduler

# Constants
DEFAULT_GROUP_WEIGHT = 1.0


logger = logging.getLogger(__name__)

# Display length constant (not configurable, used for error messages)
MAX_GROUP_ID_DISPLAY_LENGTH = 50

# Result polling constants
MAX_RESULT_POLL_COUNT = 3  # Maximum number of polls before giving up on result retrieval


class FairTaskQueue(AbstractAsyncContextManager["FairTaskQueue"]):
    """Fair task queue manager.

    Can be used as an async context manager:

    Example:
        ```python
        async with queue:
            # Scheduler is running
            await queue.enqueue(task)
            await asyncio.sleep(10)
        # Scheduler is stopped gracefully
        ```

    """

    def __init__(
        self,
        storage: AbstractFairQueueStorage,
        task_adapter: AbstractTaskAdapter,
        group_key_extractor: GroupKeyExtractor[TaskInvocation],
        fairness_algorithm: FairnessAlgorithm | None = None,
        default_rate_limit: RateLimit | None = None,
        max_queue_size_per_group: int | None = None,
        max_global_queue_size: int | None = None,
        default_max_retries: int = 3,
        scheduler_poll_interval: timedelta = timedelta(seconds=0.1),
        scheduler_error_backoff_interval: timedelta = timedelta(seconds=1.0),
        scheduler_max_concurrent_tasks: int = 10,
        retry_strategy: RetryStrategy | None = None,
        hook_manager: HookManager | None = None,
        middleware_manager: MiddlewareManager | None = None,
        enable_default_hooks: bool = True,
        distributed_lock: DistributedLock | None = None,
        recovery_lock_timeout: timedelta = timedelta(seconds=30),
        result_handler: TaskResultHandler | None = None,
        enable_auto_recovery: bool = False,
        auto_recovery_interval: timedelta = timedelta(minutes=5),
        auto_recovery_max_processing_time: timedelta = timedelta(minutes=5),
        max_group_id_length: int = 255,
        min_priority: int = -1000,
        max_priority: int = 1000,
        max_dependency_depth: int = 100,
        max_dependency_task_id_length: int = 255,
        max_pending_results: int = 10_000,
        max_result_events: int = 10_000,
        dependency_resolver: DependencyResolver | None = None,
        task_prioritizer: TaskPrioritizer | None = None,
        group_weight_calculator: GroupWeightCalculator | None = None,
    ) -> None:
        """Initialize fair task queue.

        Args:
            storage: Storage backend for tasks
            task_adapter: Task adapter for task execution
            group_key_extractor: Function/class to extract group ID from task
            fairness_algorithm: Algorithm for fair distribution
            default_rate_limit: Default rate limit for groups
            max_queue_size_per_group: Maximum queue size per group (None = unlimited)
            max_global_queue_size: Maximum global queue size (None = unlimited)
            default_max_retries: Default maximum retry attempts
            scheduler_poll_interval: Scheduler poll interval (seconds)
            scheduler_error_backoff_interval: Scheduler error backoff interval (seconds)
            scheduler_max_concurrent_tasks: Maximum concurrent tasks in scheduler
            retry_strategy: Retry strategy to use (defaults to ExponentialBackoffRetryStrategy)
            hook_manager: Optional hook manager for event hooks
            middleware_manager: Optional middleware manager for cross-cutting concerns
            enable_default_hooks: If True, automatically register LoggingHook and MetricsHook
            distributed_lock: Optional distributed lock for coordination (required for multi-instance)
            recovery_lock_timeout: Timeout for acquiring recovery lock
            result_handler: Optional result handler for storing task results (defaults to NoOpResultHandler)
            enable_auto_recovery: If True, automatically run recovery periodically
            auto_recovery_interval: Interval between automatic recovery runs
            auto_recovery_max_processing_time: Maximum processing time before task is considered stuck
            max_group_id_length: Maximum length for group IDs (default: 255)
            min_priority: Minimum allowed task priority (default: -1000)
            max_priority: Maximum allowed task priority (default: 1000)
            max_dependency_depth: Maximum depth for task dependency chains (default: 100)
            max_dependency_task_id_length: Maximum length for dependency task IDs (default: 255)
            max_pending_results: Maximum number of pending results to track (default: 10_000)
            max_result_events: Maximum number of result events to track (default: 10_000)
            dependency_resolver: Custom dependency resolver (defaults to DefaultDependencyResolver)
            task_prioritizer: Custom task prioritizer (defaults to DefaultTaskPrioritizer)
            group_weight_calculator: Custom group weight calculator (defaults to DefaultGroupWeightCalculator)

        """
        self._storage = storage
        self._task_adapter = task_adapter
        self._distributed_lock = distributed_lock
        self._recovery_lock_timeout = recovery_lock_timeout
        self._result_handler = result_handler if result_handler is not None else NoOpResultHandler()
        self._enable_auto_recovery = enable_auto_recovery
        self._auto_recovery_interval = auto_recovery_interval
        self._auto_recovery_max_processing_time = auto_recovery_max_processing_time
        self._recovery_task: asyncio.Task | None = None
        self._paused = False
        self._fairness_algorithm = fairness_algorithm if fairness_algorithm is not None else WeightedFairQueuing()
        self._default_rate_limit = default_rate_limit
        self._max_queue_size_per_group = max_queue_size_per_group
        self._max_global_queue_size = max_global_queue_size
        self._default_max_retries = default_max_retries
        # Validation constants (configurable)
        self._max_group_id_length = max_group_id_length
        self._min_priority = min_priority
        self._max_priority = max_priority
        self._max_dependency_depth = max_dependency_depth
        self._max_dependency_task_id_length = max_dependency_task_id_length
        self._hook_manager = hook_manager or HookManager()
        self._middleware_manager = middleware_manager or MiddlewareManager()

        # Customization points
        if dependency_resolver is not None:
            self._dependency_resolver = dependency_resolver
        else:
            default_resolver = DefaultDependencyResolver(
                max_depth=max_dependency_depth,
                max_task_id_length=max_dependency_task_id_length,
            )
            self._dependency_resolver = default_resolver  # type: ignore[assignment]
        self._task_prioritizer: TaskPrioritizer = (
            task_prioritizer if task_prioritizer is not None else DefaultTaskPrioritizer()
        )
        self._group_weight_calculator: GroupWeightCalculator = (
            group_weight_calculator if group_weight_calculator is not None else DefaultGroupWeightCalculator()
        )

        # Register default hooks if enabled
        if enable_default_hooks:
            self._hook_manager.register(LoggingHook(logger=logger))
            try:
                self._hook_manager.register(MetricsHook())
            except Exception:
                # Metrics hook may fail if OpenTelemetry is not configured
                logger.debug("MetricsHook not available, skipping", exc_info=True)

        # Store group key extractor
        self._group_key_extractor: GroupKeyExtractor[TaskInvocation] = group_key_extractor

        # Store configurable limits for scheduler
        self._max_pending_results = max_pending_results
        self._max_result_events = max_result_events

        # Create scheduler
        self._scheduler = FairQueueScheduler(
            storage=storage,
            task_adapter=task_adapter,
            fairness_algorithm=self._fairness_algorithm,
            poll_interval=scheduler_poll_interval,
            error_backoff_interval=scheduler_error_backoff_interval,
            max_concurrent_tasks=scheduler_max_concurrent_tasks,
            default_max_retries=default_max_retries,
            retry_strategy=retry_strategy,
            hook_manager=self._hook_manager,
            middleware_manager=self._middleware_manager,
            result_handler=self._result_handler,
            max_pending_results=max_pending_results,
            max_result_events=max_result_events,
        )

    @classmethod
    def create(
        cls,
        storage: AbstractFairQueueStorage,
        task_adapter: AbstractTaskAdapter,
        group_key_extractor: GroupKeyExtractor[TaskInvocation],
        *,
        fairness_algorithm: FairnessAlgorithm | None = None,
        default_rate_limit: RateLimit | None = None,
        max_queue_size_per_group: int | None = None,
        max_global_queue_size: int | None = None,
        default_max_retries: int = 3,
        enable_auto_result_storage: bool = False,
        result_ttl_seconds: int | None = None,
        enable_auto_recovery: bool = False,
        auto_recovery_interval: timedelta = timedelta(minutes=5),
        auto_recovery_max_processing_time: timedelta = timedelta(minutes=5),
        distributed_lock: DistributedLock | None = None,
        **kwargs: Any,
    ) -> FairTaskQueue:
        """Create FairTaskQueue with simplified configuration (recommended).

        This is the primary and recommended way to create a FairTaskQueue.
        It provides sensible defaults and only exposes commonly changed options.

        Args:
            storage: Storage backend for tasks
            task_adapter: Task adapter for task execution
            group_key_extractor: Function/class to extract group ID from task
            fairness_algorithm: Algorithm for fair distribution (defaults to WeightedFairQueuing)
            default_rate_limit: Default rate limit for groups
            max_queue_size_per_group: Maximum queue size per group (None = unlimited)
            max_global_queue_size: Maximum global queue size (None = unlimited)
            default_max_retries: Default maximum retry attempts (default: 3)
            enable_auto_result_storage: If True, automatically store task results
            result_ttl_seconds: TTL for stored results when auto storage is enabled
            enable_auto_recovery: If True, automatically recover stuck tasks periodically
            auto_recovery_interval: Interval between automatic recovery runs
            auto_recovery_max_processing_time: Maximum processing time before task is considered stuck
            distributed_lock: Optional distributed lock for multi-instance coordination
            **kwargs: Additional advanced options (scheduler_poll_interval, etc.)

        Returns:
            Configured FairTaskQueue instance

        Raises:
            ValueError: If required parameters are missing or invalid

        Example:
            ```python
            # Simple usage with defaults
            queue = FairTaskQueue.create(
                storage=redis_storage,
                task_adapter=taskiq_adapter,
                group_key_extractor=lambda task: task.kwargs["user_id"],
            )

            # With common options
            queue = FairTaskQueue.create(
                storage=redis_storage,
                task_adapter=taskiq_adapter,
                group_key_extractor=user_id_extractor,
                default_rate_limit=RateLimit(max_tasks=10, per_time=timedelta(seconds=1)),
                enable_auto_result_storage=True,
                enable_auto_recovery=True,
            )
            ```

        """
        # Handle auto result storage
        result_handler: TaskResultHandler | None = None
        if enable_auto_result_storage:
            result_handler = StorageResultHandler(
                storage=storage,
                default_ttl_seconds=result_ttl_seconds,
            )

        return cls(
            storage=storage,
            task_adapter=task_adapter,
            group_key_extractor=group_key_extractor,
            fairness_algorithm=fairness_algorithm,
            default_rate_limit=default_rate_limit,
            max_queue_size_per_group=max_queue_size_per_group,
            max_global_queue_size=max_global_queue_size,
            default_max_retries=default_max_retries,
            result_handler=result_handler,
            enable_auto_recovery=enable_auto_recovery,
            auto_recovery_interval=auto_recovery_interval,
            auto_recovery_max_processing_time=auto_recovery_max_processing_time,
            distributed_lock=distributed_lock,
            **kwargs,
        )

    @classmethod
    def from_config(
        cls,
        config: Any,  # FairQueueConfig from haolib.configs.fair
        *,
        storage: AbstractFairQueueStorage,
        task_adapter: AbstractTaskAdapter,
        group_key_extractor: GroupKeyExtractor[TaskInvocation],
        fairness_algorithm: FairnessAlgorithm | None = None,
        default_rate_limit: RateLimit | None = None,
        retry_strategy: RetryStrategy | None = None,
        hook_manager: HookManager | None = None,
        middleware_manager: MiddlewareManager | None = None,
        result_handler: TaskResultHandler | None = None,
        distributed_lock: DistributedLock | None = None,
    ) -> FairTaskQueue:
        """Create FairTaskQueue from configuration object.

        Args:
            config: FairQueueConfig instance from haolib.configs.fair
            storage: Storage backend (required runtime dependency)
            task_adapter: Task adapter (required runtime dependency)
            group_key_extractor: Group key extractor (required runtime dependency)
            fairness_algorithm: Optional fairness algorithm (defaults to WeightedFairQueuing)
            default_rate_limit: Optional default rate limit for groups
            retry_strategy: Optional retry strategy (defaults to ExponentialBackoffRetryStrategy)
            hook_manager: Optional hook manager (defaults to new HookManager)
            middleware_manager: Optional middleware manager (defaults to new MiddlewareManager)
            result_handler: Optional result handler (auto-created if enable_auto_result_storage=True)
            distributed_lock: Optional distributed lock for multi-instance coordination

        Returns:
            Configured FairTaskQueue instance

        Raises:
            ValueError: If configuration is invalid

        Example:
            ```python
            from haolib.configs.fair import FairQueueConfig

            config = FairQueueConfig(
                enable_auto_result_storage=True,
                enable_auto_recovery=True,
            )
            queue = FairTaskQueue.from_config(
                config=config,
                storage=redis_storage,
                task_adapter=taskiq_adapter,
                group_key_extractor=user_id_extractor,
            )
            ```

        """
        # Handle auto result storage
        final_result_handler = result_handler
        if config.enable_auto_result_storage and final_result_handler is None:
            final_result_handler = StorageResultHandler(
                storage=storage,
                default_ttl_seconds=config.result_ttl_seconds,
            )

        return cls(
            storage=storage,
            task_adapter=task_adapter,
            group_key_extractor=group_key_extractor,
            fairness_algorithm=fairness_algorithm,
            default_rate_limit=default_rate_limit,
            max_queue_size_per_group=config.max_queue_size_per_group,
            max_global_queue_size=config.max_global_queue_size,
            default_max_retries=config.default_max_retries,
            scheduler_poll_interval=config.scheduler_poll_interval,
            scheduler_error_backoff_interval=config.scheduler_error_backoff_interval,
            scheduler_max_concurrent_tasks=config.scheduler_max_concurrent_tasks,
            retry_strategy=retry_strategy,
            hook_manager=hook_manager,
            max_group_id_length=config.max_group_id_length,
            min_priority=config.min_priority,
            max_priority=config.max_priority,
            max_dependency_depth=config.max_dependency_depth,
            max_dependency_task_id_length=config.max_dependency_task_id_length,
            middleware_manager=middleware_manager,
            enable_default_hooks=config.enable_default_hooks,
            distributed_lock=distributed_lock,
            recovery_lock_timeout=config.recovery_lock_timeout,
            result_handler=final_result_handler,
            enable_auto_recovery=config.enable_auto_recovery,
            auto_recovery_interval=config.auto_recovery_interval,
            auto_recovery_max_processing_time=config.auto_recovery_max_processing_time,
        )

    async def _normalize_to_invocation(
        self,
        task: Callable[..., Any] | TaskRegistration | TaskInvocation,
        *args: Any,
        **kwargs: Any,
    ) -> TaskInvocation:
        """Normalize task input to TaskInvocation.

        Handles:
        - Callable: Register and create invocation
        - TaskRegistration: Create invocation from registration
        - TaskInvocation: Return as-is

        Args:
            task: Task function, TaskRegistration, or TaskInvocation
            *args: Task function arguments (if task is callable or TaskRegistration)
            **kwargs: Task function keyword arguments

        Returns:
            TaskInvocation

        Raises:
            FairQueueInvalidTaskError: If task cannot be normalized

        """
        # Handle TaskInvocation (already complete)
        if isinstance(task, TaskInvocation):
            return task

        # Handle TaskRegistration
        if isinstance(task, TaskRegistration):
            return self._task_adapter.create_invocation(task, *args, **kwargs)

        # Handle TaskiqMessage (from task.kiq() - declarative pattern)
        # Convert to TaskInvocation using adapter
        try:
            from taskiq.message import TaskiqMessage  # noqa: PLC0415

            if isinstance(task, TaskiqMessage):
                # Check if adapter is TaskIQAdapter (supports message_to_invocation)
                from haolib.background.fair.adapters import TaskIQAdapter  # noqa: PLC0415

                if not isinstance(self._task_adapter, TaskIQAdapter):
                    raise FairQueueInvalidTaskError(
                        detail=(f"TaskiqMessage requires TaskIQAdapter, but got {type(self._task_adapter).__name__}"),
                        validation_error="unsupported_adapter",
                        task_id=None,
                        group_id=None,
                    )

                # Use adapter to convert message to invocation
                # Merge any additional args/kwargs passed to enqueue
                if args or kwargs:
                    # Create a new message with merged args/kwargs
                    merged_kwargs = {**(task.kwargs or {}), **kwargs}
                    merged_args = list(task.args or []) + list(args)
                    # Create a new message with merged data
                    from taskiq.message import TaskiqMessage as TaskiqMessageType  # noqa: PLC0415

                    merged_message = TaskiqMessageType(
                        task_id=task.task_id,
                        task_name=task.task_name,
                        labels=task.labels,
                        args=merged_args,
                        kwargs=merged_kwargs,
                    )
                    return self._task_adapter.message_to_invocation(merged_message)  # type: ignore[attr-defined]
                return self._task_adapter.message_to_invocation(task)  # type: ignore[attr-defined]
        except ImportError:
            pass  # TaskIQ not available, skip this check

        # Handle callable (register + create invocation)
        if callable(task):
            try:
                registration = await self._task_adapter.register_task(task)
                return self._task_adapter.create_invocation(registration, *args, **kwargs)
            except Exception as e:
                # Extract task name safely for error message
                task_name_str = getattr(task, "__name__", "unknown")
                raise FairQueueInvalidTaskError(
                    detail=f"Failed to register task '{task_name_str}': {e}",
                    validation_error="task_registration_failed",
                    task_id=None,
                    group_id=None,
                ) from e

        # Invalid type
        raise FairQueueInvalidTaskError(
            detail=(
                f"Invalid task type: {type(task)}. "
                f"Expected Callable, TaskRegistration, TaskInvocation, or TaskiqMessage."
            ),
            validation_error="invalid_task_type",
        )

    def _validate_invocation(
        self,
        invocation: TaskInvocation,
        options: EnqueueOptions,
    ) -> None:
        """Validate task invocation before enqueueing.

        Args:
            invocation: Task invocation to validate
            options: Enqueue options to update with group_id

        Raises:
            FairQueueInvalidTaskError: If validation fails

        """
        # Validate task_name
        if not invocation.task_name or not isinstance(invocation.task_name, str):
            raise FairQueueInvalidTaskError(
                detail="Task name must be a non-empty string",
                validation_error="invalid_task_name",
                task_id=invocation.task_id,
            )

        # Validate task_id
        if not invocation.task_id or not isinstance(invocation.task_id, str):
            raise FairQueueInvalidTaskError(
                detail="Task ID must be a non-empty string",
                validation_error="invalid_task_id",
            )

        # Extract group_id if not provided
        if options.group_id is None:
            try:
                options.group_id = self._group_key_extractor(invocation)
            except Exception as e:
                raise FairQueueInvalidTaskError(
                    detail=(
                        f"Failed to extract group_id from task '{invocation.task_name}': {e}. "
                        f"Provide group_id explicitly in EnqueueOptions, "
                        f"or ensure task kwargs/args contain a group identifier."
                    ),
                    validation_error="group_id_extraction_failed",
                    task_id=invocation.task_id,
                ) from e

        # Validate group_id format and content
        if not options.group_id or not isinstance(options.group_id, str):
            raise FairQueueInvalidTaskError(
                detail="Group ID must be a non-empty string",
                group_id=options.group_id,
                validation_error="group_id is empty or not a string",
                task_id=invocation.task_id,
            )

        # Validate group_id length
        if len(options.group_id) > self._max_group_id_length:
            raise FairQueueInvalidTaskError(
                detail=(
                    f"Group ID is too long (max {self._max_group_id_length} characters, got {len(options.group_id)})"
                ),
                group_id=(
                    options.group_id[:MAX_GROUP_ID_DISPLAY_LENGTH] + "..."
                    if len(options.group_id) > MAX_GROUP_ID_DISPLAY_LENGTH
                    else options.group_id
                ),
                validation_error="group_id too long",
                task_id=invocation.task_id,
            )

        # Validate group_id characters
        valid_chars = ("_", "-", ":", ".")
        if not all(c.isalnum() or c in valid_chars for c in options.group_id):
            raise FairQueueInvalidTaskError(
                detail=(
                    "Group ID contains invalid characters. "
                    "Only alphanumeric, underscore, hyphen, colon, and dot are allowed"
                ),
                group_id=options.group_id,
                validation_error="group_id invalid characters",
                task_id=invocation.task_id,
            )

        # Validate priority
        if options.priority < self._min_priority or options.priority > self._max_priority:
            raise FairQueueInvalidTaskError(
                detail=(
                    f"Priority {options.priority} is out of allowed range [{self._min_priority}, {self._max_priority}]"
                ),
                validation_error="priority out of range",
                task_id=invocation.task_id,
                group_id=options.group_id,
            )

    async def _validate_dependencies(
        self, depends_on: list[str], group_id: str | None, current_task_id: str | None = None
    ) -> None:
        """Validate task dependencies at enqueue time with proper DAG validation.

        This method performs comprehensive dependency validation including:
        - Duplicate detection
        - Format validation
        - Circular dependency detection (direct and indirect)
        - Depth limit to prevent extremely deep dependency chains

        Args:
            depends_on: List of task IDs this task depends on
            group_id: Group ID of the task
            current_task_id: Optional current task ID for circular dependency detection

        Raises:
            FairQueueInvalidTaskError: If dependencies are invalid (circular, non-existent, etc.)

        """
        if not depends_on:
            return

        # Check for duplicate dependencies
        if len(depends_on) != len(set(depends_on)):
            raise FairQueueInvalidTaskError(
                detail="Duplicate task IDs in depends_on list",
                group_id=group_id,
                validation_error="duplicate_dependencies",
            )

        # Limit dependency chain depth to prevent abuse
        if len(depends_on) > self._max_dependency_depth:
            raise FairQueueInvalidTaskError(
                detail=(f"Too many dependencies (max {self._max_dependency_depth}, got {len(depends_on)})"),
                group_id=group_id,
                validation_error="too_many_dependencies",
            )

        # Validate that dependencies are valid task IDs
        for dep_task_id in depends_on:
            if not dep_task_id or not isinstance(dep_task_id, str):
                raise FairQueueInvalidTaskError(
                    detail=f"Invalid dependency task ID: {dep_task_id}",
                    group_id=group_id,
                    validation_error="invalid_dependency_id",
                )

            # Validate task ID format (should be UUID or similar)
            if len(dep_task_id) > self._max_dependency_task_id_length:
                truncated_id = dep_task_id[:MAX_GROUP_ID_DISPLAY_LENGTH] + "..."
                raise FairQueueInvalidTaskError(
                    detail=f"Dependency task ID is too long: {truncated_id}",
                    group_id=group_id,
                    validation_error="dependency_id_too_long",
                )

            # Check for direct circular dependency
            if current_task_id and dep_task_id == current_task_id:
                raise FairQueueInvalidTaskError(
                    detail=f"Circular dependency detected: task {current_task_id} depends on itself",
                    group_id=group_id,
                    validation_error="circular_dependency",
                )

        # Perform DAG validation for circular dependencies
        if current_task_id:
            await self._validate_dag_no_cycles(current_task_id, depends_on, group_id)

    def _raise_task_failed_error(self, task_id: str, error: Exception) -> NoReturn:
        """Raise error for task failure.

        Args:
            task_id: Task ID that failed
            error: Original error that caused the failure

        Raises:
            RuntimeError: Always raises with task failure message

        """
        msg = f"Task {task_id} failed: {error}"
        raise RuntimeError(msg) from error

    def _raise_circular_dependency_error(
        self, current_task_id: str, dep_task_id: str | None, group_id: str | None, cycle_path: list[str] | None = None
    ) -> NoReturn:
        """Raise error for circular dependency.

        Args:
            current_task_id: Current task ID
            dep_task_id: Dependency task ID (if None, uses cycle_path)
            group_id: Group ID
            cycle_path: Optional cycle path for detailed error message

        Raises:
            FairQueueInvalidTaskError: Always raises with circular dependency message

        """
        if cycle_path:
            detail = f"Circular dependency detected: {' -> '.join(cycle_path)} -> {current_task_id}"
        elif dep_task_id:
            detail = (
                f"Circular dependency detected: task {current_task_id} depends on {dep_task_id}, "
                f"which depends on {current_task_id}"
            )
        else:
            detail = f"Circular dependency detected: task {current_task_id} depends on itself"

        raise FairQueueInvalidTaskError(
            detail=detail,
            group_id=group_id,
            validation_error="circular_dependency",
        )

    async def _validate_dag_no_cycles(
        self, current_task_id: str, depends_on: list[str], group_id: str | None, visited: set[str] | None = None
    ) -> None:
        """Validate that adding this dependency doesn't create a cycle in the DAG.

        Uses DFS to detect cycles in the dependency graph.

        Args:
            current_task_id: Task ID being enqueued
            depends_on: List of task IDs this task depends on
            group_id: Group ID of the task
            visited: Set of visited task IDs (for recursion)

        Raises:
            FairQueueInvalidTaskError: If a cycle is detected

        """
        if visited is None:
            visited = set()

        # Check if current task would create a cycle
        if current_task_id in visited:
            cycle_path = [*visited, current_task_id]
            self._raise_circular_dependency_error(current_task_id, None, group_id, cycle_path)

        visited.add(current_task_id)

        # Recursively check all dependencies
        for dep_task_id in depends_on:
            try:
                dep_task = await self._storage.get_task(dep_task_id, "")
                if dep_task and dep_task.depends_on:
                    # Check if this dependency would create a cycle back to current_task_id
                    if current_task_id in dep_task.depends_on:
                        self._raise_circular_dependency_error(current_task_id, dep_task_id, group_id)
                    # Recursively validate the dependency's dependencies
                    await self._validate_dag_no_cycles(dep_task_id, dep_task.depends_on, group_id, visited.copy())
            except FairQueueInvalidTaskError:
                # Re-raise validation errors
                raise
            except Exception as e:
                # If we can't check (task might not exist yet), continue
                # This is acceptable as dependencies might be enqueued later
                logger.debug(
                    f"Could not validate dependency {dep_task_id} for task {current_task_id}: {e}",
                    exc_info=True,
                )

    async def _apply_rate_limit(self, group_id: str, rate_limit: RateLimit | None) -> None:
        """Apply rate limit to group if provided.

        Args:
            group_id: Group ID
            rate_limit: Optional rate limit to apply

        """
        if not rate_limit and not self._default_rate_limit:
            return

        # Get or create group state
        state = await self._storage.get_group_state(group_id)
        if not state:
            state = GroupQueueState(
                group_id=group_id,
                queue_length=0,
                virtual_finish_time=0.0,
                weight=DEFAULT_GROUP_WEIGHT,
            )

        # Apply rate limit
        if rate_limit:
            state.rate_limit = rate_limit
        elif self._default_rate_limit and not state.rate_limit:
            state.rate_limit = self._default_rate_limit

        await self._storage.update_group_state(state)

    async def _handle_idempotency_conflict(
        self,
        error: Exception,
        idempotency_key: str | None,
    ) -> str | None:
        """Handle idempotency conflict by returning existing task ID.

        Args:
            error: Exception that occurred
            idempotency_key: Idempotency key if available

        Returns:
            Existing task ID if found, None otherwise

        """
        # Check if it's an idempotency conflict
        if isinstance(error, FairQueueIdempotencyConflictError) and error.task_id:
            return error.task_id

        # Try to get from storage
        if idempotency_key:
            try:
                existing_task_id = await self._storage.get_task_by_idempotency_key(idempotency_key)
                if existing_task_id:
                    return existing_task_id
            except Exception as e:
                logger.debug(f"Failed to get task by idempotency key: {e}", exc_info=True)

        return None

    async def enqueue(  # noqa: C901, PLR0912
        self,
        task: Callable[..., Any] | TaskRegistration | TaskInvocation,
        *args: Any,
        options: EnqueueOptions | None = None,
        # Convenience kwargs for simple cases (merged into options)
        group_id: str | None = None,
        priority: int = 0,
        rate_limit: RateLimit | None = None,
        scheduled_at: datetime | None = None,
        expires_at: datetime | None = None,
        idempotency_key: str | None = None,
        max_retries: int | None = None,
        depends_on: list[str] | None = None,
        backpressure: bool = False,
        backpressure_max_wait: timedelta | None = None,
        backpressure_retry_interval: timedelta | None = None,
        **kwargs: Any,
    ) -> str:
        """Enqueue a task to the fair queue.

        This method supports multiple input types:
        1. **Callable**: Task function (will be registered and invoked)
        2. **TaskRegistration**: Pre-registered task (will create invocation)
        3. **TaskInvocation**: Pre-created invocation (enqueued directly)

        This method also supports two configuration patterns:
        1. **Options object (recommended)**: Pass an `EnqueueOptions` object for type safety and reusability
        2. **Convenience kwargs**: Pass individual parameters for simple cases

        Args:
            task: Task function, TaskRegistration, or TaskInvocation
            *args: Task function arguments (if task is callable or TaskRegistration)
            options: EnqueueOptions object with task configuration (recommended, takes precedence over kwargs)
            group_id: Group ID for the task (if not provided, will be extracted from task)
            priority: Task priority (higher = more priority, default: 0)
            rate_limit: Optional rate limit for this task's group
            scheduled_at: Optional scheduled execution time (delayed execution)
            expires_at: Optional expiration time
            idempotency_key: Optional idempotency key for deduplication
            max_retries: Optional maximum retry attempts (overrides default)
            depends_on: Optional list of task IDs this task depends on
            backpressure: If True, wait and retry when queue is full instead of raising error immediately
            backpressure_max_wait: Maximum time to wait when backpressure is enabled (default: 30s)
            backpressure_retry_interval: Interval between retry attempts when backpressure is enabled (default: 1s)
            **kwargs: Task function keyword arguments and/or fair queue options

        Returns:
            Task ID in the fair queue

        Raises:
            FairQueueQueueFullError: If queue size limit is reached (unless backpressure=True)
            FairQueueInvalidTaskError: If task validation fails
            FairQueueStorageError: If storage operation fails

        Example:
            ```python
            # Option 1: Enqueue function (adapter handles setup)
            async def process_image(image_url: str, user_id: str) -> dict:
                return {"status": "processed"}

            task_id = await queue.enqueue(
                process_image,
                image_url="https://example.com/image.jpg",
                user_id="user_123",
                options=EnqueueOptions(group_id="user_123", priority=10),
            )

            # Option 2: Pre-register task
            registration = await adapter.register_task(process_image)
            task_id = await queue.enqueue(
                registration,
                image_url="https://example.com/image.jpg",
                user_id="user_123",
                group_id="user_123",
            )

            # Option 3: Pre-create invocation
            invocation = adapter.create_invocation(registration, image_url="...", user_id="user_123")
            task_id = await queue.enqueue(invocation, options=EnqueueOptions(group_id="user_123"))
            ```

        Note:
            - If `options` is provided, convenience kwargs are ignored (options takes precedence)
            - For best type safety and IDE support, use `EnqueueOptions`
            - Group ID will be auto-extracted from task if not provided

        """
        # Convert task to TaskInvocation if needed
        invocation = await self._normalize_to_invocation(task, *args, **kwargs)

        # Create or merge options
        if options is None:
            # Validate depends_on before creating EnqueueOptions (Pydantic will reject None)
            if depends_on:
                for dep in depends_on:
                    if dep is None:
                        raise FairQueueInvalidTaskError(
                            detail="Invalid dependency task ID: None",
                            group_id=group_id,
                            validation_error="invalid_dependency_id",
                        )
            # Create options from convenience kwargs
            options = EnqueueOptions(
                group_id=group_id,
                priority=priority,
                rate_limit=rate_limit,
                scheduled_at=scheduled_at,
                expires_at=expires_at,
                idempotency_key=idempotency_key,
                max_retries=max_retries,
                depends_on=depends_on or [],
                backpressure=backpressure,
                backpressure_max_wait=backpressure_max_wait or timedelta(seconds=30),
                backpressure_retry_interval=backpressure_retry_interval or timedelta(seconds=1.0),
            )
        # Options provided - convenience kwargs are ignored for clarity
        # This prevents confusion about which values are used
        elif any(
            [
                group_id is not None,
                priority != 0,
                rate_limit is not None,
                scheduled_at is not None,
                expires_at is not None,
                idempotency_key is not None,
                max_retries is not None,
                depends_on is not None,
                backpressure,
                backpressure_max_wait is not None,
                backpressure_retry_interval is not None,
            ]
        ):
            logger.warning(
                "EnqueueOptions provided along with convenience kwargs. "
                "Options object takes precedence, convenience kwargs are ignored.",
                extra={"task_type": type(task).__name__},
            )

        # Validate invocation and extract group ID
        self._validate_invocation(invocation, options)

        # At this point, group_id is guaranteed to be non-None after validation
        assert options.group_id is not None, "Group ID must be set after validation"

        # Check queue size limits (with optional backpressure)
        # Note: We check WITHOUT incrementing during backpressure loop
        # The increment happens later when we actually enqueue
        if options.backpressure:
            await self._check_queue_limits_with_backpressure(
                options.group_id, options.backpressure_max_wait, options.backpressure_retry_interval
            )
            # After backpressure check passes, do the actual limit check with increment
            await self._check_queue_limits(options.group_id)
        else:
            await self._check_queue_limits(options.group_id)

        # Use task_id from invocation
        task_id = invocation.task_id

        # Validate dependencies if provided (using custom resolver)
        if options.depends_on:
            # First check for None values (before Pydantic validation)
            for dep in options.depends_on:
                if dep is None:
                    raise FairQueueInvalidTaskError(
                        detail="Invalid dependency task ID: None",
                        group_id=options.group_id,
                        validation_error="invalid_dependency_id",
                    )
            await self._dependency_resolver.validate_dependencies(
                task_id, options.depends_on, options.group_id, self._storage
            )

        # Serialize invocation for storage
        try:
            serialized_invocation = self._task_adapter.serialize_invocation(invocation)
        except Exception as e:
            from haolib.background.fair.exceptions import FairQueueSerializationError  # noqa: PLC0415

            if isinstance(e, FairQueueSerializationError):
                raise
            raise FairQueueSerializationError(
                detail=f"Failed to serialize task invocation: {e}",
                task_id=task_id,
                serialization_strategy=invocation.serialization_strategy.value,
                serialization_error=e,
            ) from e

        # Create fair task
        fair_task = FairTask(
            task_id=task_id,
            group_id=options.group_id,
            task_data=serialized_invocation,  # Store serialized TaskInvocation
            priority=options.priority,
            scheduled_at=options.scheduled_at,
            expires_at=options.expires_at,
            idempotency_key=options.idempotency_key,
            max_retries=options.max_retries if options.max_retries is not None else self._default_max_retries,
            depends_on=options.depends_on,
        )

        # Apply middleware before enqueue
        fair_task = await self._middleware_manager.apply_before_enqueue(fair_task)

        # Apply rate limit if provided
        await self._apply_rate_limit(options.group_id, options.rate_limit)

        # Enqueue task with proper error handling and rollback
        try:
            await self._storage.enqueue_task(fair_task)
            # Fire enqueued event
            await self._hook_manager.fire_task_enqueued(fair_task, options.group_id)
            # Apply middleware after enqueue
            await self._middleware_manager.apply_after_enqueue(fair_task)
        except (FairQueueIdempotencyConflictError, FairQueueStorageError) as e:
            # Try to handle idempotency conflict
            existing_task_id = await self._handle_idempotency_conflict(e, options.idempotency_key)
            if existing_task_id:
                return existing_task_id

            # Rollback global queue size increment if it was made
            # This is critical to prevent queue size drift
            await self._rollback_global_queue_size(task_id, options.group_id, "enqueue failure")

            # If we can't resolve it, raise the error with better context
            if isinstance(e, FairQueueStorageError):
                # Add actionable error message
                error_msg = (
                    f"Storage operation failed: {e.detail}. "
                    f"Possible causes: 1) Storage backend unavailable, 2) Network issues, "
                    f"3) Storage capacity exceeded. Check storage health: queue.health_check()"
                )
                raise FairQueueStorageError(
                    detail=error_msg,
                    task_id=task_id,
                    group_id=options.group_id,
                    operation=e.operation if isinstance(e, FairQueueStorageError) and e.operation else "enqueue_task",
                ) from e
            raise
        except Exception as e:
            # Unexpected error - attempt rollback
            await self._rollback_global_queue_size(task_id, options.group_id, "unexpected enqueue error")

            # Re-raise with context
            error_detail = (
                f"Unexpected error during enqueue: {e}. "
                f"This may indicate a bug or unexpected storage behavior. "
                f"Check storage health and logs for more details."
            )
            raise FairQueueStorageError(
                detail=error_detail,
                task_id=task_id,
                group_id=options.group_id,
                operation="enqueue_task",
            ) from e

        return task_id

    async def enqueue_and_wait(
        self,
        task: Any,
        *,
        group_id: str | None = None,
        priority: int = 0,
        rate_limit: RateLimit | None = None,
        scheduled_at: datetime | None = None,
        expires_at: datetime | None = None,
        idempotency_key: str | None = None,
        max_retries: int | None = None,
        depends_on: list[str] | None = None,
        options: EnqueueOptions | None = None,
        timeout_seconds: float | None = None,
        initial_poll_interval: timedelta = timedelta(seconds=0.5),
        max_poll_interval: timedelta = timedelta(seconds=5.0),
    ) -> Any:
        """Enqueue a task and wait for its result.

        This is a convenience method that combines enqueue() and wait_for_result().

        Args:
            task: TaskIQ task to enqueue
            group_id: Group ID for the task (if not provided, will be extracted)
            priority: Task priority (higher = more priority)
            rate_limit: Optional rate limit for this task's group
            scheduled_at: Optional scheduled execution time (delayed execution)
            expires_at: Optional expiration time
            idempotency_key: Optional idempotency key for deduplication
            max_retries: Optional maximum retry attempts (overrides default)
            depends_on: Optional list of task IDs this task depends on
            options: EnqueueOptions object with task configuration (alternative to individual kwargs)
            timeout_seconds: Maximum time to wait for result in seconds (None = wait indefinitely)
            initial_poll_interval: Initial interval between status checks (default: 0.5s)
            max_poll_interval: Maximum interval between status checks (default: 5.0s)

        Returns:
            Task result when available

        Raises:
            TimeoutError: If timeout is reached before task completes
            RuntimeError: If task fails or is cancelled
            FairQueueQueueFullError: If queue size limit is reached
            FairQueueInvalidTaskError: If task validation fails
            FairQueueStorageError: If storage operation fails

        Example:
            ```python
            # Simple fire-and-wait
            result = await queue.enqueue_and_wait(
                task,
                group_id="user_123",
                timeout_seconds=30.0,
            )

            # With options
            options = EnqueueOptions(group_id="user_123", priority=10)
            result = await queue.enqueue_and_wait(task, options=options, timeout_seconds=60.0)
            ```

        """
        task_id = await self.enqueue(
            task,
            group_id=group_id,
            priority=priority,
            rate_limit=rate_limit,
            scheduled_at=scheduled_at,
            expires_at=expires_at,
            idempotency_key=idempotency_key,
            max_retries=max_retries,
            depends_on=depends_on,
            options=options,
        )

        # Extract group_id from options for wait_for_result
        result_group_id = options.group_id if options is not None else group_id

        return await self.wait_for_result(
            task_id,
            group_id=result_group_id,
            timeout_seconds=timeout_seconds,
            initial_poll_interval=initial_poll_interval,
            max_poll_interval=max_poll_interval,
        )

    def _normalize_batch_tasks(
        self,
        tasks: list[tuple[Any, EnqueueOptions | None]],
    ) -> list[tuple[Any, EnqueueOptions]]:
        """Normalize batch task options.

        Args:
            tasks: List of (task, options) tuples

        Returns:
            List of normalized (task, options) tuples

        """
        normalized: list[tuple[Any, EnqueueOptions]] = []
        for task, task_options in tasks:
            normalized_options = task_options if task_options is not None else EnqueueOptions()
            normalized.append((task, normalized_options))
        return normalized

    def _can_use_atomic_batch(
        self,
        normalized_tasks: list[tuple[Any, EnqueueOptions]],
    ) -> bool:
        """Check if atomic batch enqueue can be used.

        Conditions: same group, no rate limits, no expiration.

        Args:
            normalized_tasks: List of normalized (task, options) tuples

        Returns:
            True if atomic batch can be used, False otherwise

        """
        if not normalized_tasks or len(normalized_tasks) == 1:
            return False

        first_group_id: str | None = None
        for task, options in normalized_tasks:
            group_id = options.group_id
            if group_id is None:
                try:
                    group_id = self._group_key_extractor(task)
                except Exception:
                    return False

            if first_group_id is None:
                first_group_id = group_id
            elif group_id != first_group_id:
                return False

            # Check for rate limits or expiration
            if options.rate_limit or options.expires_at:
                return False

        return True

    async def _prepare_fair_tasks_for_batch(
        self,
        normalized_tasks: list[tuple[Callable[..., Any] | TaskRegistration | TaskInvocation, EnqueueOptions]],
    ) -> list[FairTask]:
        """Prepare FairTask objects for batch enqueue.

        Args:
            normalized_tasks: List of normalized (task, options) tuples

        Returns:
            List of FairTask objects

        Raises:
            FairQueueInvalidTaskError: If task validation fails

        """
        fair_tasks: list[FairTask] = []
        for task, options in normalized_tasks:
            # Normalize to TaskInvocation
            invocation = await self._normalize_to_invocation(task)

            # Validate invocation and extract group ID
            self._validate_invocation(invocation, options)

            # Use task_id from invocation
            task_id = invocation.task_id

            # Serialize invocation
            try:
                serialized_invocation = self._task_adapter.serialize_invocation(invocation)
            except Exception as e:
                from haolib.background.fair.exceptions import FairQueueSerializationError  # noqa: PLC0415

                if isinstance(e, FairQueueSerializationError):
                    raise
                raise FairQueueSerializationError(
                    detail=f"Failed to serialize task invocation: {e}",
                    task_id=task_id,
                    serialization_strategy=invocation.serialization_strategy.value,
                    serialization_error=e,
                ) from e

            # Create fair task
            # Ensure group_id is set (should be after validation)
            if options.group_id is None:
                raise FairQueueInvalidTaskError(
                    detail="Group ID must be set before creating FairTask",
                    task_id=task_id,
                    validation_error="group_id is None",
                )

            fair_task = FairTask(
                task_id=task_id,
                group_id=options.group_id,
                task_data=serialized_invocation,  # Store serialized TaskInvocation
                priority=options.priority,
                scheduled_at=options.scheduled_at,
                expires_at=options.expires_at,
                idempotency_key=options.idempotency_key,
                max_retries=options.max_retries if options.max_retries is not None else self._default_max_retries,
                depends_on=options.depends_on,
            )
            fair_tasks.append(fair_task)

        return fair_tasks

    async def _enqueue_batch_atomic(
        self,
        fair_tasks: list[FairTask],
    ) -> list[str]:
        """Enqueue tasks using atomic batch operation.

        Args:
            fair_tasks: List of FairTask objects to enqueue

        Returns:
            List of task IDs

        Raises:
            FairQueueQueueFullError: If queue size limit is reached
            FairQueuePartialBatchError: If some tasks failed to enqueue

        """
        # Check global queue size limit atomically for the entire batch
        if self._max_global_queue_size is not None:
            allowed, current_size = await self._storage.check_and_increment_global_queue_size_batch(
                self._max_global_queue_size,
                len(fair_tasks),
            )
            if not allowed:
                message = (
                    f"Global queue is full: {current_size}/{self._max_global_queue_size}. "
                    f"Cannot enqueue batch of {len(fair_tasks)} tasks."
                )
                raise FairQueueQueueFullError(
                    detail=message,
                    queue_length=current_size,
                    max_size=self._max_global_queue_size,
                )

        # Enqueue batch atomically
        results = await self._storage.enqueue_tasks_batch(fair_tasks)
        batch_task_ids: list[str] = []
        batch_errors: list[tuple[int, Exception]] = []

        for idx, (result_task_id, success, error) in enumerate(results):
            if success:
                batch_task_ids.append(result_task_id)
                # Fire enqueued event
                fair_task = fair_tasks[idx]
                await self._hook_manager.fire_task_enqueued(fair_task, fair_task.group_id)
            else:
                if error == "idempotency_conflict":
                    # Try to get existing task ID
                    fair_task = fair_tasks[idx]
                    if fair_task.idempotency_key:
                        existing_id = await self._storage.get_task_by_idempotency_key(fair_task.idempotency_key)
                        if existing_id:
                            batch_task_ids.append(existing_id)
                            continue
                batch_errors.append((idx, Exception(error or "Unknown error")))

        if batch_errors:
            message = (
                f"Some tasks failed to enqueue: {len(batch_errors)}/{len(fair_tasks)} failed. "
                f"Errors: {[str(e) for _, e in batch_errors]}"
            )
            raise FairQueuePartialBatchError(
                detail=message,
                failed_count=len(batch_errors),
                total_count=len(fair_tasks),
                errors=batch_errors,
            )

        return batch_task_ids

    async def _enqueue_batch_sequential(
        self,
        normalized_tasks: list[tuple[Any, EnqueueOptions]],
    ) -> list[str]:
        """Enqueue tasks sequentially (fallback when atomic batch cannot be used).

        Args:
            normalized_tasks: List of normalized (task, options) tuples

        Returns:
            List of task IDs

        Raises:
            FairQueuePartialBatchError: If some tasks failed to enqueue

        """
        sequential_task_ids: list[str] = []
        sequential_errors: list[tuple[int, Exception]] = []

        for idx, (task, options) in enumerate(normalized_tasks):
            try:
                task_id = await self.enqueue(task, options=options)
                sequential_task_ids.append(task_id)
            except Exception as e:
                sequential_errors.append((idx, e))

        if sequential_errors:
            message = (
                f"Some tasks failed to enqueue: {len(sequential_errors)}/{len(normalized_tasks)} failed. "
                f"Errors: {[str(e) for _, e in sequential_errors]}"
            )
            raise FairQueuePartialBatchError(
                detail=message,
                failed_count=len(sequential_errors),
                total_count=len(normalized_tasks),
                errors=sequential_errors,
            )

        return sequential_task_ids

    async def enqueue_batch(
        self,
        tasks: list[tuple[Any, EnqueueOptions | None]] | list[Any],
        *,
        group_id: str | None = None,
        options: EnqueueOptions | None = None,
    ) -> list[str]:
        """Enqueue multiple tasks in a batch operation.

        Uses atomic batch enqueue if all tasks are for the same group and have no rate limits.
        Otherwise falls back to sequential enqueue.

        Args:
            tasks: Either:
                - List of (task, options) tuples where options is EnqueueOptions or None (advanced usage)
                - List of task objects (simplified usage, requires group_id or options.group_id)
            group_id: Optional group ID to use for all tasks (simplified usage)
            options: Optional EnqueueOptions to use for all tasks (simplified usage)

        Returns:
            List of task IDs

        Raises:
            FairQueuePartialBatchError: If some tasks failed to enqueue
            FairQueueQueueFullError: If queue size limit is reached
            FairQueueInvalidTaskError: If task validation fails

        Example:
            ```python
            # Simplified usage - all tasks in same group
            task_ids = await queue.enqueue_batch([task1, task2, task3], group_id="user_123")

            # With options
            options = EnqueueOptions(group_id="user_123", priority=10)
            task_ids = await queue.enqueue_batch([task1, task2], options=options)

            # Advanced usage - different options per task
            tasks = [
                (task1, EnqueueOptions(group_id="user_1", priority=10)),
                (task2, EnqueueOptions(group_id="user_1", priority=5)),
            ]
            task_ids = await queue.enqueue_batch(tasks)
            ```

        """
        # Handle simplified usage: list of tasks with common group_id/options
        if tasks and not isinstance(tasks[0], tuple):
            # Convert to tuple format with common options
            common_options = options or EnqueueOptions()
            if group_id is not None:
                common_options.group_id = group_id
            tasks = [(task, common_options) for task in tasks]

        # Normalize options
        normalized_tasks = self._normalize_batch_tasks(tasks)

        # Check if we can use atomic batch enqueue
        if self._can_use_atomic_batch(normalized_tasks):
            # Prepare fair tasks
            fair_tasks = await self._prepare_fair_tasks_for_batch(normalized_tasks)
            # Enqueue atomically
            return await self._enqueue_batch_atomic(fair_tasks)

        # Fall back to sequential enqueue
        return await self._enqueue_batch_sequential(normalized_tasks)

    async def recover_stuck_tasks(
        self,
        max_processing_time: timedelta = timedelta(minutes=5),
        group_id: str | None = None,
    ) -> int:
        """Recover tasks stuck in PROCESSING state (e.g., after scheduler crash).

        This method finds tasks that have been in PROCESSING state for longer than
        max_processing_time and re-enqueues them as PENDING.

        In multi-instance deployments, this method uses distributed locks to ensure
        only one instance performs recovery at a time, preventing duplicate recovery.

        Args:
            max_processing_time: Maximum time a task should be in PROCESSING state
            group_id: Optional group ID to limit recovery to specific group

        Returns:
            Number of tasks recovered

        Raises:
            TimeoutError: If distributed lock cannot be acquired (when distributed_lock is configured)
            FairQueueStorageError: If recovery operation fails

        Example:
            ```python
            # Recover tasks stuck for more than 5 minutes
            recovered = await queue.recover_stuck_tasks(max_processing_time=timedelta(minutes=5))
            print(f"Recovered {recovered} stuck tasks")
            ```

        """
        # Use distributed lock if available (for multi-instance coordination)
        if self._distributed_lock:
            lock_key = f"recovery:{group_id or 'all'}"
            async with DistributedLockContext(
                self._distributed_lock,
                lock_key,
                self._recovery_lock_timeout,
                lock_ttl=timedelta(minutes=10),  # Lock expires after 10 minutes
            ):
                return await self._recover_stuck_tasks_internal(max_processing_time, group_id)
        else:
            # Single-instance mode, no lock needed
            return await self._recover_stuck_tasks_internal(max_processing_time, group_id)

    async def _get_processing_start_time(self, task_id: str, task_group_id: str) -> datetime | None:
        """Get processing start time from task metadata.

        Args:
            task_id: Task ID
            task_group_id: Group ID

        Returns:
            Processing start time or None if not available

        """
        try:
            task_data = await self._storage.get_task(task_id, task_group_id)
            if task_data and task_data.metadata and task_data.metadata.processing_started_at:
                return datetime.fromisoformat(task_data.metadata.processing_started_at)
        except Exception as e:
            # Can't get task or metadata, return None
            logger.debug(f"Could not get processing start time for task {task_id}: {e}", exc_info=True)
        return None

    async def _should_recover_task(
        self,
        task_id: str,
        task_group_id: str,
        cutoff_time: datetime,
    ) -> bool:
        """Determine if a task should be recovered.

        Args:
            task_id: Task ID
            task_group_id: Group ID
            cutoff_time: Time threshold for considering task stuck

        Returns:
            True if task should be recovered, False otherwise

        """
        # Check if task is being processed locally
        if self._scheduler.is_task_processing(task_id):
            logger.debug(
                f"Task {task_id} is actively being processed locally, skipping recovery",
                extra={"task_id": task_id, "group_id": task_group_id},
            )
            return False

        # Get processing start time from metadata
        processing_started_at = await self._get_processing_start_time(task_id, task_group_id)

        # If we have processing_started_at, use it to check if task is truly stuck
        # Otherwise, assume it's stuck (conservative approach for multi-instance)
        return processing_started_at < cutoff_time if processing_started_at else True

    async def _recover_single_task(
        self,
        task_id: str,
        task_group_id: str,
        max_processing_time: timedelta,
    ) -> bool:
        """Recover a single stuck task.

        Args:
            task_id: Task ID
            task_group_id: Group ID
            max_processing_time: Maximum processing time threshold

        Returns:
            True if task was recovered, False otherwise

        """
        # Verify task is still in PROCESSING state
        task_status = await self._storage.get_task_status(task_id, task_group_id)
        if task_status != TaskStatus.PROCESSING:
            return False

        # Check if task should be recovered
        cutoff_time = datetime.now(UTC) - max_processing_time
        if not await self._should_recover_task(task_id, task_group_id, cutoff_time):
            return False

        # Re-enqueue the task by marking it as PENDING
        await self._storage.update_task_status(task_id, task_group_id, TaskStatus.PENDING)
        logger.info(
            f"Recovered stuck task {task_id} from group {task_group_id} "
            f"(processing time exceeded {max_processing_time})",
            extra={"task_id": task_id, "group_id": task_group_id},
        )
        return True

    async def _recover_stuck_tasks_internal(
        self,
        max_processing_time: timedelta,
        group_id: str | None,
    ) -> int:
        """Internal recovery logic (called with or without distributed lock).

        Args:
            max_processing_time: Maximum time a task should be in PROCESSING state
            group_id: Optional group ID to limit recovery to specific group

        Returns:
            Number of tasks recovered

        """
        try:
            # Get all tasks in PROCESSING state
            processing_tasks = await self._storage.get_tasks_by_status(
                status=TaskStatus.PROCESSING,
                group_id=group_id,
                limit=1000,  # Process in batches
            )

            if not processing_tasks:
                return 0

            recovered_count = 0
            for task_id, task_group_id in processing_tasks:
                try:
                    if await self._recover_single_task(task_id, task_group_id, max_processing_time):
                        recovered_count += 1
                except Exception as e:
                    logger.warning(
                        f"Error processing stuck task {task_id}: {e}",
                        extra={"task_id": task_id},
                        exc_info=True,
                    )
                    continue

        except Exception as e:
            logger.exception("Failed to recover stuck tasks")
            raise FairQueueStorageError(
                detail=f"Failed to recover stuck tasks: {e}",
                operation="recover_stuck_tasks",
            ) from e

        return recovered_count

    async def get_task_result(self, task_id: str, group_id: str | None = None) -> Any | None:
        """Get task result if available.

        Args:
            task_id: Task ID
            group_id: Optional group ID (will be derived from task_id if not provided)

        Returns:
            Task result or None if not found or not completed

        Note:
            Results are automatically retrieved from TaskIQ result backend if configured.
            If a result handler is configured, results are also stored in the queue's storage
            for later retrieval. For fire-and-forget tasks without a result backend, results
            will not be available.

        Example:
            ```python
            # Enable auto result storage
            queue = (
                FairTaskQueueBuilder()
                .with_redis_storage(redis)
                .with_taskiq_adapter(broker)  # broker must have result_backend configured
                .with_auto_result_storage(result_ttl_seconds=86400)
                .build()
            )

            task_id = await queue.enqueue(task)
            result = await queue.get_task_result(task_id)
            ```

        """
        return await self._storage.get_task_result(task_id, group_id)

    async def get_task_status(self, task_id: str, group_id: str | None = None) -> TaskStatus | None:
        """Get task status by task ID.

        Args:
            task_id: Task ID
            group_id: Optional group ID (will be derived from task_id if not provided)

        Returns:
            Task status or None if task not found

        Example:
            ```python
            status = await queue.get_task_status(task_id, "user_123")
            if status == TaskStatus.COMPLETED:
                print("Task completed!")
            ```

        """
        return await self._storage.get_task_status(task_id, group_id)

    async def get_task_info(self, task_id: str, group_id: str | None = None) -> TaskInfo | None:
        """Get comprehensive task information.

        Args:
            task_id: Task ID
            group_id: Optional group ID (will be derived from task_id if not provided)

        Returns:
            TaskInfo model with task information, or None if task not found

        Example:
            ```python
            task_info = await queue.get_task_info(task_id)
            if task_info:
                print(f"Status: {task_info.status}")
                print(f"Retries: {task_info.retry_count}/{task_info.max_retries}")
            ```

        """
        if group_id is None:
            # Try to get group_id from task
            task = await self._storage.get_task(task_id, "")
            if task:
                group_id = task.group_id
            else:
                return None

        # Get task from storage
        task = await self._storage.get_task(task_id, group_id)
        if not task:
            return None

        # Get current status
        status = await self.get_task_status(task_id, group_id)

        # Check if task is being processed
        is_processing = self._scheduler.is_task_processing(task_id)

        # Check if result is available
        result_available = False
        if status == TaskStatus.COMPLETED:
            result = await self.get_task_result(task_id, group_id)
            result_available = result is not None

        return TaskInfo(
            task_id=task_id,
            group_id=group_id,
            status=status.value if status else None,
            enqueued_at=task.enqueued_at.isoformat() if task.enqueued_at else None,
            scheduled_at=task.scheduled_at.isoformat() if task.scheduled_at else None,
            expires_at=task.expires_at.isoformat() if task.expires_at else None,
            priority=task.priority,
            retry_count=task.retry_count,
            max_retries=task.max_retries,
            is_processing=is_processing,
            result_available=result_available,
            idempotency_key=task.idempotency_key,
        )

    async def cancel_task(self, task_id: str, group_id: str | None = None) -> bool:
        """Cancel a task.

        This method attempts to cancel a task whether it's pending, processing, or scheduled.
        If the task is currently being processed, it will be cancelled immediately.
        If the task is pending, it will be marked as cancelled in storage.

        Args:
            task_id: Task ID to cancel
            group_id: Optional group ID (will be derived from task_id if not provided)

        Returns:
            True if task was cancelled, False if not found or already completed/failed

        Example:
            ```python
            task_id = await queue.enqueue(task)
            # ... later ...
            cancelled = await queue.cancel_task(task_id)
            if cancelled:
                print("Task cancelled successfully")
            ```

        """
        # Try to cancel if currently processing
        if await self._scheduler.cancel_processing_task(task_id):
            return True

        # If not processing, try to get task and mark as cancelled
        if group_id is None:
            task = await self._storage.get_task(task_id, "")
            if task:
                group_id = task.group_id
            else:
                return False

        # Check current status
        status = await self.get_task_status(task_id, group_id)
        if status is None:
            return False

        # Can't cancel tasks that are already completed or failed
        if status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.EXPIRED):
            return False

        # Mark as cancelled in storage
        try:
            await self._storage.update_task_status(task_id, group_id, TaskStatus.CANCELLED)
        except Exception as e:
            logger.warning(
                f"Failed to cancel task {task_id}: {e}",
                extra={"task_id": task_id, "group_id": group_id},
                exc_info=True,
            )
            return False
        else:
            logger.info(
                f"Task {task_id} cancelled",
                extra={"task_id": task_id, "group_id": group_id},
            )
            return True

    async def wait_for_result(  # noqa: C901, PLR0912, PLR0915
        self,
        task_id: str,
        group_id: str | None = None,
        timeout_seconds: float | None = None,
        initial_poll_interval: timedelta = timedelta(seconds=0.5),
        max_poll_interval: timedelta = timedelta(seconds=5.0),
        use_events: bool = True,
    ) -> Any:
        """Wait for task result with event-driven notifications and polling fallback.

        Uses event-driven notifications when available (if task is being processed),
        with polling as fallback for tasks that haven't started yet.

        Args:
            task_id: Task ID
            group_id: Optional group ID (will be auto-derived from task_id if not provided)
            timeout_seconds: Maximum time to wait in seconds (None = wait indefinitely)
            initial_poll_interval: Initial interval between status checks (default: 0.5s)
            max_poll_interval: Maximum interval between status checks (default: 5.0s)
            use_events: If True, use event-driven notifications when available (default: True)

        Returns:
            Task result when available

        Raises:
            TimeoutError: If timeout is reached before task completes
            RuntimeError: If task fails or is cancelled
            FairQueueStorageError: If task cannot be found

        Example:
            ```python
            task_id = await queue.enqueue(task, group_id="user_123")
            # group_id is automatically derived, no need to provide it
            result = await queue.wait_for_result(task_id, timeout_seconds=30.0)
            ```

        """
        # Auto-derive group_id if not provided
        if group_id is None:
            derived_group_id = await self._storage.get_group_id_by_task_id(task_id)
            if derived_group_id is None:
                raise FairQueueStorageError(
                    detail=f"Task {task_id} not found. Cannot derive group_id.",
                    task_id=task_id,
                    operation="wait_for_result",
                )
            group_id = derived_group_id
        else:
            # Verify task exists even if group_id is provided
            task_status = await self.get_task_status(task_id, group_id)
            if task_status is None:
                raise FairQueueStorageError(
                    detail=f"Task {task_id} not found in group {group_id}.",
                    task_id=task_id,
                    group_id=group_id,
                    operation="wait_for_result",
                )

        start_time = datetime.now(UTC)
        poll_count = 0
        base_interval_seconds = initial_poll_interval.total_seconds()
        max_interval_seconds = max_poll_interval.total_seconds()

        # Try to get result event if task is being processed
        result_event: asyncio.Event | None = None
        if use_events and self._scheduler.is_task_processing(task_id):
            result_event = self._scheduler.get_result_event(task_id)

        while True:
            # Check timeout (but allow poll_count check to happen first for completed tasks)
            elapsed = (datetime.now(UTC) - start_time).total_seconds() if timeout_seconds else 0
            timeout_reached = timeout_seconds is not None and elapsed >= timeout_seconds

            # If we have an event and task is processing, wait on event with timeout
            if result_event is not None and self._scheduler.is_task_processing(task_id):
                # Wait on event with a short timeout to allow periodic status checks
                wait_timeout = (
                    min(
                        timeout_seconds - elapsed if timeout_seconds else max_interval_seconds,
                        max_interval_seconds,
                    )
                    if timeout_seconds
                    else max_interval_seconds
                )

                with contextlib.suppress(TimeoutError):
                    await asyncio.wait_for(result_event.wait(), timeout=wait_timeout)
                    # Event was set, task completed - check status immediately
                    # Fall through to status check below

            # Get task status (polling fallback or after event)
            status = await self.get_task_status(task_id, group_id)

            if status is None:
                # Task not found, wait and retry with exponential backoff
                # Check timeout before waiting
                if timeout_reached:
                    error_msg = (
                        f"Task {task_id} did not complete within {timeout_seconds} seconds. "
                        f"Current status: unknown (task not found). "
                        f"Use queue.get_task_info(task_id, group_id) for more details."
                    )
                    raise TimeoutError(error_msg)
                poll_interval = min(base_interval_seconds * (2**poll_count), max_interval_seconds)
                await asyncio.sleep(poll_interval)
                poll_count += 1
                continue

            if status == TaskStatus.COMPLETED:
                # Task completed, get result
                result = await self.get_task_result(task_id, group_id)
                if result is not None:
                    # Clean up result event before returning
                    if result_event is not None:
                        self._scheduler.cleanup_result_event(task_id)
                    return result

                # Result not in storage, try to get it from TaskIQ result backend directly
                # This handles cases where there's no result handler but TaskIQ has a result backend
                try:
                    adapter_result_obj = await self._task_adapter.get_result(task_id)
                    if adapter_result_obj is not None:
                        # Check if result indicates a failure
                        if hasattr(adapter_result_obj, "is_err") and adapter_result_obj.is_err:  # type: ignore[attr-defined]
                            # Task failed - raise RuntimeError
                            error_value = None
                            if hasattr(adapter_result_obj, "return_value"):
                                error_value = adapter_result_obj.return_value  # type: ignore[attr-defined]
                            elif hasattr(adapter_result_obj, "error"):
                                error_value = adapter_result_obj.error  # type: ignore[attr-defined]

                            if isinstance(error_value, Exception):
                                error = error_value
                            else:
                                error = RuntimeError(str(error_value) if error_value else "Task failed")

                            # Clean up result event before raising
                            if result_event is not None:
                                self._scheduler.cleanup_result_event(task_id)
                            self._raise_task_failed_error(task_id, error)

                        # Extract successful result value
                        adapter_result = adapter_result_obj
                        if hasattr(adapter_result_obj, "return_value"):
                            adapter_result = adapter_result_obj.return_value  # type: ignore[attr-defined]
                        elif hasattr(adapter_result_obj, "result"):
                            adapter_result = adapter_result_obj.result  # type: ignore[attr-defined]

                        # Clean up result event before returning
                        if result_event is not None:
                            self._scheduler.cleanup_result_event(task_id)
                        return adapter_result
                except RuntimeError:
                    # Task failed - re-raise
                    raise
                except Exception:  # noqa: S110
                    # Adapter get_result failed, continue with normal flow
                    # This is intentional - we don't want to fail if adapter result retrieval fails
                    pass

                # Result not yet available
                # If there's no result handler (or it's NoOpResultHandler), return None immediately
                # since the result will never be available (unless TaskIQ result backend has it, which we tried above)
                from haolib.background.fair.results import NoOpResultHandler  # noqa: PLC0415

                is_noop_handler = isinstance(self._result_handler, NoOpResultHandler)
                has_effective_result_handler = self._result_handler is not None and not is_noop_handler
                if not has_effective_result_handler:
                    # No result handler, task completed but no result stored - return None
                    if result_event is not None:
                        self._scheduler.cleanup_result_event(task_id)
                    return None
                # Result handler exists but result not yet available
                # If we've been waiting too long, the result may never arrive
                # This can happen when there's no result backend and the result handler never gets called
                # In this case, return None to avoid infinite waiting
                # Check poll_count first (before timeout) to give result handler a chance
                if poll_count > MAX_RESULT_POLL_COUNT:  # After max polls (~1-2 seconds with default intervals), give up
                    logger.warning(
                        f"Task {task_id} completed but result not available after {poll_count} polls. "
                        f"This may indicate: 1) No result backend configured, 2) Result handler not called, "
                        f"3) Result storage failed. Returning None.",
                        extra={"task_id": task_id, "group_id": group_id, "poll_count": poll_count},
                    )
                    if result_event is not None:
                        self._scheduler.cleanup_result_event(task_id)
                    return None
                # Check timeout before waiting (but only if poll_count hasn't exceeded threshold)
                if timeout_reached:
                    error_msg = (
                        f"Task {task_id} did not complete within {timeout_seconds} seconds. "
                        f"Current status: {status.value if status else 'unknown'}. "
                        f"Use queue.get_task_info(task_id, group_id) for more details."
                    )
                    raise TimeoutError(error_msg)
                # Wait a bit with exponential backoff before next poll
                poll_interval = min(base_interval_seconds * (2**poll_count), max_interval_seconds)
                await asyncio.sleep(poll_interval)
                poll_count += 1
                continue

            # Check timeout for non-completed statuses
            if timeout_reached:
                error_msg = (
                    f"Task {task_id} did not complete within {timeout_seconds} seconds. "
                    f"Current status: {status.value if status else 'unknown'}. "
                    f"Use queue.get_task_info(task_id, group_id) for more details."
                )
                raise TimeoutError(error_msg)

            if status == TaskStatus.FAILED:
                # Clean up result event before raising
                if result_event is not None:
                    self._scheduler.cleanup_result_event(task_id)
                # Get task info for better error message
                task_info = await self.get_task_info(task_id, group_id)
                if task_info:
                    error_msg = (
                        f"Task {task_id} failed after {task_info.retry_count}/{task_info.max_retries} retries. "
                        f"Check dead letter queue: queue.get_dead_letter_queue(group_id='{group_id or 'all'}') "
                        f"for error details and task data."
                    )
                else:
                    error_msg = (
                        f"Task {task_id} failed. Check dead letter queue: "
                        f"queue.get_dead_letter_queue(group_id='{group_id or 'all'}') for details."
                    )
                raise RuntimeError(error_msg)

            if status == TaskStatus.CANCELLED:
                # Clean up result event before raising
                if result_event is not None:
                    self._scheduler.cleanup_result_event(task_id)
                error_msg = (
                    f"Task {task_id} was cancelled. Use queue.get_task_info(task_id, group_id) for more details."
                )
                raise RuntimeError(error_msg)

            if status == TaskStatus.EXPIRED:
                # Clean up result event before raising
                if result_event is not None:
                    self._scheduler.cleanup_result_event(task_id)
                # Get task info for better error message
                task_info = await self.get_task_info(task_id, group_id)
                expires_info = ""
                if task_info and task_info.expires_at:
                    expires_info = f" (expired at: {task_info.expires_at})"
                error_msg = (
                    f"Task {task_id} expired before it could be processed{expires_info}. "
                    f"Consider increasing task TTL or processing capacity."
                )
                raise RuntimeError(error_msg)

            # Task is still pending or processing, wait and retry with exponential backoff
            # If we have an event, use shorter poll interval since event will notify us
            if result_event is not None:
                poll_interval = min(base_interval_seconds, max_interval_seconds)
            else:
                poll_interval = min(base_interval_seconds * (2**poll_count), max_interval_seconds)
            await asyncio.sleep(poll_interval)
            poll_count += 1

    async def stream_task_result(
        self,
        task_id: str,
        group_id: str | None = None,
        poll_interval: timedelta = timedelta(seconds=0.5),
        timeout_seconds: float | None = None,
    ) -> AsyncIterator[TaskResultUpdate]:
        """Stream task result updates as they become available.

        Args:
            task_id: Task ID to stream
            group_id: Optional group ID (will be derived from task_id if not provided)
            poll_interval: Interval between status checks
            timeout_seconds: Maximum time to wait in seconds (None = wait indefinitely)

        Yields:
            TaskResultUpdate model with status information

        Example:
            ```python
            async for update in queue.stream_task_result(task_id):
                print(f"Status: {update.status}")
                if update.result is not None:
                    print(f"Result: {update.result}")
                    break
            ```

        """
        async for update in stream_task_result(
            get_status=self.get_task_status,
            get_result=self.get_task_result,
            task_id=task_id,
            group_id=group_id,
            poll_interval=poll_interval,
            timeout_duration=timedelta(seconds=timeout_seconds) if timeout_seconds is not None else None,
        ):
            yield update

    async def start_scheduler(self) -> None:
        """Start the fair queue scheduler."""
        await self._scheduler.start()

    async def get_group_queue_length(self, group_id: str) -> int:
        """Get current queue length for a group.

        Args:
            group_id: Group ID

        Returns:
            Queue length

        """
        return await self._storage.get_group_queue_length(group_id)

    async def get_dead_letter_queue(
        self,
        group_id: str | None = None,
        limit: int = 100,
        offset: int = 0,
    ) -> list[FairTask]:
        """Get tasks from dead letter queue with pagination.

        Args:
            group_id: Optional group ID filter
            limit: Maximum number of tasks to return
            offset: Number of tasks to skip

        Returns:
            List of failed tasks

        """
        return await self._storage.get_dead_letter_queue(group_id=group_id, limit=limit, offset=offset)

    async def get_all_group_stats(
        self,
        limit: int | None = None,
        offset: int = 0,
    ) -> GroupStates:
        """Get statistics for all groups with pagination support.

        Args:
            limit: Maximum number of groups to return (None = all)
            offset: Number of groups to skip

        Returns:
            GroupStates model containing all group states

        """
        # Get all group IDs from storage
        group_ids = await self._storage.get_all_groups()

        # Apply pagination
        if limit is not None:
            group_ids = group_ids[offset : offset + limit]
        elif offset > 0:
            group_ids = group_ids[offset:]

        # Get state for each group
        stats: dict[str, GroupQueueState] = {}
        for group_id in group_ids:
            state = await self._storage.get_group_state(group_id)
            if state:
                stats[group_id] = state

        return GroupStates(groups=stats)

    async def set_group_weight(self, group_id: str, weight: float) -> None:
        """Set weight for a group (for Weighted Fair Queuing).

        Args:
            group_id: Group ID
            weight: Weight value (higher = more priority)

        Raises:
            ValueError: If weight is not positive

        Example:
            ```python
            await queue.set_group_weight("user_123", weight=2.0)  # Double priority
            ```

        """
        if weight <= 0:
            raise ValueError("Weight must be positive")

        state = await self._storage.get_group_state(group_id)
        if not state:
            # Create new state if group doesn't exist
            state = GroupQueueState(
                group_id=group_id,
                queue_length=0,
                virtual_finish_time=0.0,
                weight=weight,
            )
        else:
            state.weight = weight

        await self._storage.update_group_state(state)

    async def get_group_weight(self, group_id: str) -> float:
        """Get weight for a group.

        Args:
            group_id: Group ID

        Returns:
            Group weight (defaults to 1.0 if not set)

        """
        state = await self._storage.get_group_state(group_id)
        if state:
            return state.weight
        return 1.0

    async def get_queue_length(self, group_id: str) -> int:
        """Get queue length for a group (alias for get_group_queue_length).

        Args:
            group_id: Group ID

        Returns:
            Queue length (number of pending tasks for this group)

        Example:
            ```python
            length = await queue.get_queue_length("user_123")
            print(f"User 123 has {length} pending tasks")
            ```

        """
        return await self.get_group_queue_length(group_id)

    async def _auto_recovery_loop(self) -> None:
        """Background task that periodically recovers stuck tasks.

        This runs automatically when enable_auto_recovery is True.
        """
        while True:
            try:
                await asyncio.sleep(self._auto_recovery_interval.total_seconds())
                try:
                    recovered = await self.recover_stuck_tasks(
                        max_processing_time=self._auto_recovery_max_processing_time,
                    )
                    if recovered > 0:
                        logger.info(
                            f"Auto-recovery: Recovered {recovered} stuck task(s)",
                            extra={"recovered_count": recovered},
                        )
                except Exception as e:
                    logger.warning(
                        f"Auto-recovery failed: {e}",
                        exc_info=True,
                    )
            except asyncio.CancelledError:
                break
            except Exception:
                logger.exception("Auto-recovery loop error")
                # Continue loop even on errors
                await asyncio.sleep(self._auto_recovery_interval.total_seconds())

    async def _check_queue_limits_with_backpressure(
        self,
        group_id: str,
        max_wait: timedelta,
        retry_interval: timedelta,
    ) -> None:
        """Check queue size limits with backpressure (wait and retry if full).

        This method checks limits WITHOUT incrementing counters (read-only check).
        The actual increment happens in _check_queue_limits after backpressure passes.

        Args:
            group_id: Group ID to check
            max_wait: Maximum time to wait for queue space
            retry_interval: Interval between retry attempts

        Raises:
            FairQueueQueueFullError: If queue size limit is still reached after waiting

        """
        start_time = datetime.now(UTC)
        while True:
            try:
                # Check limits WITHOUT incrementing (read-only check for backpressure loop)
                await self._check_queue_limits_readonly(group_id)
            except FairQueueQueueFullError:
                # Check if we've exceeded max wait time
                elapsed = datetime.now(UTC) - start_time
                if elapsed >= max_wait:
                    # Re-raise the error after max wait
                    raise

                # Wait before retrying
                await asyncio.sleep(retry_interval.total_seconds())
            else:
                # Success - queue has space
                return

    async def _check_queue_limits_readonly(self, group_id: str) -> None:
        """Check queue size limits without incrementing counters (read-only check).

        This is used in backpressure loops to avoid incrementing the counter
        multiple times before actually enqueueing.

        Args:
            group_id: Group ID to check

        Raises:
            FairQueueQueueFullError: If queue size limit is reached

        """
        # Check per-group limit first (cheaper check)
        if self._max_queue_size_per_group is not None:
            queue_length = await self._storage.get_group_queue_length(group_id)
            if queue_length >= self._max_queue_size_per_group:
                # Get additional context for better error message
                try:
                    group_state = await self._storage.get_group_state(group_id)
                    processing_info = ""
                    if group_state and group_state.last_processed_at:
                        time_since_last = datetime.now(UTC) - group_state.last_processed_at
                        processing_info = f" Last processed: {time_since_last.total_seconds():.1f}s ago."
                except Exception:
                    processing_info = ""
                message = (
                    f"Queue full for group '{group_id}': {queue_length}/{self._max_queue_size_per_group} tasks. "
                    f"Solutions: 1) Increase max_queue_size_per_group, 2) Process tasks faster "
                    f"(check scheduler_max_concurrent_tasks), 3) Reduce enqueue rate, "
                    f"4) Check if tasks are completing successfully.{processing_info}"
                )
                raise FairQueueQueueFullError(
                    detail=message,
                    group_id=group_id,
                    queue_length=queue_length,
                    max_size=self._max_queue_size_per_group,
                )

        # Check global limit (read-only, don't increment)
        if self._max_global_queue_size is not None:
            # Get current size without incrementing
            current_size = await self._storage.get_global_queue_size()
            if current_size >= self._max_global_queue_size:
                # Get additional context for better error message
                try:
                    metrics = await self.get_queue_metrics()
                    processing_info = f" Currently processing: {metrics.total_processing} tasks."
                except Exception:
                    processing_info = ""
                message = (
                    f"Global queue is full: {current_size}/{self._max_global_queue_size} tasks. "
                    f"Solutions: 1) Increase max_global_queue_size, 2) Process tasks faster "
                    f"(check scheduler_max_concurrent_tasks), 3) Reduce enqueue rate, "
                    f"4) Check for stuck tasks (use recover_stuck_tasks()).{processing_info}"
                )
                raise FairQueueQueueFullError(
                    detail=message,
                    group_id=group_id,
                    queue_length=current_size,
                    max_size=self._max_global_queue_size,
                )

    async def _rollback_global_queue_size(
        self,
        task_id: str,
        group_id: str | None,
        context: str,
    ) -> None:
        """Rollback global queue size increment with proper error handling.

        This is critical to prevent queue size drift when enqueue operations fail.
        Logs warnings but doesn't raise exceptions to avoid masking the original error.

        Args:
            task_id: Task ID for logging context
            group_id: Group ID for logging context
            context: Context string for error messages

        """
        if self._max_global_queue_size is not None:
            try:
                await self._storage.decrement_global_queue_size()
            except Exception as rollback_error:
                # Log rollback failure but don't fail - this is a best-effort operation
                # The counter may drift slightly, but will be corrected by actual dequeues
                context_str = str(context)
                error_str = str(rollback_error)
                logger.exception(
                    f"CRITICAL: Failed to rollback global queue size after {context_str}: {error_str}. "
                    f"This may cause queue size drift. Monitor queue metrics and consider manual correction.",
                    extra={
                        "task_id": task_id,
                        "group_id": group_id,
                        "context": context_str,
                        "rollback_error": error_str,
                    },
                )

    async def _check_queue_limits(self, group_id: str) -> None:
        """Check queue size limits and raise error if exceeded.

        Optimized to check both limits efficiently and provide better error messages.

        Args:
            group_id: Group ID to check

        Raises:
            FairQueueQueueFullError: If queue size limit is reached

        """
        # Check per-group limit first (cheaper check)
        if self._max_queue_size_per_group is not None:
            queue_length = await self._storage.get_group_queue_length(group_id)
            if queue_length >= self._max_queue_size_per_group:
                # Get additional context for better error message
                try:
                    group_state = await self._storage.get_group_state(group_id)
                    processing_info = ""
                    if group_state and group_state.last_processed_at:
                        time_since_last = datetime.now(UTC) - group_state.last_processed_at
                        processing_info = f" Last processed: {time_since_last.total_seconds():.1f}s ago."
                except Exception:
                    processing_info = ""
                message = (
                    f"Queue full for group '{group_id}': {queue_length}/{self._max_queue_size_per_group} tasks. "
                    f"Solutions: 1) Increase max_queue_size_per_group, 2) Process tasks faster "
                    f"(check scheduler_max_concurrent_tasks), 3) Reduce enqueue rate, "
                    f"4) Check if tasks are completing successfully.{processing_info}"
                )
                raise FairQueueQueueFullError(
                    detail=message,
                    group_id=group_id,
                    queue_length=queue_length,
                    max_size=self._max_queue_size_per_group,
                )

        # Check global limit atomically (this increments the counter, so we need to handle rollback on failure)
        if self._max_global_queue_size is not None:
            allowed, current_size = await self._storage.check_and_increment_global_queue_size(
                self._max_global_queue_size
            )
            if not allowed:
                # Rollback the increment since we're not actually enqueueing
                await self._rollback_global_queue_size("", group_id, "queue limit check")

                # Get additional context for better error message
                try:
                    metrics = await self.get_queue_metrics()
                    processing_info = f" Currently processing: {metrics.total_processing} tasks."
                except Exception:
                    processing_info = ""
                message = (
                    f"Global queue is full: {current_size}/{self._max_global_queue_size} tasks. "
                    f"Solutions: 1) Increase max_global_queue_size, 2) Process tasks faster "
                    f"(check scheduler_max_concurrent_tasks), 3) Reduce enqueue rate, "
                    f"4) Check for stuck tasks (use recover_stuck_tasks()).{processing_info}"
                )
                raise FairQueueQueueFullError(
                    detail=message,
                    group_id=group_id,
                    queue_length=current_size,
                    max_size=self._max_global_queue_size,
                )

    async def __aenter__(self) -> Self:
        """Enter async context manager."""
        await self.start_scheduler()
        # Start auto-recovery if enabled
        if self._enable_auto_recovery:
            self._recovery_task = asyncio.create_task(self._auto_recovery_loop())
        return self

    async def __aexit__(
        self,
        exc_type: type[BaseException] | None,
        exc_val: BaseException | None,
        exc_tb: object,
    ) -> None:
        """Exit async context manager."""
        # Stop auto-recovery
        if self._recovery_task:
            self._recovery_task.cancel()
            with contextlib.suppress(asyncio.CancelledError):
                await self._recovery_task
            self._recovery_task = None
        await self.stop_scheduler(graceful=True)

    @property
    def scheduler(self) -> FairQueueScheduler:
        """Get the scheduler instance.

        Returns:
            The fair queue scheduler instance

        """
        return self._scheduler

    @property
    def storage(self) -> AbstractFairQueueStorage:
        """Get the storage instance.

        Returns:
            The storage backend instance

        """
        return self._storage

    @property
    def task_adapter(self) -> AbstractTaskAdapter:
        """Get the task adapter instance.

        Returns:
            The task adapter instance

        """
        return self._task_adapter

    @property
    def fairness_algorithm(self) -> FairnessAlgorithm:
        """Get the fairness algorithm instance.

        Returns:
            The fairness algorithm instance

        """
        return self._fairness_algorithm

    @property
    def max_pending_results(self) -> int:
        """Get the maximum number of pending results.

        Returns:
            Maximum number of pending results to track

        """
        return self._max_pending_results

    @property
    def max_result_events(self) -> int:
        """Get the maximum number of result events.

        Returns:
            Maximum number of result events to track

        """
        return self._max_result_events

    async def stop_scheduler(
        self,
        graceful: bool = True,
        shutdown_timeout: timedelta | None = None,
    ) -> None:
        """Stop the fair queue scheduler.

        Args:
            graceful: If True, wait for active tasks to complete. If False, cancel immediately.
            shutdown_timeout: Maximum time to wait for graceful shutdown. None means wait indefinitely.

        """
        await self._scheduler.stop(graceful=graceful, shutdown_timeout=shutdown_timeout)

    async def health_check(self) -> bool:
        """Check if queue and scheduler are healthy.

        Verifies:
        1. Storage backend is healthy
        2. Scheduler is healthy

        Returns:
            True if healthy, False otherwise

        """
        status = await self.get_health_status()
        return status.healthy

    async def get_health_status(self) -> OverallHealthStatus:
        """Get detailed health status for queue and scheduler.

        Returns:
            OverallHealthStatus model with comprehensive health status

        """

        components: dict[str, ComponentHealthStatus] = {}
        overall_healthy = True
        errors: list[str] = []

        # Check storage health
        try:
            storage_status = await self._storage.get_health_status()
            components["storage"] = storage_status
            if not storage_status.healthy:
                overall_healthy = False
                if storage_status.error:
                    errors.append(f"Storage: {storage_status.error}")
        except Exception as e:
            overall_healthy = False
            components["storage"] = ComponentHealthStatus(
                healthy=False,
                component="storage",
                details=HealthDetails(custom_data={}),
                error=str(e),
                timestamp=datetime.now(UTC),
            )
            errors.append(f"Storage check failed: {e}")

        # Check scheduler health
        try:
            scheduler_status = await self._scheduler.get_health_status()
            components["scheduler"] = scheduler_status
            if not scheduler_status.healthy:
                overall_healthy = False
                if scheduler_status.error:
                    errors.append(f"Scheduler: {scheduler_status.error}")
        except Exception as e:
            overall_healthy = False
            components["scheduler"] = ComponentHealthStatus(
                healthy=False,
                component="scheduler",
                details=HealthDetails(custom_data={}),
                error=str(e),
                timestamp=datetime.now(UTC),
            )
            errors.append(f"Scheduler check failed: {e}")

        queue_stats = await self._get_queue_stats()

        return OverallHealthStatus(
            healthy=overall_healthy,
            component="fair_task_queue",
            details=OverallHealthStatusDetails(
                components=ComponentHealthStatusMap(components=components),
                queue_stats=queue_stats,
            ),
            error="; ".join(errors) if errors else None,
            timestamp=datetime.now(UTC),
        )

    async def _get_queue_stats(self) -> QueueStats:
        """Get queue statistics.

        Returns:
            QueueStats model with queue statistics

        """
        try:
            all_stats = await self.get_all_group_stats()
            total_groups = len(all_stats.groups)
            total_queue_length = sum(state.queue_length for state in all_stats.groups.values())
            return QueueStats(
                total_groups=total_groups,
                total_queue_length=total_queue_length,
                groups=GroupQueueLengths(
                    lengths={group_id: state.queue_length for group_id, state in all_stats.groups.items()}
                ),
            )
        except Exception as e:
            return QueueStats(
                total_groups=0,
                total_queue_length=0,
                groups=GroupQueueLengths(lengths={}),
                error=f"Failed to get queue stats: {e}",
            )

    async def pause(self) -> None:
        """Pause the queue scheduler.

        When paused, the scheduler will stop processing new tasks but will
        continue processing tasks that are already in progress.

        Example:
            ```python
            await queue.pause()
            # Queue is paused, no new tasks will be processed
            await queue.resume()
            # Queue is resumed
            ```

        """
        self._paused = True
        await self._scheduler.pause()
        logger.info("FairTaskQueue paused")

    async def resume(self) -> None:
        """Resume the queue scheduler.

        Resumes processing of tasks after a pause.

        Example:
            ```python
            await queue.pause()
            # ... do something ...
            await queue.resume()
            ```

        """
        self._paused = False
        await self._scheduler.resume()
        logger.info("FairTaskQueue resumed")

    def is_paused(self) -> bool:
        """Check if the queue is paused.

        Returns:
            True if queue is paused, False otherwise

        """
        return self._paused

    async def pause_scheduler(self) -> None:
        """Pause the scheduler (alias for pause).

        Example:
            ```python
            await queue.pause_scheduler()
            # ... do something ...
            await queue.resume_scheduler()
            ```

        """
        await self.pause()

    async def resume_scheduler(self) -> None:
        """Resume the scheduler (alias for resume).

        Example:
            ```python
            await queue.pause_scheduler()
            # ... do something ...
            await queue.resume_scheduler()
            ```

        """
        await self.resume()

    async def get_queue_metrics(self) -> QueueMetrics:
        """Get comprehensive queue metrics.

        Returns:
            QueueMetrics model with detailed queue metrics

        Example:
            ```python
            metrics = await queue.get_queue_metrics()
            print(f"Total pending tasks: {metrics.total_queue_length}")
            print(f"Active groups: {metrics.total_groups}")
            ```

        """
        try:
            all_stats = await self.get_all_group_stats()
            total_groups = len(all_stats.groups)
            total_queue_length = sum(state.queue_length for state in all_stats.groups.values())

            # Get scheduler metrics
            scheduler_health = await self._scheduler.get_health_status()
            scheduler_details_dict = scheduler_health.details.custom_data
            active_tasks_count = scheduler_details_dict.get("active_tasks_count", 0)

            # Get processing tasks count
            processing_tasks = await self._storage.get_tasks_by_status(
                status=TaskStatus.PROCESSING,
                limit=10000,  # Large limit to get all processing tasks
            )
            total_processing = len(processing_tasks)

            # Get completed and failed counts (approximate, from recent tasks)
            completed_tasks = await self._storage.get_tasks_by_status(
                status=TaskStatus.COMPLETED,
                limit=1000,  # Sample size
            )
            failed_tasks = await self._storage.get_tasks_by_status(
                status=TaskStatus.FAILED,
                limit=1000,  # Sample size
            )

            return QueueMetrics(
                total_groups=total_groups,
                total_queue_length=total_queue_length,
                total_processing=total_processing,
                total_completed_sample=len(completed_tasks),
                total_failed_sample=len(failed_tasks),
                is_paused=self._paused,
                scheduler=SchedulerMetrics(
                    active_tasks=active_tasks_count,
                    max_concurrent_tasks=scheduler_details_dict.get("max_concurrent_tasks", 0),
                    healthy=scheduler_health.healthy,
                ),
                groups=GroupMetricsMap(
                    metrics={
                        group_id: GroupMetrics(
                            queue_length=state.queue_length,
                            last_processed_at=state.last_processed_at.isoformat() if state.last_processed_at else None,
                            weight=state.weight,
                        )
                        for group_id, state in all_stats.groups.items()
                    }
                ),
            )
        except Exception as e:
            logger.exception("Failed to get queue metrics")
            # Return minimal metrics with error
            return QueueMetrics(
                total_groups=0,
                total_queue_length=0,
                total_processing=0,
                total_completed_sample=0,
                total_failed_sample=0,
                is_paused=self._paused,
                scheduler=SchedulerMetrics(active_tasks=0, max_concurrent_tasks=0, healthy=False),
                groups=GroupMetricsMap(metrics={}),
                error=f"Failed to get queue metrics: {e}",
            )
