"""Redis storage implementation for fair task queue.

This module includes:
- Redis storage implementation
- Lua scripts for atomic operations
- Redis distributed lock
- Retry and timeout utilities for Redis operations
"""

import asyncio
import base64
import json
import logging
import uuid
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING, Any, NoReturn, TypeVar

from redis.exceptions import (
    ConnectionError as RedisConnectionError,
)
from redis.exceptions import (
    TimeoutError as RedisTimeoutError,
)

from haolib.background.fair.abstract import (
    ComponentHealthStatus,
    FairnessAlgorithm,
    FairTask,
    GroupMetadata,
    GroupQueueState,
    GroupStates,
    HealthDetails,
    RateLimit,
    StorageDetails,
    TaskMetadata,
    TaskStatus,
)
from haolib.background.fair.exceptions import (
    FairQueueIdempotencyConflictError,
    FairQueueStorageError,
)

if TYPE_CHECKING:
    from redis.asyncio import Redis

T = TypeVar("T")

# ============================================================================
# Constants
# ============================================================================

# Priority score multiplier for Redis sorted sets
# Used to convert priority (integer) to a score that can be combined with timestamp
# Higher priority tasks get lower scores (negative priority * multiplier + timestamp)
# This ensures priority ordering while maintaining FIFO within same priority
PRIORITY_SCORE_MULTIPLIER: float = 10_000_000_000.0  # 1e10

# ============================================================================
# Lua Scripts for Atomic Redis Operations
# ============================================================================

# Lua script for atomic rate limit check and increment
RATE_LIMIT_CHECK_AND_INCREMENT = """
local rate_limit_key = KEYS[1]
local max_tasks = tonumber(ARGV[1])
local per_time_seconds = tonumber(ARGV[2])
local current_time = tonumber(ARGV[3])

local rate_limit_data = redis.call('HGETALL', rate_limit_key)

if #rate_limit_data == 0 then
    -- Initialize rate limit
    redis.call('HSET', rate_limit_key,
        'max_tasks', max_tasks,
        'per_time', per_time_seconds,
        'current_count', '1',
        'window_start', current_time
    )
    return {1, current_time, 1}
end

-- Decode rate limit data
local max_tasks_stored = tonumber(rate_limit_data[2])
local per_time_stored = tonumber(rate_limit_data[4])
local current_count = tonumber(rate_limit_data[6])
local window_start = tonumber(rate_limit_data[8])

-- Check if window has expired
if (current_time - window_start) >= per_time_stored then
    -- Reset window
    redis.call('HSET', rate_limit_key,
        'current_count', '1',
        'window_start', current_time
    )
    return {1, current_time, 1}
end

-- Check if limit is reached
if current_count >= max_tasks_stored then
    return {0, window_start, current_count}
end

-- Increment count
local new_count = current_count + 1
redis.call('HINCRBY', rate_limit_key, 'current_count', 1)

return {1, window_start, new_count}
"""

# Lua script for atomic dequeue with task data preservation and expiration check
# Checks expiration and scheduled_at BEFORE removing from queue to prevent race conditions
DEQUEUE_TASK_ATOMIC = """
local group_tasks_key = KEYS[1]
local task_data_key_prefix = ARGV[1]
local task_status_key_prefix = ARGV[2]
local current_time = tonumber(ARGV[3])

-- Get all tasks with scores (to check expiration)
local tasks = redis.call('ZRANGE', group_tasks_key, 0, 0, 'WITHSCORES')

if #tasks == 0 then
    return nil
end

local task_id = tasks[1]
local task_data_key = task_data_key_prefix .. task_id
local task_status_key = task_status_key_prefix .. task_id

-- Get task data
local task_data = redis.call('HGET', task_data_key, 'data')

if not task_data then
    -- Remove invalid task from queue
    redis.call('ZREM', group_tasks_key, task_id)
    return nil
end

-- Note: Expiration and scheduled_at checks are handled in Python after dequeue
-- This avoids JSON parsing in Lua which can cause timeouts
-- The Python code will re-enqueue tasks that are scheduled for future or mark expired tasks
-- This is slightly less optimal but more reliable than parsing JSON in Lua

-- Task is ready to process - remove from queue (atomic operation)
redis.call('ZREM', group_tasks_key, task_id)

-- Update task status to PROCESSING (but keep task data for recovery)
redis.call('SET', task_status_key, 'processing')

return {task_id, task_data}
"""

# Lua script for atomic idempotency check
IDEMPOTENCY_CHECK = """
local idempotency_key = KEYS[1]
local task_id = ARGV[1]
local ttl = tonumber(ARGV[2])

local exists = redis.call('SET', idempotency_key, task_id, 'NX', 'EX', ttl)

if exists then
    return {1, task_id}
else
    local existing_task_id = redis.call('GET', idempotency_key)
    return {0, existing_task_id}
end
"""

# Lua script for atomic batch enqueue
BATCH_ENQUEUE_TASKS = """
local groups_key = KEYS[1]
local num_tasks = tonumber(ARGV[1])
local task_data_key_prefix = ARGV[2]
local task_status_key_prefix = ARGV[3]
local idempotency_key_prefix = ARGV[4]
local idempotency_ttl = tonumber(ARGV[5])

local results = {}
local idx = 6  -- Start of task data in ARGV

for i = 1, num_tasks do
    local task_id = ARGV[idx]
    local group_id = ARGV[idx + 1]
    local task_data = ARGV[idx + 2]
    local priority = tonumber(ARGV[idx + 3])
    local idempotency_key = ARGV[idx + 4]
    local enqueued_at = tonumber(ARGV[idx + 5])
    idx = idx + 6

    local success = true
    local error_msg = nil

    -- Check idempotency if provided
    if idempotency_key and idempotency_key ~= "" then
        local idempotency_redis_key = idempotency_key_prefix .. idempotency_key
        local exists = redis.call('SET', idempotency_redis_key, task_id, 'NX', 'EX', idempotency_ttl)
        if not exists then
            success = false
            error_msg = "idempotency_conflict"
        end
    end

    if success then
        -- Store task data
        local task_redis_key = task_data_key_prefix .. task_id
        redis.call('HSET', task_redis_key, 'data', task_data)

        -- Store task status
        local task_status_key = task_status_key_prefix .. task_id
        redis.call('SET', task_status_key, 'pending')

        -- Add to group's priority queue
        -- Extract prefix from task_data_key_prefix (format: "prefix:task:")
        local prefix_end = string.find(task_data_key_prefix, ":task:")
        local key_prefix = string.sub(task_data_key_prefix, 1, prefix_end - 1)
        local group_tasks_key = key_prefix .. ':group:' .. group_id .. ':tasks'
        local priority_multiplier = 10000000000  -- 1e10, matches PRIORITY_SCORE_MULTIPLIER constant
        local score = -priority * priority_multiplier + enqueued_at
        redis.call('ZADD', group_tasks_key, score, task_id)

        -- Add group to groups set
        redis.call('SADD', groups_key, group_id)
    end

    table.insert(results, {task_id, success, error_msg or ""})
end

return results
"""

# Lua script for atomic global queue size check and increment
GLOBAL_QUEUE_SIZE_CHECK_AND_INCREMENT = """
-- Atomically check and increment global queue size
-- KEYS[1]: global_queue_size_key
-- ARGV[1]: max_size
local current_size = redis.call('GET', KEYS[1])
if current_size == false then
    current_size = 0
else
    current_size = tonumber(current_size)
end

local max_size = tonumber(ARGV[1])
local new_size = current_size + 1

if new_size > max_size then
    -- Size would exceed limit, return not allowed with current size
    return {0, current_size}
else
    -- Increment and return allowed with new size
    redis.call('SET', KEYS[1], new_size)
    return {1, new_size}
end
"""

# Lua script for atomic batch global queue size check and increment
GLOBAL_QUEUE_SIZE_BATCH_CHECK_AND_INCREMENT = """
-- Atomically check and increment global queue size for a batch
-- KEYS[1]: global_queue_size_key
-- ARGV[1]: max_size
-- ARGV[2]: batch_size (number of tasks to add)
local current_size = redis.call('GET', KEYS[1])
if current_size == false then
    current_size = 0
else
    current_size = tonumber(current_size)
end

local max_size = tonumber(ARGV[1])
local batch_size = tonumber(ARGV[2])
local new_size = current_size + batch_size

if new_size > max_size then
    -- Size would exceed limit, return not allowed with current size
    return {0, current_size}
else
    -- Increment by batch size and return allowed with new size
    redis.call('INCRBY', KEYS[1], batch_size)
    return {1, new_size}
end
"""

# Lua scripts for distributed lock
ACQUIRE_LOCK = """
local lock_key = KEYS[1]
local lock_value = ARGV[1]
local lock_ttl = tonumber(ARGV[2])
local current_time = tonumber(ARGV[3])

-- Try to acquire lock using SET NX (set if not exists)
local acquired = redis.call('SET', lock_key, lock_value, 'NX', 'EX', lock_ttl)

if acquired then
    return {1, current_time}
else
    -- Check if lock exists and get its value
    local existing_value = redis.call('GET', lock_key)
    return {0, existing_value or ""}
end
"""

RELEASE_LOCK = """
local lock_key = KEYS[1]
local lock_value = ARGV[1]

-- Get current lock value
local current_value = redis.call('GET', lock_key)

if current_value == lock_value then
    -- We own the lock, release it
    redis.call('DEL', lock_key)
    return 1
else
    -- Lock doesn't exist or owned by someone else
    return 0
end
"""

CHECK_LOCK = """
local lock_key = KEYS[1]
local exists = redis.call('EXISTS', lock_key)
return exists
"""

# ============================================================================
# Retry and Timeout Utilities
# ============================================================================


async def retry_operation[T](
    operation: Callable[[], Awaitable[T]],
    max_retries: int = 3,
    initial_delay: float = 0.1,
    max_delay: float = 5.0,
    backoff_multiplier: float = 2.0,
    retryable_exceptions: tuple[type[Exception], ...] = (FairQueueStorageError, ConnectionError, TimeoutError),
) -> T:
    """Retry an async operation with exponential backoff.

    Args:
        operation: Async operation to retry
        max_retries: Maximum number of retry attempts
        initial_delay: Initial delay in seconds
        max_delay: Maximum delay in seconds
        backoff_multiplier: Multiplier for exponential backoff
        retryable_exceptions: Tuple of exception types that should trigger retry

    Returns:
        Result of the operation

    Raises:
        Last exception if all retries fail

    """
    last_exception: Exception | None = None
    delay = initial_delay

    for attempt in range(max_retries + 1):
        try:
            return await operation()
        except retryable_exceptions as e:
            last_exception = e
            if attempt < max_retries:
                await asyncio.sleep(min(delay, max_delay))
                delay *= backoff_multiplier
            else:
                raise

    # This should never be reached, but type checker needs it
    if last_exception:
        raise last_exception
    raise RuntimeError("Retry operation failed without exception")


async def with_timeout[T](
    coro: Awaitable[T],
    timeout_duration: timedelta,
    operation: str = "operation",
) -> T:
    """Execute coroutine with timeout.

    Args:
        coro: Coroutine to execute
        timeout_duration: Maximum time to wait
        operation: Operation name for error messages

    Returns:
        Result of coroutine

    Raises:
        FairQueueStorageError: If timeout is exceeded

    """
    try:
        return await asyncio.wait_for(coro, timeout=timeout_duration.total_seconds())
    except TimeoutError as e:
        message = f"{operation} timed out after {timeout_duration}"
        raise FairQueueStorageError(detail=message, operation=operation) from e


# ============================================================================
# Redis Distributed Lock
# ============================================================================


class RedisDistributedLock:
    """Redis-based distributed lock implementation.

    Uses Redis SET NX (set if not exists) with expiration for atomic lock acquisition.
    Each lock instance has a unique identifier to ensure only the owner can release it.

    Example:
        ```python
        lock = RedisDistributedLock(redis_client, key_prefix="fair_queue:lock")
        async with DistributedLockContext(lock, "recovery", timeout_duration=timedelta(seconds=5)):
            # Lock acquired, perform operations
            await recover_stuck_tasks()
        # Lock automatically released
        ```

    """

    def __init__(
        self,
        redis: Redis,
        key_prefix: str = "fair_queue:lock",
        default_lock_ttl: timedelta = timedelta(minutes=5),
    ) -> None:
        """Initialize Redis distributed lock.

        Args:
            redis: Redis async client instance
            key_prefix: Prefix for all lock keys
            default_lock_ttl: Default TTL for locks (auto-release after this time)

        """
        self.redis = redis
        self._key_prefix = key_prefix
        self._default_lock_ttl = default_lock_ttl
        self._lock_identifier = str(uuid.uuid4())  # Unique identifier for this lock instance

        # Load Lua scripts
        self._acquire_script = redis.register_script(ACQUIRE_LOCK)
        self._release_script = redis.register_script(RELEASE_LOCK)
        self._check_script = redis.register_script(CHECK_LOCK)

    def _get_lock_key(self, lock_key: str) -> str:
        """Get full Redis key for lock.

        Args:
            lock_key: Base lock key

        Returns:
            Full Redis key with prefix

        """
        return f"{self._key_prefix}:{lock_key}"

    def _raise_acquire_lock_error(self) -> NoReturn:
        """Raise error for failed lock acquisition.

        Raises:
            FairQueueStorageError: Always raises with lock acquisition failure message

        """
        raise FairQueueStorageError(
            detail="Failed to acquire lock",
            operation="acquire_lock",
        )

    def _raise_release_lock_error(self) -> NoReturn:
        """Raise error for failed lock release.

        Raises:
            FairQueueStorageError: Always raises with lock release failure message

        """
        raise FairQueueStorageError(
            detail="Failed to release lock",
            operation="release_lock",
        )

    def _raise_rate_limit_error(self) -> NoReturn:
        """Raise error for failed rate limit check.

        Raises:
            FairQueueStorageError: Always raises with rate limit check failure message

        """
        raise FairQueueStorageError("Failed to check rate limit")

    async def acquire(
        self,
        lock_key: str,
        timeout_duration: timedelta,
        lock_ttl: timedelta | None = None,
    ) -> bool:
        """Acquire a distributed lock.

        Args:
            lock_key: Unique key for the lock
            timeout_duration: Maximum time to wait for lock acquisition
            lock_ttl: Optional TTL for the lock (defaults to default_lock_ttl)

        Returns:
            True if lock was acquired, False if timeout was reached

        Raises:
            FairQueueStorageError: If lock operation fails

        """
        full_lock_key = self._get_lock_key(lock_key)
        ttl_seconds = int((lock_ttl or self._default_lock_ttl).total_seconds())
        start_time = datetime.now(UTC)
        timeout_seconds = timeout_duration.total_seconds()

        while True:
            try:
                current_time = datetime.now(UTC).timestamp()
                result = await self._acquire_script(
                    keys=[full_lock_key],
                    args=[self._lock_identifier, str(ttl_seconds), str(current_time)],
                )

                if result is None:
                    self._raise_acquire_lock_error()

                acquired, _ = result
                if acquired == 1:
                    return True

                # Lock is held by someone else, wait and retry
                elapsed = (datetime.now(UTC) - start_time).total_seconds()
                if elapsed >= timeout_seconds:
                    return False

                # Exponential backoff: wait a bit before retrying
                wait_time = min(0.1 * (1 + elapsed / timeout_seconds), 1.0)
                await asyncio.sleep(wait_time)

            except Exception as e:
                if isinstance(e, FairQueueStorageError):
                    raise
                raise FairQueueStorageError(
                    detail=f"Failed to acquire lock: {e}",
                    operation="acquire_lock",
                ) from e

    async def release(self, lock_key: str) -> None:
        """Release a distributed lock.

        Only releases the lock if we own it (using lock identifier).

        Args:
            lock_key: Key of the lock to release

        Raises:
            FairQueueStorageError: If lock release fails

        """
        full_lock_key = self._get_lock_key(lock_key)

        try:
            result = await self._release_script(
                keys=[full_lock_key],
                args=[self._lock_identifier],
            )

            if result is None:
                self._raise_release_lock_error()

            released = result
            if released == 0:
                # Lock doesn't exist or owned by someone else
                # This is not necessarily an error - lock might have expired
                pass

        except Exception as e:
            if isinstance(e, FairQueueStorageError):
                raise
            raise FairQueueStorageError(
                detail=f"Failed to release lock: {e}",
                operation="release_lock",
            ) from e

    async def is_locked(self, lock_key: str) -> bool:
        """Check if a lock is currently held.

        Args:
            lock_key: Key of the lock to check

        Returns:
            True if lock is held, False otherwise

        Raises:
            FairQueueStorageError: If check operation fails

        """
        full_lock_key = self._get_lock_key(lock_key)

        try:
            result = await self._check_script(keys=[full_lock_key], args=[])
            if result is None:
                return False
            return bool(result)
        except Exception as e:
            raise FairQueueStorageError(
                detail=f"Failed to check lock status: {e}",
                operation="is_locked",
            ) from e


# ============================================================================
# Redis Storage Implementation
# ============================================================================

# Redis key prefixes
KEY_PREFIX = "fair_queue"
GROUPS_KEY = f"{KEY_PREFIX}:groups"


def _group_tasks_key(group_id: str) -> str:
    """Get Redis key for group tasks."""
    return f"{KEY_PREFIX}:group:{group_id}:tasks"


def _group_state_key(group_id: str) -> str:
    """Get Redis key for group state."""
    return f"{KEY_PREFIX}:group:{group_id}:state"


def _task_key(task_id: str) -> str:
    """Get Redis key for task."""
    return f"{KEY_PREFIX}:task:{task_id}"


def _task_status_key(task_id: str) -> str:
    """Get Redis key for task status."""
    return f"{KEY_PREFIX}:task_status:{task_id}"


def _task_result_key(task_id: str) -> str:
    """Get Redis key for task result."""
    return f"{KEY_PREFIX}:result:{task_id}"


def _rate_limit_key(group_id: str) -> str:
    """Get Redis key for rate limit."""
    return f"{KEY_PREFIX}:rate_limit:{group_id}"


def _idempotency_key(key: str) -> str:
    """Get Redis key for idempotency."""
    return f"{KEY_PREFIX}:idempotency:{key}"


def _dead_letter_queue_key(group_id: str) -> str:
    """Get Redis key for dead letter queue."""
    return f"{KEY_PREFIX}:dlq:{group_id}"


# Legacy lambda-based key functions (kept for backward compatibility)
GROUP_TASKS_KEY = _group_tasks_key
GROUP_STATE_KEY = _group_state_key
TASK_KEY = _task_key
TASK_STATUS_KEY = _task_status_key
TASK_RESULT_KEY = _task_result_key
RATE_LIMIT_KEY = _rate_limit_key
IDEMPOTENCY_KEY = _idempotency_key
DEAD_LETTER_QUEUE_KEY = _dead_letter_queue_key
ROUND_ROBIN_STATE_KEY = f"{KEY_PREFIX}:round_robin:last_group"  # Shared state for round-robin
GLOBAL_QUEUE_SIZE_KEY = f"{KEY_PREFIX}:global_queue_size"

# Time constants (in seconds)
IDEMPOTENCY_KEY_TTL_SECONDS = 3600  # 1 hour default TTL for idempotency keys
TASK_STATUS_TTL_SECONDS = 86400  # 24 hours TTL for completed/failed task status
DEAD_LETTER_QUEUE_TTL_SECONDS = 2592000  # 30 days TTL for DLQ entries
HEALTH_CHECK_TEST_TTL_SECONDS = 1  # 1 second TTL for health check test key

# Status verification constants
STATUS_VERIFICATION_MAX_ATTEMPTS = 3  # Maximum number of verification attempts for status updates
STATUS_VERIFICATION_RETRY_DELAY_SECONDS = 0.01  # Delay between verification retries in seconds


# Default weight for group queues
DEFAULT_GROUP_WEIGHT = 1.0

# Task result storage constants
DEFAULT_TASK_RESULT_TTL_SECONDS = 86400  # 24 hours default TTL for task results


class RedisFairQueueStorage:
    """Redis storage implementation for fair task queue."""

    def __init__(
        self,
        redis: Redis,
        key_prefix: str = KEY_PREFIX,
        task_ttl_seconds: int | None = None,
        operation_timeout: timedelta | None = None,
        enable_connection_retry: bool = True,
        connection_retry_max_attempts: int = 3,
        connection_retry_initial_delay: float = 0.1,
        connection_retry_max_delay: float = 5.0,
    ) -> None:
        """Initialize Redis storage.

        Args:
            redis: Redis async client instance
            key_prefix: Prefix for all Redis keys
            task_ttl_seconds: Optional TTL for task keys in seconds
            operation_timeout: Optional timeout for storage operations (defaults to 5 seconds)
            enable_connection_retry: If True, automatically retry operations on connection failures
            connection_retry_max_attempts: Maximum retry attempts for connection failures
            connection_retry_initial_delay: Initial delay in seconds for retry backoff
            connection_retry_max_delay: Maximum delay in seconds for retry backoff

        """
        self.redis = redis
        self._key_prefix = key_prefix
        self._task_ttl = task_ttl_seconds
        self._operation_timeout = operation_timeout or timedelta(seconds=5)
        self._enable_connection_retry = enable_connection_retry
        self._connection_retry_max_attempts = connection_retry_max_attempts
        self._connection_retry_initial_delay = connection_retry_initial_delay
        self._connection_retry_max_delay = connection_retry_max_delay

        # Load Lua scripts for atomic operations
        self._rate_limit_script = self.redis.register_script(RATE_LIMIT_CHECK_AND_INCREMENT)
        self._dequeue_script = self.redis.register_script(DEQUEUE_TASK_ATOMIC)
        self._idempotency_script = self.redis.register_script(IDEMPOTENCY_CHECK)
        self._batch_enqueue_script = self.redis.register_script(BATCH_ENQUEUE_TASKS)
        self._global_queue_size_script = self.redis.register_script(GLOBAL_QUEUE_SIZE_CHECK_AND_INCREMENT)
        self._global_queue_size_batch_script = self.redis.register_script(GLOBAL_QUEUE_SIZE_BATCH_CHECK_AND_INCREMENT)

    async def _execute_with_retry[T](self, operation: Callable[[], Awaitable[T]]) -> T:
        """Execute a Redis operation with automatic retry on connection failures.

        Args:
            operation: Async operation to execute

        Returns:
            Result of the operation

        Raises:
            FairQueueStorageError: If operation fails after all retries

        """
        if not self._enable_connection_retry:
            return await operation()

        try:
            # Import here to avoid circular dependencies

            return await retry_operation(
                operation,
                max_retries=self._connection_retry_max_attempts,
                initial_delay=self._connection_retry_initial_delay,
                max_delay=self._connection_retry_max_delay,
                retryable_exceptions=(
                    FairQueueStorageError,
                    ConnectionError,
                    TimeoutError,
                    RedisConnectionError,
                    RedisTimeoutError,
                ),
            )
        except Exception as e:
            if isinstance(e, FairQueueStorageError):
                raise
            message = "Redis operation failed after retries"
            raise FairQueueStorageError(
                detail=message,
                operation="redis_operation",
            ) from e

    def _raise_rate_limit_error(self) -> NoReturn:
        """Raise error for failed rate limit check.

        Raises:
            FairQueueStorageError: Always raises with rate limit check failure message

        """
        raise FairQueueStorageError("Failed to check rate limit")

    async def enqueue_task(self, task: FairTask) -> None:
        """Enqueue a task to storage.

        Args:
            task: Task to enqueue

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            await with_timeout(
                self._check_idempotency(task),
                self._operation_timeout,
                "check_idempotency",
            )
            await with_timeout(
                self._store_task_data(task),
                self._operation_timeout,
                "store_task_data",
            )
            await with_timeout(
                self._store_task_status(task),
                self._operation_timeout,
                "store_task_status",
            )
            await with_timeout(
                self._set_task_ttl(task),
                self._operation_timeout,
                "set_task_ttl",
            )
            await with_timeout(
                self._add_task_to_queue(task),
                self._operation_timeout,
                "add_task_to_queue",
            )
            await with_timeout(
                self._add_group_to_set(task.group_id),
                self._operation_timeout,
                "add_group_to_set",
            )
            await with_timeout(
                self._update_group_state_internal(task.group_id, increment_queue_length=True),
                self._operation_timeout,
                "update_group_state",
            )

        except Exception as e:
            if isinstance(e, FairQueueStorageError):
                raise
            message = "Failed to enqueue task"
            raise FairQueueStorageError(
                detail=message,
                task_id=task.task_id,
                group_id=task.group_id,
                operation="enqueue_task",
            ) from e

    async def enqueue_tasks_batch(self, tasks: list[FairTask]) -> list[tuple[str, bool, str | None]]:
        """Enqueue multiple tasks atomically.

        Args:
            tasks: List of tasks to enqueue

        Returns:
            List of (task_id, success, error) tuples

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        if not tasks:
            return []

        try:
            # Prepare arguments for Lua script
            groups_key = f"{self._key_prefix}:groups"
            task_data_key_prefix = f"{self._key_prefix}:task:"
            task_status_key_prefix = f"{self._key_prefix}:task_status:"
            idempotency_key_prefix = f"{self._key_prefix}:idempotency:"
            idempotency_ttl = IDEMPOTENCY_KEY_TTL_SECONDS

            args = [
                str(len(tasks)),
                task_data_key_prefix,
                task_status_key_prefix,
                idempotency_key_prefix,
                str(idempotency_ttl),
            ]

            # Add task data for each task
            for task in tasks:
                # Serialize task data
                task_dict = task.model_dump(mode="json", exclude={"task_data"})
                task_data_bytes = task.task_data if isinstance(task.task_data, bytes) else str(task.task_data).encode()
                task_dict["task_data"] = base64.b64encode(task_data_bytes).decode()

                # Convert datetime fields to timestamps for Lua script compatibility
                if task.scheduled_at:
                    task_dict["scheduled_at_ts"] = task.scheduled_at.timestamp()
                if task.expires_at:
                    task_dict["expires_at_ts"] = task.expires_at.timestamp()

                task_data_json = json.dumps(task_dict, default=str)

                # Use scheduled_at if available, otherwise use enqueued_at for score calculation
                time_timestamp = (task.scheduled_at or task.enqueued_at).timestamp()
                args.extend(
                    [
                        task.task_id,
                        task.group_id,
                        task_data_json,
                        str(task.priority),
                        task.idempotency_key or "",
                        str(time_timestamp),
                    ]
                )

            # Execute batch enqueue script
            results = await self._batch_enqueue_script(keys=[groups_key], args=args)

            # Process results and update group states
            processed_results: list[tuple[str, bool, str | None]] = []
            for result in results:
                task_id_raw, success, error_msg = result
                # Decode task_id if it's bytes
                task_id = task_id_raw.decode() if isinstance(task_id_raw, bytes) else task_id_raw
                success_bool = bool(success)
                error = error_msg.decode() if isinstance(error_msg, bytes) else error_msg if error_msg else None

                # Find corresponding task to update group state
                if success_bool:
                    matching_task = next((t for t in tasks if t.task_id == task_id), None)
                    if matching_task:
                        await self._set_task_ttl(matching_task)
                        await self._update_group_state_internal(matching_task.group_id, increment_queue_length=True)

                processed_results.append((task_id, success_bool, error))

        except Exception as e:
            message = "Failed to enqueue tasks batch"
            raise FairQueueStorageError(
                detail=message,
                operation="enqueue_tasks_batch",
            ) from e

        return processed_results

    async def _check_idempotency(self, task: FairTask) -> None:
        """Check and register idempotency key if provided.

        Uses atomic Lua script to prevent race conditions.

        Args:
            task: Task to check idempotency for

        Raises:
            FairQueueIdempotencyConflictError: If idempotency key conflict occurs
            FairQueueStorageError: If storage operation fails

        """
        if not task.idempotency_key:
            return

        idempotency_redis_key = f"{self._key_prefix}:idempotency:{task.idempotency_key}"
        ttl = IDEMPOTENCY_KEY_TTL_SECONDS

        # Use atomic Lua script
        result = await self._idempotency_script(keys=[idempotency_redis_key], args=[task.task_id, str(ttl)])

        if result is None:
            message = "Failed to check idempotency"
            raise FairQueueStorageError(
                detail=message,
                task_id=task.task_id,
                group_id=task.group_id,
                operation="check_idempotency",
            )

        success, existing_task_id = result
        if success == 0:  # Key already exists
            existing_id = existing_task_id.decode() if isinstance(existing_task_id, bytes) else existing_task_id
            raise FairQueueIdempotencyConflictError(task_id=existing_id, idempotency_key=task.idempotency_key)

    async def _store_task_data(self, task: FairTask) -> None:
        """Store task data in Redis.

        Args:
            task: Task to store

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        task_dict = task.model_dump(mode="json", exclude={"task_data"})
        task_data_bytes = task.task_data if isinstance(task.task_data, bytes) else str(task.task_data).encode()
        task_dict["task_data"] = base64.b64encode(task_data_bytes).decode()

        # Convert datetime fields to timestamps for Lua script compatibility
        # Lua script needs numeric timestamps for comparison
        if task_dict.get("scheduled_at"):
            task_dict["scheduled_at_ts"] = task.scheduled_at.timestamp() if task.scheduled_at else None
        if task_dict.get("expires_at"):
            task_dict["expires_at_ts"] = task.expires_at.timestamp() if task.expires_at else None

        task_redis_key = f"{self._key_prefix}:task:{task.task_id}"

        async def _store_data() -> int:
            result = await self.redis.hset(  # type: ignore[misc]
                task_redis_key,
                mapping={
                    "data": json.dumps(task_dict, default=str),
                },
            )
            return int(result) if not isinstance(result, int) else result

        await self._execute_with_retry(_store_data)

    async def _store_task_status(self, task: FairTask) -> None:
        """Store task status in Redis.

        Args:
            task: Task to store status for

        """
        task_status_key = f"{self._key_prefix}:task_status:{task.task_id}"

        async def _store_status() -> None:
            await self.redis.set(task_status_key, task.status.value)

        await self._execute_with_retry(_store_status)

    async def _set_task_ttl(self, task: FairTask) -> None:
        """Set TTL for task data and status if configured.

        Args:
            task: Task to set TTL for

        """
        if not self._task_ttl:
            return

        task_redis_key = f"{self._key_prefix}:task:{task.task_id}"
        task_status_key = f"{self._key_prefix}:task_status:{task.task_id}"
        ttl = self._task_ttl
        if ttl:

            async def _expire_task() -> None:
                await self.redis.expire(task_redis_key, ttl)

            async def _expire_status() -> None:
                await self.redis.expire(task_status_key, ttl)

            await self._execute_with_retry(_expire_task)
            await self._execute_with_retry(_expire_status)

    async def _add_task_to_queue(self, task: FairTask) -> None:
        """Add task to group's priority queue.

        For scheduled tasks, use scheduled_at timestamp for score calculation
        to ensure they're dequeued at the right time. For immediate tasks,
        use enqueued_at timestamp.

        Args:
            task: Task to add to queue

        """
        # Use scheduled_at if available, otherwise use enqueued_at
        # Scheduled tasks should be dequeued at their scheduled time
        time_timestamp = (task.scheduled_at or task.enqueued_at).timestamp()
        score = -task.priority * PRIORITY_SCORE_MULTIPLIER + time_timestamp
        group_tasks_key = f"{self._key_prefix}:group:{task.group_id}:tasks"

        async def _add_task() -> int:
            result = await self.redis.zadd(group_tasks_key, {task.task_id: score})
            return int(result) if not isinstance(result, int) else result

        await self._execute_with_retry(_add_task)

    async def _add_group_to_set(self, group_id: str) -> None:
        """Add group ID to groups set.

        Args:
            group_id: Group ID to add

        """
        groups_key = f"{self._key_prefix}:groups"

        async def _add_group() -> int:
            result = await self.redis.sadd(groups_key, group_id)  # type: ignore[misc]
            return int(result) if not isinstance(result, int) else result

        await self._execute_with_retry(_add_group)

    async def dequeue_task(self, group_id: str) -> FairTask | None:  # noqa: C901
        """Dequeue next task from a group's queue.

        Task data is preserved until mark_task_completed or mark_task_failed is called.
        This allows task recovery if processing fails.

        Args:
            group_id: Group ID to dequeue from

        Returns:
            Dequeued task or None if queue is empty

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        # Use iterative approach to handle expired tasks (max 10 iterations to prevent infinite loops)
        max_iterations = 10
        iteration = 0

        while iteration < max_iterations:
            try:
                group_tasks_key = f"{self._key_prefix}:group:{group_id}:tasks"
                task_data_key_prefix = f"{self._key_prefix}:task:"
                task_status_key_prefix = f"{self._key_prefix}:task_status:"

                # Use atomic Lua script for dequeue
                current_timestamp = datetime.now(UTC).timestamp()
                result = await self._dequeue_script(
                    keys=[group_tasks_key],
                    args=[task_data_key_prefix, task_status_key_prefix, str(current_timestamp)],
                )

                # Handle None result - could mean:
                # 1. No tasks in queue (return None)
                # 2. Task is scheduled for future (continue iteration)
                # 3. Task expired (continue iteration)
                if result is None:
                    # Check if queue is empty to distinguish between "no tasks" and "task skipped"
                    queue_length = await self.get_group_queue_length(group_id)
                    if queue_length == 0:
                        # Queue is empty, return None
                        return None
                    # Queue has tasks but they're scheduled/expired, continue iteration
                    iteration += 1
                    continue

                # Handle empty list or None first element
                if not result or result[0] is None:
                    # Similar check - queue might have tasks but they're scheduled/expired
                    queue_length = await self.get_group_queue_length(group_id)
                    if queue_length == 0:
                        return None
                    iteration += 1
                    continue

                _, task_data_bytes = result

                if task_data_bytes is None:
                    return None

                # Deserialize task
                task_data_str = (
                    task_data_bytes.decode("utf-8") if isinstance(task_data_bytes, bytes) else task_data_bytes
                )
                task_dict = json.loads(task_data_str)

                # Decode base64 task_data back to bytes
                if "task_data" in task_dict and isinstance(task_dict["task_data"], str):
                    task_dict["task_data"] = base64.b64decode(task_dict["task_data"])

                # Remove timestamp fields added for Lua script (not part of FairTask model)
                task_dict.pop("scheduled_at_ts", None)
                task_dict.pop("expires_at_ts", None)

                task = FairTask.model_validate(task_dict)

                # Check expiration and scheduled_at in Python (Lua script doesn't parse JSON to avoid timeouts)
                current_datetime = datetime.now(UTC)

                # Check if task is scheduled for future execution
                if task.scheduled_at and task.scheduled_at > current_datetime:
                    # Task is scheduled for future, re-enqueue it and try next task
                    # Re-add to queue with updated score (will be picked up when scheduled time arrives)
                    await self._add_task_to_queue(task)
                    # Revert status back to PENDING
                    await self.update_task_status(task.task_id, group_id, TaskStatus.PENDING)
                    iteration += 1
                    continue  # Try next task iteratively

                # Check expiration before returning task
                if task.expires_at and task.expires_at < current_datetime:
                    # Task expired, mark as expired and try next task
                    await self.update_task_status(task.task_id, group_id, TaskStatus.EXPIRED)
                    iteration += 1
                    continue  # Try next task iteratively

                # Task is ready to process - Lua script already marked it as PROCESSING and removed it from queue
                # Decrement global queue size (task was successfully dequeued and is ready to process)
                await self.decrement_global_queue_size()
                # Note: Group state queue_length is recalculated dynamically in get_group_state()
            except Exception as e:
                error_type = type(e).__name__
                error_str = str(e)
                message = (
                    f"Failed to dequeue task from group {group_id} (iteration {iteration}): {error_type}: {error_str}"
                )
                raise FairQueueStorageError(
                    detail=message,
                    group_id=group_id,
                    operation="dequeue_task",
                ) from e
            else:
                # Return task only if no exception occurred and task is ready
                return task

        # If we've exhausted max iterations, return None (likely all tasks expired)
        return None

    async def get_next_group(self, algorithm: FairnessAlgorithm) -> str | None:
        """Get next group to process based on fairness algorithm.

        Args:
            algorithm: Fairness algorithm to use

        Returns:
            Group ID to process next, or None if no groups available

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # Get all groups
            groups_key = f"{self._key_prefix}:groups"

            async def _get_groups() -> set[bytes]:
                result = await self.redis.smembers(groups_key)  # type: ignore[misc]
                return result if isinstance(result, set) else set()

            group_ids_bytes = await self._execute_with_retry(_get_groups)
            group_ids = group_ids_bytes if group_ids_bytes else set()

            if not group_ids:
                return None

            # Get states for all groups
            group_states_dict: dict[str, GroupQueueState] = {}
            for group_id_bytes in group_ids:
                group_id = group_id_bytes.decode() if isinstance(group_id_bytes, bytes) else group_id_bytes
                state = await self.get_group_state(group_id)
                if state and state.queue_length > 0:
                    group_states_dict[group_id] = state

            if not group_states_dict:
                return None

            # Use algorithm to select next group
            group_states = GroupStates(groups=group_states_dict)
            return algorithm.select_next_group(group_states)

        except Exception as e:
            message = "Failed to get next group"
            raise FairQueueStorageError(message) from e

    async def get_group_queue_length(self, group_id: str) -> int:
        """Get queue length for a group.

        Args:
            group_id: Group ID

        Returns:
            Queue length

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            group_tasks_key = f"{self._key_prefix}:group:{group_id}:tasks"

            async def _get_length() -> int:
                result = await self.redis.zcard(group_tasks_key)
                return int(result) if not isinstance(result, int) else result

            return await self._execute_with_retry(_get_length)
        except Exception as e:
            message = "Failed to get queue length"
            raise FairQueueStorageError(message) from e

    async def get_group_queue_lengths(self, group_ids: list[str]) -> dict[str, int]:
        """Get queue lengths for multiple groups.

        Args:
            group_ids: List of group IDs

        Returns:
            Dictionary mapping group ID to queue length

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            lengths: dict[str, int] = {}
            for group_id in group_ids:
                # Capture group_id in closure to avoid B023
                group_tasks_key = f"{self._key_prefix}:group:{group_id}:tasks"

                async def _get_group_length(key: str = group_tasks_key) -> int:
                    result = await self.redis.zcard(key)
                    return int(result) if not isinstance(result, int) else result

                length = await self._execute_with_retry(_get_group_length)
                lengths[group_id] = length
        except Exception as e:
            message = "Failed to get group queue lengths"
            raise FairQueueStorageError(message) from e
        else:
            return lengths

    async def get_group_state(self, group_id: str) -> GroupQueueState | None:
        """Get current state of a group's queue.

        Args:
            group_id: Group ID

        Returns:
            Group queue state or None if group doesn't exist

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            state_key = f"{self._key_prefix}:group:{group_id}:state"
            state_data = await self.redis.hgetall(state_key)  # type: ignore[misc]

            if not state_data:
                # Group doesn't exist, create default state
                queue_length = await self.get_group_queue_length(group_id)
                if queue_length == 0:
                    return None

                return GroupQueueState(
                    group_id=group_id,
                    queue_length=queue_length,
                    virtual_finish_time=0.0,
                    weight=DEFAULT_GROUP_WEIGHT,
                )

            # Decode bytes to strings
            decoded_data = {
                k.decode() if isinstance(k, bytes) else k: v.decode() if isinstance(v, bytes) else v
                for k, v in state_data.items()
            }

            # Get queue length
            queue_length = await self.get_group_queue_length(group_id)

            # Parse state
            state = GroupQueueState(
                group_id=group_id,
                queue_length=queue_length,
                virtual_finish_time=float(decoded_data.get("virtual_finish_time", "0.0")),
                weight=float(decoded_data.get("weight", "1.0")),
                last_processed_at=(
                    datetime.fromtimestamp(float(decoded_data["last_processed_at"]), UTC)
                    if decoded_data.get("last_processed_at")
                    else None
                ),
            )

            # Get rate limit if exists
            rate_limit_key = f"{self._key_prefix}:rate_limit:{group_id}"
            rate_limit_data = await self.redis.hgetall(rate_limit_key)  # type: ignore[misc]
            if rate_limit_data:
                decoded_rate_limit = {
                    k.decode() if isinstance(k, bytes) else k: v.decode() if isinstance(v, bytes) else v
                    for k, v in rate_limit_data.items()
                }

                state.rate_limit = RateLimit(
                    max_tasks=int(decoded_rate_limit.get("max_tasks", "0")),
                    per_time=timedelta(seconds=float(decoded_rate_limit.get("per_time", "0"))),
                    current_count=int(decoded_rate_limit.get("current_count", "0")),
                    window_start=datetime.fromtimestamp(float(decoded_rate_limit.get("window_start", "0")), UTC),
                )

            # Get metadata if exists
            metadata_str = await self.redis.hget(state_key, "metadata")  # type: ignore[misc]
            if metadata_str:
                metadata_bytes = metadata_str.decode() if isinstance(metadata_str, bytes) else metadata_str
                metadata_dict = json.loads(metadata_bytes)
                # Convert dict to GroupMetadata model
                state.metadata = GroupMetadata.model_validate(metadata_dict) if metadata_dict else GroupMetadata()

        except Exception as e:
            message = "Failed to get group state"
            raise FairQueueStorageError(message) from e

        return state

    async def get_all_groups(self) -> list[str]:
        """Get all active group IDs.

        Returns:
            List of all active group IDs

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            groups_key = f"{self._key_prefix}:groups"
            group_ids = await self.redis.smembers(groups_key)  # type: ignore[misc]
            return [group_id.decode() if isinstance(group_id, bytes) else group_id for group_id in group_ids]
        except Exception as e:
            message = "Failed to get all groups"
            raise FairQueueStorageError(message) from e

    async def update_group_state(self, state: GroupQueueState) -> None:
        """Update group queue state.

        Args:
            state: Group state to update

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            state_key = f"{self._key_prefix}:group:{state.group_id}:state"

            # Update state hash
            await self.redis.hset(  # type: ignore[misc]
                state_key,
                mapping={
                    "virtual_finish_time": str(state.virtual_finish_time),
                    "weight": str(state.weight),
                    "last_processed_at": (str(state.last_processed_at.timestamp()) if state.last_processed_at else ""),
                },
            )

            # Update rate limit if exists
            if state.rate_limit:
                rate_limit_key = f"{self._key_prefix}:rate_limit:{state.group_id}"
                await self.redis.hset(  # type: ignore[misc]
                    rate_limit_key,
                    mapping={
                        "max_tasks": str(state.rate_limit.max_tasks),
                        "per_time": str(state.rate_limit.per_time.total_seconds()),
                        "current_count": str(state.rate_limit.current_count),
                        "window_start": str(state.rate_limit.window_start.timestamp()),
                    },
                )

            # Update metadata if exists
            if state.metadata:
                # Convert GroupMetadata to dict for JSON serialization
                metadata_dict = state.metadata.model_dump()
                metadata_json = json.dumps(metadata_dict, default=str)
                await self.redis.hset(state_key, "metadata", metadata_json)  # type: ignore[misc]

        except Exception as e:
            message = "Failed to update group state"
            raise FairQueueStorageError(message) from e

    async def get_group_id_by_task_id(self, task_id: str) -> str | None:
        """Get group ID by task ID.

        Searches through all groups to find the task.

        Args:
            task_id: Task ID

        Returns:
            Group ID if found, None otherwise

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # First, check if task data exists and contains group_id
            task_redis_key = f"{self._key_prefix}:task:{task_id}"
            task_data = await self.redis.hget(task_redis_key, "data")  # type: ignore[misc]

            if task_data:
                task_data_str = task_data.decode("utf-8") if isinstance(task_data, bytes) else task_data
                task_dict = json.loads(task_data_str)
                if "group_id" in task_dict:
                    return task_dict["group_id"]

            # If not in task data, search through all groups
            group_ids = await self.get_all_groups()
            for group_id in group_ids:
                group_tasks_key = f"{self._key_prefix}:group:{group_id}:tasks"
                score = await self.redis.zscore(group_tasks_key, task_id)
                if score is not None:
                    return group_id

        except Exception as e:
            message = f"Failed to get group ID for task {task_id}"
            raise FairQueueStorageError(message) from e
        else:
            return None

    async def get_task_status(self, task_id: str, group_id: str | None = None) -> TaskStatus | None:  # noqa: PLR0911
        """Get task status by task ID and optional group ID.

        If group_id is not provided, it will be derived from task_id.

        Args:
            task_id: Task ID
            group_id: Optional group ID (will be derived if not provided)

        Returns:
            Task status or None if task not found

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # Derive group_id if not provided
            if group_id is None:
                group_id = await self.get_group_id_by_task_id(task_id)
                if group_id is None:
                    # Check status storage even if group_id not found
                    task_status_key = f"{self._key_prefix}:task_status:{task_id}"
                    status_value = await self.redis.get(task_status_key)
                    if status_value is None:
                        return None
                    status_str = status_value.decode() if isinstance(status_value, bytes) else status_value
                    try:
                        return TaskStatus(status_str)
                    except ValueError:
                        return None

            # First check if task is in queue (PENDING)
            group_tasks_key = f"{self._key_prefix}:group:{group_id}:tasks"
            score = await self.redis.zscore(group_tasks_key, task_id)
            if score is not None:
                return TaskStatus.PENDING

            # Check status storage
            task_status_key = f"{self._key_prefix}:task_status:{task_id}"
            status_value = await self.redis.get(task_status_key)

            if status_value is None:
                return None

            status_str = status_value.decode() if isinstance(status_value, bytes) else status_value
            return TaskStatus(status_str)

        except ValueError:
            # Invalid status value
            return None
        except Exception as e:
            message = f"Failed to get task status for task {task_id}"
            raise FairQueueStorageError(
                detail=message,
                task_id=task_id,
                group_id=group_id,
                operation="get_task_status",
            ) from e

    async def get_task(self, task_id: str, group_id: str) -> FairTask | None:
        """Get task by ID and group ID.

        Args:
            task_id: Task ID
            group_id: Group ID

        Returns:
            FairTask if found, None otherwise

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            task_redis_key = f"{self._key_prefix}:task:{task_id}"
            task_data = await self.redis.hget(task_redis_key, "data")  # type: ignore[misc]

            if task_data is None:
                return None

            # Deserialize task
            task_data_str = task_data.decode("utf-8") if isinstance(task_data, bytes) else task_data
            task_dict = json.loads(task_data_str)

            # Decode base64 task_data back to bytes
            if "task_data" in task_dict and isinstance(task_dict["task_data"], str):
                task_dict["task_data"] = base64.b64decode(task_dict["task_data"])

            return FairTask.model_validate(task_dict)

        except Exception as e:
            message = f"Failed to get task {task_id}"
            raise FairQueueStorageError(
                detail=message,
                task_id=task_id,
                group_id=group_id,
                operation="get_task",
            ) from e

    async def get_task_by_idempotency_key(self, idempotency_key: str) -> str | None:
        """Get task ID by idempotency key.

        Args:
            idempotency_key: Idempotency key

        Returns:
            Task ID if found, None otherwise

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            idempotency_redis_key = f"{self._key_prefix}:idempotency:{idempotency_key}"
            task_id = await self.redis.get(idempotency_redis_key)

            if task_id is None:
                return None

            return task_id.decode() if isinstance(task_id, bytes) else task_id

        except Exception as e:
            message = "Failed to get task by idempotency key"
            raise FairQueueStorageError(message) from e

    async def update_task_status(  # noqa: C901, PLR0912
        self, task_id: str, group_id: str, status: TaskStatus, metadata: TaskMetadata | None = None
    ) -> None:
        """Update task status and optionally update metadata.

        Args:
            task_id: Task ID
            group_id: Group ID
            status: New task status
            metadata: Optional metadata to update in task data

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            task_status_key = f"{self._key_prefix}:task_status:{task_id}"
            # Use execute_with_retry to ensure the status update persists
            # Direct set operation with retry for reliability

            async def _set_status() -> None:
                await self.redis.set(task_status_key, status.value)

            await self._execute_with_retry(_set_status)

            # Verify the status was actually set (helps catch race conditions and persistence issues)
            # This is a synchronous verification to ensure the write completed
            # Note: We use a small retry loop to handle potential race conditions
            for verification_attempt in range(STATUS_VERIFICATION_MAX_ATTEMPTS):
                verification_status = await self.redis.get(task_status_key)
                if verification_status is not None:
                    verification_str = (
                        verification_status.decode() if isinstance(verification_status, bytes) else verification_status
                    )
                    if verification_str == status.value:
                        # Verification successful
                        break
                # If verification failed, wait a tiny bit and retry (handles race conditions)
                if verification_attempt < STATUS_VERIFICATION_MAX_ATTEMPTS - 1:
                    await asyncio.sleep(STATUS_VERIFICATION_RETRY_DELAY_SECONDS)
            else:
                # All verification attempts failed - log warning but don't fail
                # This could be a race condition or timing issue, but the SET should have worked

                logger = logging.getLogger(__name__)
                if verification_status is None:
                    logger.warning(
                        # Here is no SQL injection
                        f"Status update verification failed: key {task_status_key} not found after SET operation. "  # noqa: S608
                        f"This may indicate a race condition or Redis issue.",
                        extra={
                            "task_id": task_id,
                            "group_id": group_id,
                            "status": status.value,
                        },
                    )
                else:
                    verification_str = (
                        verification_status.decode() if isinstance(verification_status, bytes) else verification_status
                    )
                    logger.warning(
                        f"Status update verification failed: expected {status.value}, got {verification_str}. "
                        f"This may indicate a race condition or status overwrite.",
                        extra={
                            "task_id": task_id,
                            "group_id": group_id,
                            "expected": status.value,
                            "actual": verification_str,
                        },
                    )
                # Don't raise exception - the SET operation should have succeeded
                # The verification is just a safety check

            # Remove cancelled tasks from the queue (they shouldn't be processed)
            if status == TaskStatus.CANCELLED:
                group_tasks_key = f"{self._key_prefix}:group:{group_id}:tasks"

                async def _remove_task() -> int:
                    result = await self.redis.zrem(group_tasks_key, task_id)
                    return int(result) if not isinstance(result, int) else result

                await self._execute_with_retry(_remove_task)
                # Decrement global queue size
                await self.decrement_global_queue_size()
                # Note: Group state queue_length is recalculated dynamically in get_group_state()

            # Set TTL for completed/failed tasks to auto-cleanup
            match status:
                case TaskStatus.COMPLETED | TaskStatus.FAILED | TaskStatus.EXPIRED | TaskStatus.CANCELLED:
                    await self.redis.expire(task_status_key, TASK_STATUS_TTL_SECONDS)
                case _:
                    pass

            # Update task metadata if provided
            if metadata:
                task_redis_key = f"{self._key_prefix}:task:{task_id}"
                task_data = await self.redis.hget(task_redis_key, "data")  # type: ignore[misc]
                if task_data:
                    task_data_str = task_data.decode("utf-8") if isinstance(task_data, bytes) else task_data
                    task_dict = json.loads(task_data_str)

                    # Update metadata - convert TaskMetadata to dict for storage
                    if "metadata" not in task_dict:
                        task_dict["metadata"] = {}
                    # Merge metadata fields
                    metadata_dict = metadata.model_dump(exclude_none=True)
                    task_dict["metadata"].update(metadata_dict)

                    # Save updated task data
                    task_data_json = json.dumps(task_dict, default=str)
                    await self.redis.hset(task_redis_key, "data", task_data_json)  # type: ignore[misc]

        except Exception as e:
            message = "Failed to update task status"
            raise FairQueueStorageError(message) from e

    async def mark_task_completed(self, task_id: str, group_id: str) -> None:
        """Mark task as completed and clean up task data.

        This should be called after successful task processing to clean up task data.

        Args:
            task_id: Task ID
            group_id: Group ID

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # Update status
            await self.update_task_status(task_id, group_id, TaskStatus.COMPLETED)

            # Clean up task data (but keep status for a while)
            task_redis_key = f"{self._key_prefix}:task:{task_id}"
            await self.redis.delete(task_redis_key)

            # Decrement global queue size
            await self.decrement_global_queue_size()

        except Exception as e:
            message = "Failed to mark task as completed"
            raise FairQueueStorageError(message) from e

    async def delete_task_data(self, task_id: str) -> None:
        """Delete task data from storage.

        This method deletes only the task data, not the task status.
        Useful for cleanup after task completion.

        Args:
            task_id: Task ID to delete data for

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            task_redis_key = f"{self._key_prefix}:task:{task_id}"
            await self.redis.delete(task_redis_key)
        except Exception as e:
            message = f"Failed to delete task data for task {task_id}"
            raise FairQueueStorageError(message) from e

    async def mark_task_failed(self, task_id: str, group_id: str) -> None:
        """Mark task as failed and preserve task data for DLQ.

        Task data is preserved so it can be moved to dead letter queue.

        Args:
            task_id: Task ID
            group_id: Group ID

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # Update status (task data is preserved for DLQ)
            await self.update_task_status(task_id, group_id, TaskStatus.FAILED)

        except Exception as e:
            message = "Failed to mark task as failed"
            raise FairQueueStorageError(message) from e

    async def add_to_dead_letter_queue(self, task: FairTask) -> None:
        """Add a task to the dead letter queue.

        This should be called when a task has exhausted all retries.
        The task should already be marked as failed.

        Args:
            task: Failed task to add to DLQ

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # Ensure task data is stored
            await self._store_task_data(task)
            await self._store_task_status(task)

            # Add to dead letter queue list
            dlq_key = DEAD_LETTER_QUEUE_KEY(task.group_id)
            await self.redis.lpush(dlq_key, task.task_id)  # type: ignore[misc]

            # Set TTL for DLQ entries (keep for 30 days by default)
            await self.redis.expire(dlq_key, DEAD_LETTER_QUEUE_TTL_SECONDS)

        except Exception as e:
            message = "Failed to add task to dead letter queue"
            raise FairQueueStorageError(message) from e

    async def check_and_increment_rate_limit(
        self,
        group_id: str,
        max_tasks: int,
        per_time: timedelta,
    ) -> tuple[bool, datetime, int]:
        """Atomically check and increment rate limit.

        Args:
            group_id: Group ID
            max_tasks: Maximum tasks allowed
            per_time: Time window

        Returns:
            Tuple of (allowed, window_start, current_count)

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            rate_limit_key = f"{self._key_prefix}:rate_limit:{group_id}"
            current_time = datetime.now(UTC).timestamp()

            result = await self._rate_limit_script(
                keys=[rate_limit_key],
                args=[str(max_tasks), str(per_time.total_seconds()), str(current_time)],
            )

            if result is None:
                self._raise_rate_limit_error()

            allowed, window_start_ts, current_count = result
            window_start = datetime.fromtimestamp(float(window_start_ts), UTC)

            return bool(allowed), window_start, int(current_count)

        except Exception as e:
            message = "Failed to check rate limit"
            raise FairQueueStorageError(message) from e

    async def check_and_increment_global_queue_size(
        self,
        max_size: int,
    ) -> tuple[bool, int]:
        """Atomically check and increment global queue size.

        Args:
            max_size: Maximum global queue size

        Returns:
            Tuple of (allowed, current_size)

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            result = await with_timeout(
                self._global_queue_size_script(
                    keys=[GLOBAL_QUEUE_SIZE_KEY],
                    args=[str(max_size)],
                ),
                self._operation_timeout,
                "check_and_increment_global_queue_size",
            )
            # Result is [allowed (1 or 0), current_size]  # noqa: ERA001
            allowed = bool(int(result[0]))
            current_size = int(result[1])
        except Exception as e:
            message = "Failed to check and increment global queue size"
            raise FairQueueStorageError(
                detail=message,
                operation="check_and_increment_global_queue_size",
            ) from e
        else:
            return (allowed, current_size)

    async def check_and_increment_global_queue_size_batch(
        self,
        max_size: int,
        batch_size: int,
    ) -> tuple[bool, int]:
        """Atomically check and increment global queue size for a batch of tasks.

        Args:
            max_size: Maximum global queue size
            batch_size: Number of tasks to add in the batch

        Returns:
            Tuple of (allowed, current_size) where current_size is the size after increment

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            result = await with_timeout(
                self._global_queue_size_batch_script(
                    keys=[GLOBAL_QUEUE_SIZE_KEY],
                    args=[str(max_size), str(batch_size)],
                ),
                self._operation_timeout,
                "check_and_increment_global_queue_size_batch",
            )
            # Result is [allowed (1 or 0), current_size]  # noqa: ERA001
            allowed = bool(int(result[0]))
            current_size = int(result[1])
        except Exception as e:
            message = "Failed to check and increment global queue size for batch"
            raise FairQueueStorageError(
                detail=message,
                operation="check_and_increment_global_queue_size_batch",
            ) from e
        else:
            return (allowed, current_size)

    async def get_global_queue_size(self) -> int:
        """Get current global queue size without incrementing.

        Returns:
            Current global queue size

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            result = await with_timeout(
                self.redis.get(GLOBAL_QUEUE_SIZE_KEY),
                self._operation_timeout,
                "get_global_queue_size",
            )
            if result is None:
                return 0
            if isinstance(result, bytes):
                return int(result.decode())
            return int(result)
        except Exception as e:
            message = "Failed to get global queue size"
            raise FairQueueStorageError(
                detail=message,
                operation="get_global_queue_size",
            ) from e

    async def decrement_global_queue_size(self) -> None:
        """Atomically decrement global queue size.

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            await with_timeout(
                self.redis.decr(GLOBAL_QUEUE_SIZE_KEY),
                self._operation_timeout,
                "decrement_global_queue_size",
            )
        except Exception as e:
            message = "Failed to decrement global queue size"
            raise FairQueueStorageError(
                detail=message,
                operation="decrement_global_queue_size",
            ) from e

    async def health_check(self) -> bool:
        """Check if storage is healthy.

        Performs comprehensive health checks:
        1. Verifies Redis connection with ping
        2. Tests basic read/write operations
        3. Verifies Lua scripts are loaded

        Returns:
            True if storage is healthy and functional, False otherwise

        """
        status = await self.get_health_status()
        return status.healthy

    async def get_health_status(self) -> ComponentHealthStatus:  # noqa: PLR0915
        """Get detailed health status.

        Returns:
            ComponentHealthStatus model with health status details

        """
        error: str | None = None
        healthy = True
        storage_details = StorageDetails(key_prefix=self._key_prefix)

        try:
            # 1. Check Redis connection
            try:
                await self.redis.ping()
                storage_details.redis_connection = "ok"
            except Exception as e:
                healthy = False
                error = f"Redis connection failed: {e}"
                storage_details.redis_connection = "failed"
                return ComponentHealthStatus(
                    healthy=False,
                    component="storage",
                    details=HealthDetails(custom_data=storage_details.model_dump()),
                    error=error,
                    timestamp=datetime.now(UTC),
                )

            # 2. Test basic read/write operation
            try:
                test_key = f"{self._key_prefix}:health_check:test"
                test_value = "health_check"
                await self.redis.set(test_key, test_value, ex=HEALTH_CHECK_TEST_TTL_SECONDS)
                retrieved = await self.redis.get(test_key)
                if retrieved is None:
                    healthy = False
                    error = "Redis read/write test failed: value not retrieved"
                    storage_details.read_write_test = "failed"
                else:
                    retrieved_str = retrieved.decode() if isinstance(retrieved, bytes) else retrieved
                    if retrieved_str != test_value:
                        healthy = False
                        error = "Redis read/write test failed: value mismatch"
                        storage_details.read_write_test = "failed"
                    else:
                        storage_details.read_write_test = "ok"
            except Exception as e:
                healthy = False
                error = f"Redis read/write test failed: {e}"
                storage_details.read_write_test = "failed"

            # 3. Verify Lua scripts are accessible
            scripts_ok = not (
                self._rate_limit_script is None or self._dequeue_script is None or self._idempotency_script is None
            )
            custom_data = storage_details.model_dump()
            if not scripts_ok:
                healthy = False
                if not error:
                    error = "Lua scripts not properly initialized"
                custom_data["lua_scripts"] = "failed"
            else:
                custom_data["lua_scripts"] = "ok"

            # 4. Get queue statistics
            try:
                group_ids = await self.get_all_groups()
                queue_lengths = [await self.get_group_queue_length(group_id) for group_id in group_ids]
                total_queue_length = sum(queue_lengths)
                custom_data["total_groups"] = len(group_ids)
                custom_data["total_queue_length"] = total_queue_length
            except Exception as e:
                custom_data["queue_stats"] = f"error: {e}"

        except Exception as e:
            healthy = False
            error = f"Health check failed: {e}"

        return ComponentHealthStatus(
            healthy=healthy,
            component="storage",
            details=HealthDetails(custom_data=custom_data),
            error=error,
            timestamp=datetime.now(UTC),
        )

    async def remove_task(self, task_id: str, group_id: str) -> None:
        """Remove a task from the queue.

        Args:
            task_id: Task ID to remove
            group_id: Group ID of the task

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # Remove from sorted set
            group_tasks_key = GROUP_TASKS_KEY(group_id)
            await self.redis.zrem(group_tasks_key, task_id)

            # Remove task data
            task_redis_key = TASK_KEY(task_id)
            await self.redis.delete(task_redis_key)

            # Remove task status as well
            task_status_key = TASK_STATUS_KEY(task_id)
            await self.redis.delete(task_status_key)

            # Decrement global queue size
            await self.decrement_global_queue_size()
            # Note: Group state queue_length is recalculated dynamically in get_group_state()

        except Exception as e:
            message = "Failed to remove task"
            raise FairQueueStorageError(message) from e

    async def purge_expired_tasks(self, before: datetime) -> int:
        """Remove all expired tasks before the given timestamp.

        Args:
            before: Remove tasks expired before this timestamp

        Returns:
            Number of tasks removed

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            count = 0
            group_ids = await self.redis.smembers(GROUPS_KEY)  # type: ignore[misc]

            for group_id_bytes in group_ids:
                group_id = group_id_bytes.decode() if isinstance(group_id_bytes, bytes) else group_id_bytes
                group_tasks_key = GROUP_TASKS_KEY(group_id)

                # Get all tasks in this group
                tasks = await self.redis.zrange(group_tasks_key, 0, -1, withscores=True)

                for task_id_item, _ in tasks:
                    task_id = task_id_item.decode() if isinstance(task_id_item, bytes) else task_id_item

                    # Get task data to check expiration
                    task_redis_key = TASK_KEY(task_id)
                    task_data = await self.redis.hget(task_redis_key, "data")  # type: ignore[misc]

                    if task_data:
                        task_data_str = task_data.decode("utf-8") if isinstance(task_data, bytes) else task_data
                        task_dict = json.loads(task_data_str)
                        # Decode base64 task_data back to bytes
                        if "task_data" in task_dict and isinstance(task_dict["task_data"], str):
                            task_dict["task_data"] = base64.b64decode(task_dict["task_data"])
                        task = FairTask.model_validate(task_dict)

                        if task.expires_at and task.expires_at < before:
                            # Remove expired task
                            await self.redis.zrem(group_tasks_key, task_id)
                            await self.redis.delete(task_redis_key)
                            count += 1

        except Exception as e:
            message = "Failed to purge expired tasks"
            raise FairQueueStorageError(message) from e

        return count

    async def get_dead_letter_queue(  # noqa: C901, PLR0912
        self,
        group_id: str | None = None,
        limit: int = 100,
        offset: int = 0,
    ) -> list[FairTask]:
        """Get tasks from dead letter queue with pagination.

        Args:
            group_id: Optional group ID filter
            limit: Maximum number of tasks to return
            offset: Number of tasks to skip

        Returns:
            List of failed tasks

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            tasks: list[FairTask] = []

            if group_id:
                # Get tasks for specific group with pagination
                dlq_key = DEAD_LETTER_QUEUE_KEY(group_id)
                # lrange is inclusive on both ends, so we need offset to offset+limit-1
                end_index = offset + limit - 1
                task_ids = await self.redis.lrange(dlq_key, offset, end_index)  # type: ignore[misc]

                for task_id_bytes in task_ids:
                    task_id = task_id_bytes.decode() if isinstance(task_id_bytes, bytes) else task_id_bytes
                    task_redis_key = TASK_KEY(task_id)
                    task_data = await self.redis.hget(task_redis_key, "data")  # type: ignore[misc]

                    if task_data:
                        task_data_str = task_data.decode("utf-8") if isinstance(task_data, bytes) else task_data
                        task_dict = json.loads(task_data_str)
                        # Decode base64 task_data back to bytes
                        if "task_data" in task_dict and isinstance(task_dict["task_data"], str):
                            task_dict["task_data"] = base64.b64decode(task_dict["task_data"])
                        tasks.append(FairTask.model_validate(task_dict))
            else:
                # Get tasks from all groups with pagination
                group_ids = await self.redis.smembers(GROUPS_KEY)  # type: ignore[misc]
                remaining = limit
                skipped = 0

                for group_id_bytes in group_ids:
                    if remaining <= 0:
                        break

                    group_id = group_id_bytes.decode() if isinstance(group_id_bytes, bytes) else group_id_bytes
                    dlq_key = DEAD_LETTER_QUEUE_KEY(group_id)
                    # Get all task IDs for this group to handle offset across groups
                    all_task_ids = await self.redis.lrange(dlq_key, 0, -1)  # type: ignore[misc]

                    # Apply offset
                    if skipped < offset:
                        skip_count = min(offset - skipped, len(all_task_ids))
                        all_task_ids = all_task_ids[skip_count:]
                        skipped += skip_count

                    # Take remaining tasks
                    for task_id_bytes in all_task_ids:
                        if remaining <= 0:
                            break

                        task_id = task_id_bytes.decode() if isinstance(task_id_bytes, bytes) else task_id_bytes
                        task_redis_key = TASK_KEY(task_id)
                        task_data = await self.redis.hget(task_redis_key, "data")  # type: ignore[misc]

                        if task_data:
                            task_data_str = task_data.decode("utf-8") if isinstance(task_data, bytes) else task_data
                            task_dict = json.loads(task_data_str)
                            # Decode base64 task_data back to bytes
                            if "task_data" in task_dict and isinstance(task_dict["task_data"], str):
                                task_dict["task_data"] = base64.b64decode(task_dict["task_data"])
                            tasks.append(FairTask.model_validate(task_dict))
                            remaining -= 1

        except Exception as e:
            message = "Failed to get dead letter queue"
            raise FairQueueStorageError(message) from e

        return tasks

    async def _update_group_state_internal(
        self,
        group_id: str,
        increment_queue_length: bool,  # noqa: ARG002
    ) -> None:
        """Internal method to update group state queue length.

        Args:
            group_id: Group ID
            increment_queue_length: If True, increment queue length; if False, decrement

        """
        # Queue length is calculated dynamically, so we just ensure state exists

        async def _get_state() -> GroupQueueState | None:
            return await self.get_group_state(group_id)

        state = await self._execute_with_retry(_get_state)
        if state:

            async def _update_state() -> None:
                await self.update_group_state(state)

            await self._execute_with_retry(_update_state)
        else:
            # Create initial state

            async def _get_queue_length() -> int:
                return await self.get_group_queue_length(group_id)

            queue_length = await self._execute_with_retry(_get_queue_length)
            if queue_length > 0:
                initial_state = GroupQueueState(
                    group_id=group_id,
                    queue_length=queue_length,
                    virtual_finish_time=0.0,
                    weight=DEFAULT_GROUP_WEIGHT,
                )

                async def _update_initial_state() -> None:
                    await self.update_group_state(initial_state)

                await self._execute_with_retry(_update_initial_state)

    async def store_task_result(self, task_id: str, group_id: str, result: Any, ttl_seconds: int | None = None) -> None:
        """Store task result.

        Args:
            task_id: Task ID
            group_id: Group ID
            result: Task result to store
            ttl_seconds: Optional TTL for result in seconds

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            result_key = TASK_RESULT_KEY(task_id)
            result_json = json.dumps(result, default=str)
            ttl = ttl_seconds if ttl_seconds is not None else DEFAULT_TASK_RESULT_TTL_SECONDS

            async def _store_result() -> None:
                await self.redis.set(result_key, result_json, ex=ttl)

            await self._execute_with_retry(_store_result)
        except Exception as e:
            message = "Failed to store task result"
            raise FairQueueStorageError(
                detail=message,
                task_id=task_id,
                group_id=group_id,
                operation="store_task_result",
            ) from e

    async def get_task_result(self, task_id: str, group_id: str | None = None) -> Any | None:
        """Get task result.

        If group_id is not provided, it will be derived from task_id (though not strictly needed for result retrieval).

        Args:
            task_id: Task ID
            group_id: Optional group ID (not strictly needed, kept for API consistency)

        Returns:
            Task result or None if not found

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            result_key = TASK_RESULT_KEY(task_id)
            result_data = await self.redis.get(result_key)

            if result_data is None:
                return None

            result_str = result_data.decode() if isinstance(result_data, bytes) else result_data
            return json.loads(result_str)
        except json.JSONDecodeError as e:
            message = f"Failed to decode task result for task {task_id}"
            raise FairQueueStorageError(
                detail=message,
                task_id=task_id,
                group_id=group_id,
                operation="get_task_result",
            ) from e
        except Exception as e:
            message = f"Failed to get task result for task {task_id}"
            raise FairQueueStorageError(
                detail=message,
                task_id=task_id,
                group_id=group_id,
                operation="get_task_result",
            ) from e

    async def get_tasks_by_status(  # noqa: C901, PLR0912, PLR0915
        self,
        status: TaskStatus,
        group_id: str | None = None,
        limit: int = 100,
        offset: int = 0,
    ) -> list[tuple[str, str]]:
        """Get task IDs by status with optional group filter.

        Note: This method uses SCAN which may be slow for large deployments.
        For better performance with PROCESSING tasks, consider using a dedicated index.
        For production use, limit should typically be <= 1000.

        Args:
            status: Task status to filter by
            group_id: Optional group ID filter (significantly improves performance)
            limit: Maximum number of tasks to return (recommended: <= 1000)
            offset: Number of tasks to skip

        Returns:
            List of (task_id, group_id) tuples

        Raises:
            FairQueueStorageError: If storage operation fails

        """
        try:
            # Optimize for PROCESSING status by checking active groups first
            if status == TaskStatus.PROCESSING and group_id:
                # For specific group, check if group has processing tasks
                # Check if group has any tasks (quick check)
                queue_length = await self.get_group_queue_length(group_id)
                if queue_length == 0:
                    # Still need to check status storage for tasks that were dequeued
                    pass

            status_value = status.value
            task_status_key_pattern = f"{self._key_prefix}:task_status:*"
            results: list[tuple[str, str]] = []

            # Optimize SCAN: use larger count when possible, but respect Redis limits
            scan_count = min(500, max(100, limit * 5))  # Adaptive count based on limit
            max_scans = min(100, (offset + limit) * 2)  # Limit total scans to prevent runaway
            cursor = 0
            scanned_count = 0
            matched_count = 0
            scan_iterations = 0

            while scan_iterations < max_scans and matched_count < limit:
                cursor, keys = await self.redis.scan(cursor, match=task_status_key_pattern, count=scan_count)
                scan_iterations += 1

                if not keys:
                    if cursor == 0:
                        break
                    continue

                # Batch get statuses for better performance
                if len(keys) > 1:
                    # Use pipeline for batch operations
                    pipe = self.redis.pipeline()
                    for key in keys:
                        pipe.get(key)
                    status_values = await pipe.execute()
                else:
                    status_values = [await self.redis.get(keys[0])]

                # Process keys with their statuses
                for key, status_bytes in zip(keys, status_values, strict=False):
                    scanned_count += 1

                    # Skip if status doesn't match
                    if status_bytes is None:
                        continue

                    status_str = status_bytes.decode() if isinstance(status_bytes, bytes) else status_bytes
                    if status_str != status_value:
                        continue

                    # Extract task_id from key (format: prefix:task_status:task_id)
                    task_id = key.decode().split(":")[-1] if isinstance(key, bytes) else key.split(":")[-1]

                    # Get group_id efficiently
                    task_group_id: str | None = None

                    # If group_id filter is provided, try to get it efficiently
                    if group_id is not None:
                        # Quick path: try to get from task data first (most common case)
                        task_key = TASK_KEY(task_id)
                        task_data: bytes | str | None = await self.redis.hget(task_key, "data")  # type: ignore[assignment,misc]
                        if task_data:
                            try:
                                task_dict = json.loads(
                                    task_data.decode() if isinstance(task_data, bytes) else task_data
                                )
                                task_group_id = task_dict.get("group_id")
                                if task_group_id != group_id:
                                    continue  # Skip if doesn't match filter
                            except Exception:
                                # Fallback to slower method
                                task_group_id = await self.get_group_id_by_task_id(task_id)
                        else:
                            task_group_id = await self.get_group_id_by_task_id(task_id)

                        if task_group_id != group_id:
                            continue
                    else:
                        # No filter, get group_id when needed
                        task_group_id = await self.get_group_id_by_task_id(task_id)
                        if task_group_id is None:
                            task_group_id = "unknown"

                    # Apply offset
                    if matched_count < offset:
                        matched_count += 1
                        continue

                    results.append((task_id, task_group_id or "unknown"))
                    matched_count += 1

                    if matched_count >= offset + limit:
                        break

                if matched_count >= offset + limit:
                    break

                if cursor == 0:
                    break

            return results[offset : offset + limit] if offset > 0 else results[:limit]

        except Exception as e:
            message = (
                f"Failed to get tasks by status '{status.value}'. "
                f"Tip: For better performance, specify group_id when possible. "
                f"Error: {e}"
            )
            raise FairQueueStorageError(
                detail=message,
                group_id=group_id,
                operation="get_tasks_by_status",
            ) from e
