"""Main job queue implementation with automatic retry and exponential backoff."""

from __future__ import annotations

import logging
import math
import os
import random
import signal
import threading
import time
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Callable, Literal

from oclawma.cache import CacheConfig, CacheKey, ResultCache
from oclawma.cancellation import (
    CancellationError,
    CancellationReason,
    CancellationToken,
    TimeoutConfig,
    TimeoutManager,
)
from oclawma.encryption import DecryptionError, EncryptionManager, EncryptionMetadata
from oclawma.queue.dlq import DeadLetterQueue
from oclawma.queue.models import Job, JobPriority, JobStats, JobStatus, _int_to_priority
from oclawma.queue.store import JobNotFoundError, JobStore
from oclawma.ratelimit import RateLimitConfig, RateLimiter, RateLimitInfo
from oclawma.scheduler.cron import get_next_run

# Optional distributed locking import
try:
    from oclawma.distributed import LockManager

    DISTRIBUTED_AVAILABLE = True
except ImportError:
    LockManager = Any  # type: ignore
    DISTRIBUTED_AVAILABLE = False

# Optional maintenance check import
try:
    from oclawma.maintenance import MaintenanceManager

    MAINTENANCE_AVAILABLE = True
except ImportError:
    MaintenanceManager = Any  # type: ignore
    MAINTENANCE_AVAILABLE = False

logger = logging.getLogger(__name__)

# Default configuration
DEFAULT_MAX_RETRIES = 3
DEFAULT_BASE_DELAY_SECONDS = 5  # Base delay for exponential backoff
DEFAULT_MAX_DELAY_SECONDS = 3600  # Max 1 hour delay

# Preemption configuration
PREEMPTION_ENABLED = True  # Enable CRITICAL job preemption

# DLQ callback type
DLQCallback = Callable[[Job, Exception], None]


class JobQueueError(Exception):
    """Raised when there's an error with the job queue."""

    pass


class JobQueue:
    """Persistent job queue with SQLite backend and automatic retry.

    Features:
    - ACID-compliant SQLite storage
    - Automatic retry with exponential backoff
    - Priority-based job scheduling
    - Delayed job execution
    - Rate limiting per job type
    - Optional payload encryption for sensitive data
    - Maintenance window awareness - pauses during maintenance
    - Thread-safe operations

    Example:
        >>> queue = JobQueue("/path/to/queue.db")
        >>> job = queue.enqueue({"task": "send_email", "to": "user@example.com"})
        >>> job, rate_info = queue.dequeue_with_rate_limit()
        >>> if job:
        ...     try:
        ...         # Process job...
        ...         queue.complete(job.id)
        ...     except Exception as e:
        ...         queue.fail(job.id, str(e))
    """

    def __init__(
        self,
        db_path: str | Path = ":memory:",
        max_retries: int = DEFAULT_MAX_RETRIES,
        base_delay_seconds: float = DEFAULT_BASE_DELAY_SECONDS,
        max_delay_seconds: float = DEFAULT_MAX_DELAY_SECONDS,
        dlq: DeadLetterQueue | None = None,
        on_dlq: DLQCallback | None = None,
        rate_limiter: RateLimiter | RateLimitConfig | None = None,
        rate_limit_behavior: Literal["block", "drop", "defer"] = "block",
        encryption_manager: EncryptionManager | None = None,
        lock_manager: LockManager | None = None,
        result_cache: ResultCache | CacheConfig | None = None,
        maintenance_manager: MaintenanceManager | None = None,
        skip_maintenance_check: bool = False,
    ) -> None:
        """Initialize the job queue.

        Args:
            db_path: Path to SQLite database. Use ":memory:" for in-memory.
            max_retries: Default max retries for new jobs
            base_delay_seconds: Base delay for exponential backoff
            max_delay_seconds: Maximum delay between retries
            dlq: Optional DeadLetterQueue to move exhausted jobs to
            on_dlq: Optional callback when a job is moved to DLQ
            rate_limiter: Optional RateLimiter instance or RateLimitConfig
            rate_limit_behavior: Behavior when rate limited ("block", "drop", "defer")
            encryption_manager: Optional EncryptionManager for payload encryption
            lock_manager: Optional LockManager for distributed locking
            result_cache: Optional ResultCache instance or CacheConfig
            maintenance_manager: Optional MaintenanceManager to check maintenance windows
            skip_maintenance_check: If True, skip maintenance checks during dequeue
        """
        self.store = JobStore(db_path)
        self.max_retries = max_retries
        self.base_delay_seconds = base_delay_seconds
        self.max_delay_seconds = max_delay_seconds
        self.dlq = dlq
        self.on_dlq = on_dlq
        self.encryption_manager = encryption_manager
        self.lock_manager = lock_manager
        self.maintenance_manager = maintenance_manager
        self.skip_maintenance_check = skip_maintenance_check
        self._lock = threading.RLock()
        self._running_jobs: set[int] = set()

        # Initialize rate limiter
        if rate_limiter is None:
            self.rate_limiter = None
        elif isinstance(rate_limiter, RateLimiter):
            self.rate_limiter = rate_limiter
        else:
            self.rate_limiter = RateLimiter(rate_limiter)

        self.rate_limit_behavior = rate_limit_behavior
        self._last_rate_limit_info: RateLimitInfo | None = None

        # Initialize result cache
        if result_cache is None:
            self.result_cache = None
        elif isinstance(result_cache, ResultCache):
            self.result_cache = result_cache
        else:
            self.result_cache = ResultCache(result_cache)

    def enqueue(
        self,
        payload: dict[str, Any],
        priority: JobPriority | int = JobPriority.NORMAL,
        scheduled_at: datetime | None = None,
        max_retries: int | None = None,
        depends_on: list[int] | None = None,
        job_type: str = "default",
        tenant_id: str | None = None,
        encrypted: bool = False,
        cron_expression: str | None = None,
        timezone: str = "UTC",
    ) -> Job:
        """Add a new job to the queue.

        If the job has dependencies, it will inherit the maximum priority
        from its dependencies (priority inheritance).

        If a CRITICAL job is enqueued and preemption is enabled, lower
        priority running jobs may be paused.

        If an encryption_manager is configured and the job_type is in
        encrypted_types, the payload will be encrypted before storage.
        Alternatively, set encrypted=True to force encryption.

        Args:
            payload: Job data/payload
            priority: Job priority level (CRITICAL, HIGH, NORMAL, LOW)
            scheduled_at: When to run the job (None = run ASAP)
            max_retries: Max retries for this job (None = use default)
            depends_on: List of job IDs that must complete before this job
            job_type: Type of job (for encryption opt-in)
            tenant_id: Optional tenant identifier for multi-tenant support
            encrypted: Force encryption of this job (requires encryption_manager)
            cron_expression: Cron expression for recurring jobs
            timezone: Timezone for cron expression (default: UTC)

        Returns:
            The created job
        """
        with self._lock:
            # Convert int priority to enum if needed
            if isinstance(priority, int):
                priority = _int_to_priority(priority)

            # Handle priority inheritance from dependencies
            if depends_on:
                inherited_priority = self._get_inherited_priority(depends_on)
                if inherited_priority > priority:
                    logger.debug(
                        f"Job inherits higher priority {inherited_priority.name} from dependencies"
                    )
                    priority = inherited_priority

            # Determine if payload should be encrypted
            should_encrypt = self.encryption_manager is not None and (
                encrypted or self.encryption_manager.should_encrypt(job_type)
            )

            # Prepare payload and encryption metadata
            payload_to_store = payload
            encrypted = False
            encryption_metadata = None

            if should_encrypt:
                try:
                    ciphertext = self.encryption_manager.encrypt_dict(payload)
                    payload_to_store = {"__encrypted__": ciphertext}
                    encrypted = True
                    encryption_metadata = EncryptionMetadata(
                        encrypted=True,
                        key_version=self.encryption_manager.current_key_version,
                        algorithm="AES-256-GCM",
                    ).to_dict()
                    logger.debug(f"Encrypted payload for job type '{job_type}'")
                except Exception as e:
                    logger.warning(f"Failed to encrypt payload: {e}. Storing unencrypted.")
                    payload_to_store = payload

            # Calculate next_run_at for cron jobs
            next_run_at = None
            if cron_expression:
                try:
                    # Import here to avoid circular import
                    from zoneinfo import ZoneInfo

                    tz = ZoneInfo(timezone) if timezone else None
                    next_run_at = get_next_run(cron_expression, tz)
                except Exception as e:
                    logger.warning(
                        f"Failed to calculate next_run_at for cron '{cron_expression}': {e}"
                    )

            job = Job(
                payload=payload_to_store,
                status=JobStatus.PENDING,
                priority=priority,
                scheduled_at=scheduled_at,
                max_retries=max_retries if max_retries is not None else self.max_retries,
                depends_on=depends_on,
                job_type=job_type,
                encrypted=encrypted,
                encryption_metadata=encryption_metadata,
                tenant_id=tenant_id,
                cron_expression=cron_expression,
                timezone=timezone,
                next_run_at=next_run_at,
            )
            inserted_job = self.store.insert(job)

            # Check if we need to preempt running jobs (CRITICAL priority)
            if PREEMPTION_ENABLED and priority == JobPriority.CRITICAL:
                self._preempt_lower_priority_jobs(priority)

            logger.debug(f"Enqueued job {inserted_job.id} with priority {priority.name}")
            return inserted_job

    def _get_inherited_priority(self, dependency_ids: list[int]) -> JobPriority:
        """Get the maximum priority from dependency jobs.

        Args:
            dependency_ids: List of job IDs to check

        Returns:
            Maximum priority found among dependencies (defaults to NORMAL)
        """
        max_priority = JobPriority.NORMAL
        for dep_id in dependency_ids:
            try:
                dep_job = self.store.get(dep_id)
                if dep_job.priority > max_priority:
                    max_priority = dep_job.priority
            except JobNotFoundError:
                logger.warning(f"Dependency job {dep_id} not found for priority inheritance")
        return max_priority

    def _preempt_lower_priority_jobs(self, new_priority: JobPriority) -> list[Job]:
        """Preempt running jobs with lower priority than the new job.

        Args:
            new_priority: Priority of the new job

        Returns:
            List of jobs that were preempted
        """
        preempted = []
        running_jobs = self.store.get_running_jobs()

        for job in running_jobs:
            if job.priority < new_priority:
                # Preempt this job
                job.status = JobStatus.PAUSED
                job.updated_at = datetime.utcnow()
                self.store.update(job)
                self._running_jobs.discard(job.id)
                preempted.append(job)
                logger.info(
                    f"Preempted job {job.id} (priority {job.priority.name}) "
                    f"for higher priority job (priority {new_priority.name})"
                )

        return preempted

    def dequeue(self, block: bool = False, timeout: float | None = None) -> Job | None:
        """Get the next job to process.

        Atomically updates the job status to RUNNING.

        Jobs are selected based on:
        1. Priority (higher priority first)
        2. Creation time (FIFO within same priority)
        3. Dependencies must be completed

        If the job has an encrypted payload, it will be decrypted.

        Args:
            block: If True, block until a job is available
            timeout: Max seconds to block (None = forever)

        Returns:
            Next job to process, or None if no jobs available
        """
        result = self.dequeue_with_rate_limit(block=block, timeout=timeout)
        if result and result[0]:
            return self._decrypt_job_if_needed(result[0])
        return None

    def dequeue_with_rate_limit(
        self,
        block: bool = False,
        timeout: float | None = None,
        job_type: str | None = None,
    ) -> tuple[Job | None, RateLimitInfo | None]:
        """Get the next job to process with rate limiting.

        Atomically updates the job status to RUNNING. Checks rate limits
        before returning a job.

        Jobs are selected based on:
        1. Priority (higher priority first)
        2. Creation time (FIFO within same priority)
        3. Dependencies must be completed
        4. Rate limits for the job type

        Args:
            block: If True, block until a job is available
            timeout: Max seconds to block (None = forever)
            job_type: Job type for rate limiting (default: extracted from job payload)

        Returns:
            Tuple of (Job | None, RateLimitInfo | None). If rate limited,
            returns (None, RateLimitInfo) with allowed=False.
        """
        start_time = time.time()

        while True:
            # Check if in maintenance mode
            if (
                not self.skip_maintenance_check
                and self.maintenance_manager is not None
                and MAINTENANCE_AVAILABLE
            ) and self.maintenance_manager.is_in_maintenance():
                if not block:
                    return None, None
                # Wait and retry during maintenance
                time.sleep(1.0)
                # Check timeout
                if timeout is not None and (time.time() - start_time) >= timeout:
                    return None, None
                continue

            with self._lock:
                # Get next pending job, checking dependencies
                job = self._get_next_available_job()

                if job:
                    # Check rate limit if rate limiter is configured
                    if self.rate_limiter is not None:
                        # Determine job type
                        actual_job_type = job_type or job.job_type

                        # Try to acquire rate limit token
                        rate_info = self.rate_limiter.try_acquire(actual_job_type)
                        self._last_rate_limit_info = rate_info

                        if not rate_info.allowed:
                            # Rate limited - handle according to behavior
                            behavior = self.rate_limit_behavior

                            if behavior == "drop":
                                # Mark job as failed due to rate limiting
                                job.status = JobStatus.FAILED
                                job.error = (
                                    f"Rate limited: retry after {rate_info.retry_after:.2f}s"
                                )
                                job.completed_at = datetime.utcnow()
                                job.updated_at = datetime.utcnow()
                                self.store.update(job)
                                logger.warning(
                                    f"Dropped job {job.id} due to rate limit "
                                    f"(retry after: {rate_info.retry_after:.2f}s)"
                                )
                                # Return None to indicate job was dropped
                                return None, rate_info

                            elif behavior == "defer":
                                # Reschedule job for later
                                retry_after = max(rate_info.retry_after, 1.0)
                                job.scheduled_at = datetime.utcnow() + timedelta(
                                    seconds=retry_after
                                )
                                job.updated_at = datetime.utcnow()
                                self.store.update(job)
                                logger.info(
                                    f"Deferred job {job.id} for {retry_after:.2f}s due to rate limit"
                                )
                                # Return None but job will be retried later
                                return None, rate_info

                            else:  # behavior == "block"
                                # Block and wait for rate limit to clear
                                if not block:
                                    return None, rate_info

                                # Check if we have time to wait
                                if timeout is not None:
                                    elapsed = time.time() - start_time
                                    if elapsed >= timeout:
                                        return None, rate_info

                                # Release lock and wait outside
                                pass

                        else:
                            # Rate limit check passed
                            job.status = JobStatus.RUNNING
                            job.started_at = datetime.utcnow()
                            job.updated_at = datetime.utcnow()
                            self.store.update(job)
                            self._running_jobs.add(job.id)
                            logger.debug(
                                f"Dequeued job {job.id} with priority {job.priority.name} "
                                f"(rate limit: {rate_info.tokens_remaining}/{rate_info.tokens_total})"
                            )
                            return self._decrypt_job_if_needed(job), rate_info

                    else:
                        # No rate limiting configured
                        job.status = JobStatus.RUNNING
                        job.started_at = datetime.utcnow()
                        job.updated_at = datetime.utcnow()
                        self.store.update(job)
                        self._running_jobs.add(job.id)
                        logger.debug(f"Dequeued job {job.id} with priority {job.priority.name}")
                        return self._decrypt_job_if_needed(job), None

            # No job available or rate limited and blocking
            if not block:
                return None, None

            # Check timeout
            if timeout is not None and (time.time() - start_time) >= timeout:
                return None, None

            # Wait before retry
            time.sleep(0.1)

    def _decrypt_job_if_needed(self, job: Job) -> Job:
        """Decrypt job payload if encrypted.

        Args:
            job: Job that may have encrypted payload

        Returns:
            Job with decrypted payload

        Raises:
            DecryptionError: If decryption fails
        """
        if not job.encrypted or not job.payload.get("__encrypted__"):
            return job

        if self.encryption_manager is None:
            raise DecryptionError(
                f"Job {job.id} has encrypted payload but no encryption_manager configured"
            )

        try:
            encrypted_data = job.payload["__encrypted__"]
            decrypted_payload = self.encryption_manager.decrypt_dict(encrypted_data)
            job.payload = decrypted_payload
            logger.debug(f"Decrypted payload for job {job.id}")
            return job
        except Exception as e:
            raise DecryptionError(f"Failed to decrypt job {job.id}: {e}") from e

    def _get_next_available_job(self) -> Job | None:
        """Get the next job that is ready to run (checks dependencies).

        Returns:
            Next available job or None
        """
        # Get all pending jobs ordered by priority, excluding running jobs
        exclude_ids = set(self._running_jobs)
        max_iterations = 1000  # Safety limit
        iterations = 0

        while iterations < max_iterations:
            iterations += 1
            job = self.store.get_next_pending(exclude_ids=exclude_ids)

            if job is None:
                return None

            # Check if dependencies are met
            if self._dependencies_completed(job):
                return job

            # Job has unmet dependencies, exclude it and try next
            exclude_ids.add(job.id)

        # Safety limit reached
        return None

    def _dependencies_completed(self, job: Job) -> bool:
        """Check if all dependencies for a job are completed.

        Args:
            job: Job to check

        Returns:
            True if all dependencies are completed or job has no dependencies
        """
        if not job.depends_on:
            return True

        for dep_id in job.depends_on:
            try:
                dep_job = self.store.get(dep_id)
                if dep_job.status != JobStatus.COMPLETED:
                    return False
            except JobNotFoundError:
                # Dependency doesn't exist - treat as not completed
                return False

        return True

    def complete(self, job_id: int) -> Job:
        """Mark a job as completed.

        Args:
            job_id: Job identifier

        Returns:
            The updated job

        Raises:
            JobNotFoundError: If job not found
            JobQueueError: If update fails
        """
        with self._lock:
            try:
                job = self.store.get(job_id)
                job.status = JobStatus.COMPLETED
                job.completed_at = datetime.utcnow()
                job.updated_at = datetime.utcnow()
                self._running_jobs.discard(job_id)
                return self.store.update(job)
            except JobNotFoundError:
                raise
            except Exception as e:
                raise JobQueueError(f"Failed to complete job {job_id}: {e}") from e

    def fail(self, job_id: int, error: str | None = None) -> Job:
        """Mark a job as failed.

        The job will be in FAILED status after this call. If the job is retryable,
        callers can explicitly call retry() to retry it. If not retryable,
        it will be moved to DLQ and the on_dlq callback will be invoked.

        Args:
            job_id: Job identifier
            error: Error message

        Returns:
            The updated job (in FAILED status)

        Raises:
            JobNotFoundError: If job not found
            JobQueueError: If update fails
        """
        with self._lock:
            try:
                job = self.store.get(job_id)
                job.status = JobStatus.FAILED
                job.error = error
                job.completed_at = datetime.utcnow()
                job.updated_at = datetime.utcnow()
                self._running_jobs.discard(job_id)
                updated_job = self.store.update(job)

                # Job is not retryable - call on_dlq callback and move to DLQ
                if not job.is_retryable():
                    error_exception = Exception(error or "Unknown error")

                    # Move to DLQ if configured
                    if self.dlq is not None:
                        try:
                            self.dlq.add(job, error_exception)
                            logger.info(f"Moved job {job_id} to Dead Letter Queue")
                        except Exception as dlq_err:
                            logger.error(f"Failed to move job {job_id} to DLQ: {dlq_err}")

                    # Call the on_dlq callback if configured
                    if self.on_dlq is not None:
                        try:
                            self.on_dlq(job, error_exception)
                        except Exception as cb_err:
                            logger.error(f"Error in on_dlq callback for job {job_id}: {cb_err}")

                return updated_job
            except JobNotFoundError:
                raise
            except Exception as e:
                raise JobQueueError(f"Failed to mark job {job_id} as failed: {e}") from e

    def retry(self, job_id: int, error: Exception | None = None) -> Job | None:
        """Retry a failed job with exponential backoff.

        If max retries exceeded, job is moved to the Dead Letter Queue (if configured)
        and the on_dlq callback is invoked.

        Args:
            job_id: Job identifier
            error: Optional exception that caused the failure (for DLQ)

        Returns:
            Updated job if retry scheduled, None if max retries exceeded

        Raises:
            JobNotFoundError: If job not found
            JobQueueError: If update fails
        """
        with self._lock:
            try:
                job = self.store.get(job_id)

                if job.status != JobStatus.FAILED:
                    raise JobQueueError(f"Job {job_id} is not in failed status")

                if not job.is_retryable():
                    logger.warning(f"Job {job_id} has exceeded max retries ({job.max_retries})")

                    # Move to DLQ if configured
                    if self.dlq is not None:
                        try:
                            self.dlq.add(job, error or Exception(job.error or "Unknown error"))
                            logger.info(f"Moved job {job_id} to Dead Letter Queue")
                        except Exception as dlq_err:
                            logger.error(f"Failed to move job {job_id} to DLQ: {dlq_err}")

                    # Call the on_dlq callback if configured
                    if self.on_dlq is not None:
                        try:
                            self.on_dlq(job, error or Exception(job.error or "Unknown error"))
                        except Exception as cb_err:
                            logger.error(f"Error in on_dlq callback for job {job_id}: {cb_err}")

                    return None

                # Calculate exponential backoff delay
                delay = self._calculate_backoff_delay(job.retry_count)
                # For first retry (retry_count=0), make job immediately available
                if job.retry_count == 0:
                    scheduled_at = None
                else:
                    scheduled_at = datetime.utcnow() + timedelta(seconds=delay)

                # Update job for retry
                job.retry_count += 1
                job.status = JobStatus.PENDING
                job.scheduled_at = scheduled_at
                job.error = None
                job.updated_at = datetime.utcnow()
                # Keep started_at and completed_at from previous attempt

                updated_job = self.store.update(job)
                logger.info(
                    f"Scheduled retry {updated_job.retry_count}/{updated_job.max_retries} "
                    f"for job {job_id} in {delay:.1f}s"
                )
                return updated_job

            except JobNotFoundError:
                raise
            except Exception as e:
                raise JobQueueError(f"Failed to retry job {job_id}: {e}") from e

    def _calculate_backoff_delay(self, retry_count: int) -> float:
        """Calculate exponential backoff delay.

        Uses exponential backoff with jitter:
        delay = min(base_delay * 2^retry_count, max_delay)

        Args:
            retry_count: Current retry attempt (0-indexed)

        Returns:
            Delay in seconds
        """
        # Exponential backoff: base_delay * 2^retry_count
        delay = self.base_delay_seconds * math.pow(2, retry_count)
        # Add small jitter (+-10%) to prevent thundering herd
        jitter = delay * 0.1 * (random.random() * 2 - 1)
        delay = delay + jitter
        # Cap at max delay
        return min(delay, self.max_delay_seconds)

    def get_job(self, job_id: int, decrypt: bool = True) -> Job:
        """Get a job by id.

        Args:
            job_id: Job identifier
            decrypt: Whether to decrypt payload if encrypted (default: True)

        Returns:
            The job

        Raises:
            JobNotFoundError: If job not found
        """
        job = self.store.get(job_id)
        if decrypt and job.encrypted:
            return self._decrypt_job_if_needed(job)
        return job

    def cancel(self, job_id: int) -> bool:
        """Cancel a pending, running, or paused job.

        Args:
            job_id: Job identifier

        Returns:
            True if cancelled, False if not found or not cancellable
        """
        with self._lock:
            try:
                job = self.store.get(job_id)
                if job.status not in (JobStatus.PENDING, JobStatus.RUNNING, JobStatus.PAUSED):
                    logger.warning(f"Cannot cancel job {job_id}: status is {job.status.value}")
                    return False

                self._running_jobs.discard(job_id)
                return self.store.delete(job_id)
            except JobNotFoundError:
                return False

    def cancel_with_signal(
        self,
        job_id: int,
        reason: str = "User requested",
        cancelled_by: str | None = None,
        target_pid: int | None = None,
    ) -> Job | None:
        """Cancel a job and optionally signal the process.

        This is useful for cancelling long-running jobs that may be
        running in a separate process.

        Args:
            job_id: Job identifier
            reason: Reason for cancellation
            cancelled_by: User or process that cancelled the job
            target_pid: Process ID to signal (SIGTERM)

        Returns:
            Cancelled job if successful, None if not found or not cancellable
        """
        with self._lock:
            job = self.cancel(job_id, reason, cancelled_by)

            if job and target_pid:
                try:
                    os.kill(target_pid, signal.SIGTERM)
                    logger.info(f"Sent SIGTERM to process {target_pid} for job {job_id}")
                except ProcessLookupError:
                    logger.warning(f"Process {target_pid} not found for job {job_id}")
                except PermissionError:
                    logger.error(f"Permission denied signaling process {target_pid}")
                except Exception as e:
                    logger.error(f"Failed to signal process {target_pid}: {e}")

            return job

    def resume(self, job_id: int) -> Job:
        """Resume a paused job.

        Args:
            job_id: Job identifier of the paused job

        Returns:
            The resumed job

        Raises:
            JobNotFoundError: If job not found
            JobQueueError: If job is not in PAUSED status
        """
        with self._lock:
            try:
                job = self.store.get(job_id)

                if job.status != JobStatus.PAUSED:
                    raise JobQueueError(
                        f"Job {job_id} is not in PAUSED status (current: {job.status.value})"
                    )

                job.status = JobStatus.RUNNING
                job.updated_at = datetime.utcnow()
                self.store.update(job)
                self._running_jobs.add(job_id)
                logger.info(f"Resumed job {job_id}")
                return job

            except JobNotFoundError:
                raise
            except JobQueueError:
                raise
            except Exception as e:
                raise JobQueueError(f"Failed to resume job {job_id}: {e}") from e

    def change_priority(self, job_id: int, new_priority: JobPriority | int) -> Job:
        """Change the priority of a pending job.

        This will reorder the queue so the job is dequeued according to
        its new priority.

        Args:
            job_id: Job identifier
            new_priority: New priority level

        Returns:
            The updated job

        Raises:
            JobNotFoundError: If job not found
            JobQueueError: If job is not in PENDING status
        """
        with self._lock:
            try:
                job = self.store.get(job_id)

                if job.status != JobStatus.PENDING:
                    raise JobQueueError(
                        f"Cannot change priority of job {job_id}: "
                        f"job must be in PENDING status (current: {job.status.value})"
                    )

                # Convert int to enum if needed
                if isinstance(new_priority, int):
                    new_priority = _int_to_priority(new_priority)

                old_priority = job.priority
                job.priority = new_priority
                job.updated_at = datetime.utcnow()
                updated = self.store.update(job)

                logger.info(
                    f"Changed priority of job {job_id} from {old_priority.name} to {new_priority.name}"
                )
                return updated

            except JobNotFoundError:
                raise
            except JobQueueError:
                raise
            except Exception as e:
                raise JobQueueError(f"Failed to change priority of job {job_id}: {e}") from e

    def get_preemptable_jobs(self, priority: JobPriority | int) -> list[Job]:
        """Get running jobs that can be preempted by a job with given priority.

        A job can be preempted if its priority is lower than the given priority.

        Args:
            priority: Priority level to check against

        Returns:
            List of jobs that can be preempted
        """
        with self._lock:
            # Convert int to enum if needed
            if isinstance(priority, int):
                priority = _int_to_priority(priority)

            running_jobs = self.store.get_running_jobs()
            return [job for job in running_jobs if job.priority < priority]

    def get_blocked_jobs(self) -> list[Job]:
        """Get jobs that are blocked by unmet dependencies.

        Returns:
            List of jobs waiting on dependencies
        """
        with self._lock:
            return self.store.get_blocked_jobs()

    def get_stats_by_priority(self) -> dict[JobPriority, int]:
        """Get job counts grouped by priority.

        Returns:
            Dictionary mapping priority to count
        """
        stats = self.store.get_by_priority()
        result = {}
        for priority in JobPriority:
            result[priority] = stats.get(priority.value, 0)
        return result

    def get_stats(self) -> JobStats:
        """Get queue statistics.

        Returns:
            JobStats with current queue metrics
        """
        stats = self.store.get_stats()
        return JobStats(
            total_jobs=stats["total"],
            pending=stats["pending"],
            running=stats["running"],
            completed=stats["completed"],
            failed=stats["failed"],
            retryable=stats["retryable"],
        )

    def queue_depth(self) -> int:
        """Get current queue depth (number of pending jobs).

        Returns:
            Number of pending jobs
        """
        return self.get_stats().queue_depth

    def list_jobs(
        self, status: JobStatus | None = None, limit: int | None = None, decrypt: bool = True
    ) -> list[Job]:
        """List jobs in the queue.

        Args:
            status: Filter by status
            limit: Max jobs to return
            decrypt: Whether to decrypt encrypted payloads (default: True)

        Returns:
            List of jobs
        """
        jobs = self.store.list_all(status=status, limit=limit)
        if decrypt:
            return [self._decrypt_job_if_needed(job) if job.encrypted else job for job in jobs]
        return jobs

    def process_next(self, handler: Callable[[Job], None]) -> bool:
        """Process the next available job with the given handler.

        This is a convenience method that dequeues, calls the handler,
        and handles completion/failure automatically.

        Args:
            handler: Function to process the job

        Returns:
            True if a job was processed, False if no jobs available
        """
        job = self.dequeue()
        if not job:
            return False

        try:
            handler(job)
            self.complete(job.id)
            return True
        except Exception as e:
            error_msg = str(e)
            self.fail(job.id, error_msg)

            # Always call retry - it handles both retryable and non-retryable cases
            # If retryable, it schedules a retry; if not, it moves to DLQ if configured
            self.retry(job.id, error=e)

            raise

    def cleanup_old_jobs(self, max_age_days: int, status: JobStatus | None = None) -> int:
        """Delete old jobs.

        Args:
            max_age_days: Delete jobs older than this many days
            status: Only delete jobs with this status (optional)

        Returns:
            Number of jobs deleted
        """
        return self.store.cleanup_old_jobs(max_age_days, status)

    def requeue_stuck_jobs(self, timeout_seconds: int = 300) -> int:
        """Requeue jobs that have been running too long.

        This is useful for recovering from crashes where jobs were left
        in RUNNING status.

        Args:
            timeout_seconds: Jobs running longer than this are considered stuck

        Returns:
            Number of jobs requeued
        """
        with self._lock:
            cutoff = datetime.utcnow() - timedelta(seconds=timeout_seconds)

            # Get stuck jobs
            stuck_jobs = []
            for job in self.store.list_all(status=JobStatus.RUNNING):
                if job.started_at and job.started_at < cutoff:
                    stuck_jobs.append(job)

            # Requeue them
            count = 0
            for job in stuck_jobs:
                job.status = JobStatus.PENDING
                job.started_at = None
                job.updated_at = datetime.utcnow()
                # Increment retry count since this is effectively a failure
                job.retry_count += 1
                self.store.update(job)
                count += 1
                logger.warning(f"Requeued stuck job {job.id} (was running since {job.started_at})")

            return count

    def check_rate_limit(self, job_type: str = "default") -> RateLimitInfo:
        """Check rate limit for a job type without consuming tokens.

        Args:
            job_type: The job type to check rate limit for

        Returns:
            RateLimitInfo with current quota status
        """
        if self.rate_limiter is None:
            return RateLimitInfo(
                allowed=True,
                tokens_remaining=-1,
                tokens_total=-1,
                job_type=job_type,
            )
        return self.rate_limiter.get_quota(job_type)

    def get_rate_limit_headers(self, job_type: str = "default") -> dict[str, str] | None:
        """Get rate limit info as HTTP-style headers.

        Args:
            job_type: The job type to get headers for

        Returns:
            Dictionary with rate limit headers, or None if no rate limiter
        """
        info = self.check_rate_limit(job_type)
        if self.rate_limiter is None:
            return None
        return info.to_headers()

    def get_last_rate_limit_info(self) -> RateLimitInfo | None:
        """Get rate limit info from last dequeue operation.

        Returns:
            RateLimitInfo from last dequeue, or None if no dequeue yet
        """
        return self._last_rate_limit_info

    def update_rate_limiter(self, config: RateLimitConfig) -> None:
        """Update the rate limiter configuration.

        Args:
            config: New RateLimitConfig to use
        """
        with self._lock:
            self.rate_limiter = RateLimiter(config)
            self.rate_limit_behavior = config.default_behavior

    def reset_rate_limits(self, job_type: str | None = None) -> None:
        """Reset rate limiter state.

        Args:
            job_type: Specific job type to reset, or None for all
        """
        if self.rate_limiter is not None:
            self.rate_limiter.reset(job_type)

    def get_rate_limit_stats(self) -> dict[str, dict] | None:
        """Get rate limiter statistics.

        Returns:
            Dictionary with rate limit stats, or None if no rate limiter
        """
        if self.rate_limiter is None:
            return None
        return self.rate_limiter.get_stats()

    def acquire_job_lock(
        self,
        job_id: int,
        timeout: float | None = None,
        blocking: bool = True,
    ) -> bool:
        """Acquire a distributed lock for a specific job.

        This prevents multiple instances from processing the same job
        when running in a distributed setup.

        Args:
            job_id: The job ID to lock
            timeout: Maximum time to wait for lock (None = forever)
            blocking: If False, return immediately if lock unavailable

        Returns:
            True if lock was acquired, False otherwise

        Raises:
            JobQueueError: If no lock manager is configured
        """
        if self.lock_manager is None:
            raise JobQueueError(
                "No lock manager configured. "
                "Pass lock_manager to JobQueue constructor for distributed locking."
            )

        lock_name = f"job:{job_id}"
        return self.lock_manager.acquire_lock(
            lock_name=lock_name,
            timeout=timeout,
            blocking=blocking,
            lock_timeout=self.max_delay_seconds * 2,  # Lock for max retry duration
        )

    def release_job_lock(self, job_id: int) -> bool:
        """Release a distributed lock for a specific job.

        Args:
            job_id: The job ID to unlock

        Returns:
            True if lock was released, False if not held

        Raises:
            JobQueueError: If no lock manager is configured
        """
        if self.lock_manager is None:
            raise JobQueueError(
                "No lock manager configured. "
                "Pass lock_manager to JobQueue constructor for distributed locking."
            )

        lock_name = f"job:{job_id}"
        return self.lock_manager.release_lock(lock_name)

    def is_job_locked(self, job_id: int) -> bool:
        """Check if a job is currently locked by any instance.

        Args:
            job_id: The job ID to check

        Returns:
            True if job is locked, False otherwise

        Raises:
            JobQueueError: If no lock manager is configured
        """
        if self.lock_manager is None:
            raise JobQueueError(
                "No lock manager configured. "
                "Pass lock_manager to JobQueue constructor for distributed locking."
            )

        lock_name = f"job:{job_id}"
        return self.lock_manager.is_locked(lock_name)

    def process_next_distributed(
        self,
        handler: Callable[[Job], None],
        lock_timeout: float | None = None,
    ) -> bool:
        """Process the next available job with distributed locking.

        This method acquires a distributed lock before processing to ensure
        only one instance processes each job when running multiple Oclawma
        instances.

        Args:
            handler: Function to process the job
            lock_timeout: Maximum time to wait for job lock (None = do not wait)

        Returns:
            True if a job was processed, False if no jobs available

        Raises:
            JobQueueError: If no lock manager is configured
        """
        if self.lock_manager is None:
            raise JobQueueError(
                "No lock manager configured. "
                "Pass lock_manager to JobQueue constructor for distributed locking."
            )

        job = self.dequeue()
        if not job:
            return False

        # Try to acquire distributed lock for this job
        lock_acquired = self.acquire_job_lock(job.id, timeout=lock_timeout, blocking=False)

        if not lock_acquired:
            # Another instance has the lock - put job back in queue
            logger.debug(f"Job {job.id} is being processed by another instance, requeueing")
            # Requeue by resetting status to PENDING
            job.status = JobStatus.PENDING
            job.started_at = None
            job.updated_at = datetime.utcnow()
            self.store.update(job)
            return False

        try:
            handler(job)
            self.complete(job.id)
            return True
        except Exception as e:
            error_msg = str(e)
            self.fail(job.id, error_msg)
            self.retry(job.id, error=e)
            raise
        finally:
            # Always release the lock
            self.release_job_lock(job.id)

    def process_next_with_cancellation(
        self,
        handler: Callable[[Job, CancellationToken], None],
        timeout_config: TimeoutConfig | None = None,
        target_pid: int | None = None,
    ) -> bool:
        """Process the next available job with cancellation support.

        This method provides cooperative cancellation through a CancellationToken
        and optional hard timeout enforcement.

        Args:
            handler: Function to process the job, receives (job, token)
            timeout_config: Optional timeout configuration
            target_pid: Optional process ID to signal on hard timeout

        Returns:
            True if a job was processed, False if no jobs available

        Raises:
            CancellationError: If the job times out
            Exception: Any exception raised by the handler
        """
        job = self.dequeue()
        if not job:
            return False

        # Set up timeout manager if timeout specified in job or config
        effective_timeout = timeout_config
        if job.timeout_seconds is not None and effective_timeout is None:
            effective_timeout = TimeoutConfig(hard_timeout=job.timeout_seconds)

        timeout_manager = None
        token = CancellationToken()

        if effective_timeout:
            timeout_manager = TimeoutManager(effective_timeout)
            token = timeout_manager.start_monitoring(
                job_id=job.id,
                target_pid=target_pid,
            )

        exception: Exception | None = None

        def run_handler():
            nonlocal exception
            try:
                handler(job, token)
            except Exception as e:
                exception = e

        try:
            # Run handler in a separate thread
            thread = threading.Thread(target=run_handler)
            thread.start()

            # Wait for completion or timeout
            if effective_timeout and effective_timeout.hard_timeout:
                thread.join(timeout=effective_timeout.hard_timeout + 1)
            else:
                thread.join()

            if thread.is_alive():
                # Hard timeout - thread still running
                logger.error(f"Job {job.id} exceeded hard timeout")
                token.request_cancel(CancellationReason.TIMEOUT_HARD, "Hard timeout exceeded")
                self.fail(job.id, "Job exceeded hard timeout")
                raise CancellationError(
                    CancellationReason.TIMEOUT_HARD,
                    f"Job {job.id} exceeded hard timeout",
                )

            if exception:
                raise exception

            # Check if cancelled
            if token.is_cancelled():
                if token.reason == CancellationReason.TIMEOUT_SOFT:
                    self.fail(job.id, "Job exceeded soft timeout")
                raise CancellationError(
                    token.reason or CancellationReason.USER_REQUEST,
                    token.message,
                )

            self.complete(job.id)
            return True

        except CancellationError:
            # Job was cancelled
            if job.status == JobStatus.RUNNING:
                self.fail(job.id, f"Cancelled: {token.reason.value if token.reason else 'unknown'}")
            raise
        except Exception:
            # Other exception - fail and retry
            if job.status == JobStatus.RUNNING:
                error_msg = str(exception) if exception else "Unknown error"
                self.fail(job.id, error_msg)
                self.retry(job.id, error=exception)
            raise
        finally:
            if timeout_manager:
                timeout_manager.stop_monitoring()

    def get_cancellable_jobs(
        self,
        status: JobStatus | None = None,
        limit: int | None = None,
    ) -> list[Job]:
        """Get jobs that can be cancelled.

        Args:
            status: Filter by status (default: pending/running/paused)
            limit: Maximum number of jobs to return

        Returns:
            List of cancellable jobs
        """
        if status:
            jobs = self.store.list_all(status=status, limit=limit)
        else:
            # Get pending, running, and paused jobs
            jobs = []
            for s in [JobStatus.PENDING, JobStatus.RUNNING, JobStatus.PAUSED]:
                jobs.extend(self.store.list_all(status=s, limit=limit))
                if limit and len(jobs) >= limit:
                    jobs = jobs[:limit]
                    break

        return jobs

    # =============================================================================
    # Tenant-aware Methods
    # =============================================================================

    def list_jobs_by_tenant(
        self,
        tenant_id: str,
        status: JobStatus | None = None,
        limit: int | None = None,
        decrypt: bool = True,
    ) -> list[Job]:
        """List jobs for a specific tenant.

        Args:
            tenant_id: The tenant identifier to filter by
            status: Filter by status
            limit: Max jobs to return
            decrypt: Whether to decrypt encrypted payloads (default: True)

        Returns:
            List of jobs belonging to the tenant
        """
        jobs = self.store.list_by_tenant(tenant_id, status=status, limit=limit)
        if decrypt:
            return [self._decrypt_job_if_needed(job) if job.encrypted else job for job in jobs]
        return jobs

    def get_stats_by_tenant(self, tenant_id: str) -> JobStats:
        """Get queue statistics for a specific tenant.

        Args:
            tenant_id: The tenant identifier to filter by

        Returns:
            JobStats with tenant-specific queue metrics
        """
        stats = self.store.get_stats_by_tenant(tenant_id)
        return JobStats(
            total_jobs=stats["total"],
            pending=stats["pending"],
            running=stats["running"],
            completed=stats["completed"],
            failed=stats["failed"],
            retryable=stats["retryable"],
        )

    def queue_depth_by_tenant(self, tenant_id: str) -> int:
        """Get queue depth (pending jobs) for a specific tenant.

        Args:
            tenant_id: The tenant identifier to filter by

        Returns:
            Number of pending jobs for the tenant
        """
        return self.get_stats_by_tenant(tenant_id).queue_depth

    def dequeue_by_tenant(
        self,
        tenant_id: str,
        block: bool = False,
        timeout: float | None = None,
    ) -> Job | None:
        """Get the next job for a specific tenant.

        This method filters jobs by tenant_id and returns only jobs
        belonging to the specified tenant.

        Args:
            tenant_id: The tenant identifier to filter by
            block: If True, block until a job is available
            timeout: Max seconds to block (None = forever)

        Returns:
            Next job for the tenant, or None if no jobs available
        """
        result = self.dequeue_by_tenant_with_rate_limit(tenant_id, block=block, timeout=timeout)
        if result and result[0]:
            return self._decrypt_job_if_needed(result[0])
        return None

    def dequeue_by_tenant_with_rate_limit(
        self,
        tenant_id: str,
        block: bool = False,
        timeout: float | None = None,
        job_type: str | None = None,
    ) -> tuple[Job | None, RateLimitInfo | None]:
        """Get the next job for a specific tenant with rate limiting.

        Args:
            tenant_id: The tenant identifier to filter by
            block: If True, block until a job is available
            timeout: Max seconds to block (None = forever)
            job_type: Job type for rate limiting

        Returns:
            Tuple of (Job | None, RateLimitInfo | None)
        """
        start_time = time.time()

        while True:
            with self._lock:
                # Get next pending job for this tenant
                job = self.store.get_next_pending_by_tenant(
                    tenant_id, exclude_ids=self._running_jobs
                )

                if job:
                    # Check rate limit if configured
                    if self.rate_limiter is not None:
                        actual_job_type = job_type or job.job_type
                        rate_info = self.rate_limiter.try_acquire(actual_job_type)
                        self._last_rate_limit_info = rate_info

                        if not rate_info.allowed:
                            behavior = self.rate_limit_behavior

                            if behavior == "drop":
                                job.status = JobStatus.FAILED
                                job.error = (
                                    f"Rate limited: retry after {rate_info.retry_after:.2f}s"
                                )
                                job.completed_at = datetime.utcnow()
                                job.updated_at = datetime.utcnow()
                                self.store.update(job)
                                return None, rate_info

                            elif behavior == "defer":
                                retry_after = max(rate_info.retry_after, 1.0)
                                job.scheduled_at = datetime.utcnow() + timedelta(
                                    seconds=retry_after
                                )
                                job.updated_at = datetime.utcnow()
                                self.store.update(job)
                                return None, rate_info

                            else:  # block
                                if not block:
                                    return None, rate_info

                                if timeout is not None:
                                    elapsed = time.time() - start_time
                                    if elapsed >= timeout:
                                        return None, rate_info
                        else:
                            job.status = JobStatus.RUNNING
                            job.started_at = datetime.utcnow()
                            job.updated_at = datetime.utcnow()
                            self.store.update(job)
                            self._running_jobs.add(job.id)
                            return job, rate_info
                    else:
                        job.status = JobStatus.RUNNING
                        job.started_at = datetime.utcnow()
                        job.updated_at = datetime.utcnow()
                        self.store.update(job)
                        self._running_jobs.add(job.id)
                        return job, None

            if not block:
                return None, None

            if timeout is not None and (time.time() - start_time) >= timeout:
                return None, None

            time.sleep(0.1)

    def cleanup_old_jobs_by_tenant(
        self,
        tenant_id: str,
        max_age_days: int,
        status: JobStatus | None = None,
    ) -> int:
        """Delete old jobs for a specific tenant.

        Args:
            tenant_id: The tenant identifier
            max_age_days: Delete jobs older than this many days
            status: Only delete jobs with this status (optional)

        Returns:
            Number of jobs deleted
        """
        return self.store.cleanup_old_jobs_by_tenant(tenant_id, max_age_days, status)

    def get_tenants(self) -> list[str]:
        """Get all tenant IDs that have jobs in the queue.

        Returns:
            List of tenant IDs
        """
        return self.store.list_all_tenants()

    def close(self) -> None:
        """Close the queue and release resources."""
        self.store.close()
        if self.result_cache is not None:
            self.result_cache.close()

    def get_cache_key(self, job: Job, key_fields: set[str] | None = None) -> CacheKey:
        """Generate a cache key for a job.

        Args:
            job: The job to generate key for
            key_fields: Specific payload fields to include (None = all)

        Returns:
            CacheKey for the job
        """
        # Use job's key_fields if not specified and cache config has them
        if key_fields is None and self.result_cache is not None:
            key_fields = self.result_cache.config.key_fields

        return CacheKey.from_job(
            job_type=job.job_type,
            payload=job.payload,
            key_fields=key_fields,
        )

    def check_cache(self, job: Job) -> Any | None:
        """Check if a cached result exists for a job.

        Args:
            job: Job to check cache for

        Returns:
            Cached result or None if not found/expired
        """
        if self.result_cache is None:
            return None

        cache_key = self.get_cache_key(job)
        return self.result_cache.get(cache_key)

    def cache_result(self, job: Job, result: Any, ttl: int | None = None) -> None:
        """Cache the result of a job execution.

        Args:
            job: The job that was executed
            result: The result to cache
            ttl: Optional TTL override
        """
        if self.result_cache is None:
            return

        cache_key = self.get_cache_key(job)
        try:
            self.result_cache.set(cache_key, result, ttl=ttl)
        except Exception as e:
            logger.warning(f"Failed to cache result for job {job.id}: {e}")

    def process_next_cached(
        self,
        handler: Callable[[Job], Any],
        use_cache: bool = True,
        cache_ttl: int | None = None,
    ) -> tuple[bool, Any]:
        """Process the next job with caching support.

        This is a convenience method that checks the cache before executing,
        and caches the result after successful execution.

        Args:
            handler: Function to process the job, should return result
            use_cache: Whether to use caching (default: True)
            cache_ttl: Optional TTL for caching result

        Returns:
            Tuple of (was_processed, result_or_cached_value)
        """
        job = self.dequeue()
        if not job:
            return False, None

        # Check cache first
        if use_cache and self.result_cache is not None:
            cached = self.check_cache(job)
            if cached is not None:
                logger.debug(f"Cache hit for job {job.id}, skipping execution")
                # Complete the job without running handler
                self.complete(job.id)
                return True, cached

        try:
            result = handler(job)
            self.complete(job.id)

            # Cache the result
            if use_cache:
                self.cache_result(job, result, ttl=cache_ttl)

            return True, result
        except Exception as e:
            error_msg = str(e)
            self.fail(job.id, error_msg)
            self.retry(job.id, error=e)
            raise

    def get_cache_stats(self) -> dict[str, Any] | None:
        """Get cache statistics if caching is enabled.

        Returns:
            Cache stats dict or None if caching not enabled
        """
        if self.result_cache is None:
            return None

        stats = self.result_cache.get_stats()
        return {
            "total_entries": stats.total_entries,
            "expired_entries": stats.expired_entries,
            "hits": stats.hits,
            "misses": stats.misses,
            "hit_rate": stats.hit_rate,
            "size_bytes": stats.size_bytes,
        }

    def clear_cache(self) -> int:
        """Clear all cached results.

        Returns:
            Number of entries cleared
        """
        if self.result_cache is None:
            return 0

        return self.result_cache.clear()

    def invalidate_cache(self, pattern: str | None = None) -> int:
        """Invalidate cache entries.

        Args:
            pattern: Optional pattern to match (None = clear all)

        Returns:
            Number of entries invalidated
        """
        if self.result_cache is None:
            return 0

        if pattern is None:
            return self.result_cache.clear()

        return self.result_cache.invalidate_pattern(pattern)

    def __enter__(self) -> JobQueue:
        """Context manager entry."""
        return self

    def __exit__(self, *args: Any) -> None:
        """Context manager exit."""
        self.close()

    def __len__(self) -> int:
        """Return total number of jobs in queue."""
        return self.store.get_stats()["total"]
