"""
MUXI Scheduler Service

Main orchestration service for proactive task scheduling in MUXI formations.
Enables users to schedule recurring AI tasks through natural language requests.

Key Features:
- Map/Reduce job selection pattern (no next_run_at calculations)
- Session-based execution with f"job_{job.id}" as session_id
- Formation integration leveraging existing RequestTracker and webhooks
- Multi-user support with secure execution isolation
- Dynamic exclusion rules generated by LLM
- Auto-pause after consecutive failures
- Timezone-aware cron scheduling with DST handling

Architecture:
- Background worker using @multitasking.task decorator
- Direct integration with Overlord for job execution via overlord.chat()
- Database operations through JobManager with map/reduce pattern
- Natural language parsing through ScheduleParser and PromptRewriter
"""

import asyncio
import signal
import time
import traceback
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional

import multitasking
import pytz
from croniter import croniter

from ...datatypes.observability import InitEventFormatter
from ...utils.datetime_utils import utc_now
from .. import observability
from ..db import get_database_manager
from .batch_processor import JobBatchProcessor
from .cache import SchedulerCache
from .circuit_breaker import LLMCircuitBreaker
from .manager import JobManager
from .parser import ScheduleParser
from .rewriter import PromptRewriter

# Configure multitasking
multitasking.set_engine("thread")
# Only register signal handlers in main thread to avoid errors in tests
try:
    signal.signal(signal.SIGINT, multitasking.killall)
except ValueError:
    # Signal handlers can only be registered in main thread
    # This is expected in tests or when imported from threads
    pass


class SchedulerService:
    """
    Scheduler service for managing recurring AI tasks in MUXI formations.

    Provides proactive task scheduling capabilities with natural language
    configuration and formation-aware execution using existing MUXI infrastructure.
    """

    _instance: Optional["SchedulerService"] = None
    _lock = asyncio.Lock()

    @classmethod
    async def get_instance(cls, overlord=None) -> "SchedulerService":
        """
        Get singleton instance of SchedulerService.

        Args:
            overlord: Optional overlord instance for dependency injection

        Returns:
            SchedulerService instance
        """
        if cls._instance is None:
            async with cls._lock:
                if cls._instance is None:
                    cls._instance = cls(overlord)
                    await cls._instance._initialize()
        return cls._instance

    def __init__(self, overlord=None):
        """
        Initialize scheduler service.

        Args:
            overlord: Overlord instance for formation integration
        """
        self.overlord = overlord
        self._running = False
        self._worker_task = None
        self._last_execution_time = None

        # Configuration
        self._config = self._get_scheduler_config()
        self.check_interval_minutes = self._config.get("check_interval_minutes", 1)
        self.max_concurrent_jobs = self._config.get("max_concurrent_jobs", 10)
        self.max_failures_before_pause = self._config.get("max_failures_before_pause", 3)
        self.formation_timezone = self._config.get("timezone", "UTC")

        # Services - use unified database manager from overlord
        if hasattr(overlord, "db_manager") and overlord.db_manager:
            self.db_manager = overlord.db_manager
        else:
            # Fallback to creating own database manager
            self.db_manager = get_database_manager()
        # Get formation_id from overlord
        formation_id = (
            overlord.formation_id
            if overlord and hasattr(overlord, "formation_id")
            else "default-formation"
        )

        self.job_manager = JobManager(self.db_manager, formation_id=formation_id)

        # Performance optimization components
        scheduler_config = None
        if overlord and hasattr(overlord, "_configured_services"):
            scheduler_config = overlord._configured_services.get("scheduler_config")

        self.batch_processor = JobBatchProcessor(self.job_manager, config=scheduler_config)
        self.cache = SchedulerCache(
            cache_ttl=self._config.get("cache_ttl", 300),
            max_cache_size=self._config.get("max_cache_size", 1000),
        )
        self.llm_circuit_breaker = LLMCircuitBreaker(
            failure_threshold=self._config.get("llm_failure_threshold", 5),
            timeout=self._config.get("llm_circuit_timeout", 60.0),
        )

        # Initialize parser and rewriter with performance components.
        # Pass the overlord's extraction model so they use the formation's
        # configured LLM (with proper API key) instead of a bare LLM() default.
        formation_llm = None
        if overlord and hasattr(overlord, "extraction_model") and overlord.extraction_model:
            extraction = overlord.extraction_model
            if isinstance(extraction, str):
                # extraction_model is a model name string, create LLM from it
                from ..llm import LLM as LLMClass

                text_config = (
                    overlord._capability_models.get("text", {})
                    if hasattr(overlord, "_capability_models")
                    else {}
                )
                api_key = text_config.get("api_key") if isinstance(text_config, dict) else None
                try:
                    formation_llm = LLMClass(model=extraction, api_key=api_key)
                except Exception:
                    pass
            else:
                formation_llm = extraction

        self.schedule_parser = ScheduleParser(
            cache=self.cache, circuit_breaker=self.llm_circuit_breaker
        )
        if formation_llm:
            self.schedule_parser.llm = formation_llm

        self.prompt_rewriter = PromptRewriter()
        if formation_llm:
            self.prompt_rewriter.llm = formation_llm

        # State tracking
        self._active_executions = set()
        self._performance_stats = {
            "cycles_completed": 0,
            "jobs_processed": 0,
            "llm_calls_saved": 0,
            "batch_processing_time": 0.0,
        }

        # Initialization event will be emitted in _initialize() after components are ready

    async def _initialize(self):
        """Initialize service components."""
        await self.job_manager.initialize()

        # Emit single formatted initialization event
        details = (
            f"checks every {self.check_interval_minutes}m, "
            f"up to {self.max_concurrent_jobs} concurrent jobs, "
            f"{self.formation_timezone}"
        )
        print(InitEventFormatter.format_ok("Background scheduler initialized", details))

        pass  # REMOVED: init-phase observe() call

    def _get_scheduler_config(self) -> Dict[str, Any]:
        """
        Get scheduler configuration from formation.

        Returns:
            Scheduler configuration dict
        """
        if self.overlord and hasattr(self.overlord, "formation_config"):
            return self.overlord.formation_config.get("scheduler", {})
        return {}

    async def start(self) -> Dict[str, Any]:
        """
        Start the scheduler service.

        Returns:
            Service status dict
        """
        if self._running:
            return {"status": "already_running", "service": "SchedulerService"}

        # Check if scheduler is enabled in formation config
        if not self._config.get("enabled", True):
            return {"status": "disabled", "service": "SchedulerService"}

        self._running = True

        # Start background worker in a daemon thread so it doesn't block the event loop
        import threading

        self._worker_thread = threading.Thread(
            target=self.process_due_jobs_continuously, daemon=True
        )
        self._worker_thread.start()

        # count_active_jobs uses a sync psycopg2 session which blocks the event
        # loop. Run it in a thread executor with a timeout so a slow or failing
        # database connection doesn't prevent the formation from starting.
        active_jobs_count = 0
        try:
            active_jobs_count = await asyncio.wait_for(
                asyncio.get_event_loop().run_in_executor(
                    None, self.job_manager.count_active_jobs_sync
                ),
                timeout=10,
            )
        except Exception:
            pass  # Non-critical -- default to 0

        return {
            "status": "started",
            "service": "SchedulerService",
            "active_jobs": active_jobs_count,
            "check_interval_minutes": self.check_interval_minutes,
        }

    async def stop(self) -> Dict[str, Any]:
        """
        Stop the scheduler service.

        Returns:
            Service status dict
        """
        if not self._running:
            return {"status": "already_stopped", "service": "SchedulerService"}

        self._running = False

        # Wait for active executions to complete (with timeout)
        timeout = 30  # seconds
        start_time = time.time()
        while self._active_executions and (time.time() - start_time) < timeout:
            await asyncio.sleep(1)

        return {
            "status": "stopped",
            "service": "SchedulerService",
            "active_executions_remaining": len(self._active_executions),
        }

    async def get_status(self) -> Dict[str, Any]:
        """
        Get scheduler service status.

        Returns:
            Service status dict
        """
        active_jobs = await self.job_manager.count_active_jobs()

        return {
            "running": self._running,
            "enabled": self._config.get("enabled", True),
            "jobs_active": active_jobs,
            "active_executions": len(self._active_executions),
            "last_execution": self._last_execution_time,
            "check_interval_minutes": self.check_interval_minutes,
            "max_concurrent_jobs": self.max_concurrent_jobs,
            "formation_timezone": self.formation_timezone,
        }

    @multitasking.task
    def process_due_jobs_continuously(self):
        """
        Background task for continuous job processing using map/reduce pattern.

        Runs continuously checking for due jobs and executing them through
        the formation's overlord. Uses map/reduce pattern without next_run_at calculations.
        """
        # Create event loop for this thread if needed
        try:
            loop = asyncio.get_event_loop()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

        while self._running:
            try:
                # Run the async processing cycle
                loop.run_until_complete(self._process_jobs_cycle())

            except Exception as e:
                observability.observe(
                    event_type=observability.ErrorEvents.WARNING,
                    level=observability.EventLevel.WARNING,
                    data={"error": str(e), "error_type": type(e).__name__},
                    description=f"Scheduler worker error: {e}",
                )

            # Sleep for check interval (convert minutes to seconds)
            time.sleep(self.check_interval_minutes * 60)

    async def _process_jobs_cycle(self):
        """
        Process one cycle of job checking and execution.
        This method contains all the async logic.
        """
        try:
            # Get current time in formation timezone
            formation_tz = pytz.timezone(self.formation_timezone)
            current_time = datetime.now(formation_tz)

            # MAP/REDUCE: Get jobs due for execution
            due_jobs = await self.get_due_jobs_map_reduce(current_time)
        except Exception as e:
            observability.observe(
                event_type=observability.ErrorEvents.INTERNAL_ERROR,
                level=observability.EventLevel.ERROR,
                data={
                    "error": str(e),
                    "error_type": type(e).__name__,
                    "traceback": traceback.format_exc(),
                },
                description=f"Error in scheduler cycle: {e}",
            )
            due_jobs = []

        if due_jobs:
            observability.observe(
                event_type=observability.ConversationEvents.SCHEDULED_JOBS_FOUND,
                level=observability.EventLevel.INFO,
                data={"jobs_count": len(due_jobs)},
                description=f"Found {len(due_jobs)} jobs due for execution",
            )

        # Execute due jobs (respecting concurrency limits)
        await self._execute_due_jobs(due_jobs)

        self._last_execution_time = utc_now().isoformat()

        # Update performance stats
        self._performance_stats["cycles_completed"] += 1

    async def get_due_jobs_map_reduce(self, current_time: datetime) -> List[Dict[str, Any]]:
        """
        Get jobs due for execution using map/reduce pattern with batch processing.

        MAP: Fetch active jobs in batches
        REDUCE: Filter jobs that match current time and don't match exclusion rules

        Args:
            current_time: Current time in formation timezone

        Returns:
            List of jobs due for execution
        """
        start_time = time.time()

        # Define helper functions for batch processing
        async def is_job_due(job: Dict[str, Any], current_time: datetime) -> bool:
            """Check if a job is due for execution."""
            if job.get("is_recurring", True):
                # Handle recurring jobs
                if job.get("cron_expression"):
                    return await self._is_recurring_job_due(job, current_time)
            else:
                # Handle one-time jobs
                return await self._is_onetime_job_due(job, current_time)
            return False

        # Use batch processor to get due jobs efficiently
        due_jobs = await self.batch_processor.process_due_jobs(
            current_time=current_time,
            is_job_due_func=is_job_due,
            check_exclusion_func=self._check_exclusion_rules,
        )

        # Update performance stats
        batch_time = time.time() - start_time
        self._performance_stats["batch_processing_time"] = batch_time
        self._performance_stats["jobs_processed"] += len(due_jobs)

        return due_jobs

    async def _is_recurring_job_due(self, job: Dict[str, Any], current_time: datetime) -> bool:
        """
        Check if a recurring job is due for execution.

        Args:
            job: Job data dict
            current_time: Current time in formation timezone

        Returns:
            True if job should execute, False otherwise
        """
        try:
            # Check if job matches cron schedule
            cron = croniter(job["cron_expression"], current_time)

            # Get the most recent time this job should have run
            prev_time = cron.get_prev(datetime)

            # Check if job should run now (within our check interval window)
            time_diff = (current_time - prev_time).total_seconds()
            window_seconds = self.check_interval_minutes * 60

            if time_diff <= window_seconds:
                # Check if job has already run recently
                return await self._should_execute_job(job, prev_time)

            return False
        except Exception:
            return False

    async def _is_onetime_job_due(self, job: Dict[str, Any], current_time: datetime) -> bool:
        """
        Check if a one-time job is due for execution.

        Args:
            job: Job data dict
            current_time: Current time in formation timezone

        Returns:
            True if job should execute, False otherwise
        """
        scheduled_for_str = job.get("scheduled_for")
        if not scheduled_for_str:
            return False

        try:
            # Parse the scheduled datetime (stored in UTC)
            scheduled_for_utc = datetime.fromisoformat(scheduled_for_str.replace("Z", "+00:00"))

            # Convert current time to UTC for comparison
            current_time_utc = current_time.astimezone(pytz.UTC)

            # Check if the scheduled time has passed and we're within the execution window
            time_diff = (current_time_utc - scheduled_for_utc).total_seconds()
            window_seconds = self.check_interval_minutes * 60

            # Job is due if:
            # 1. The scheduled time has passed (time_diff >= 0)
            # 2. We're within the execution window (time_diff <= window_seconds)
            # 3. The job hasn't been executed yet
            if 0 <= time_diff <= window_seconds:
                # Check if job has already been executed
                return not job.get("last_run_at")  # One-time jobs should only run once

            return False
        except Exception:
            return False

    async def _should_execute_job(self, job: Dict[str, Any], scheduled_time: datetime) -> bool:
        """
        Check if job should execute based on last run time.

        Args:
            job: Job data dict
            scheduled_time: Time job was scheduled to run

        Returns:
            True if job should execute, False otherwise
        """
        if not job.get("last_run_at"):
            return True  # Never run before

        last_run = datetime.fromisoformat(job["last_run_at"].replace("Z", "+00:00"))

        # Don't run if we already executed this scheduled time
        return scheduled_time > last_run

    async def _check_exclusion_rules(self, job: Dict[str, Any], current_time: datetime) -> bool:
        """
        Check if current time matches any exclusion rules.

        Args:
            job: Job data dict
            current_time: Current time to check

        Returns:
            True if excluded (should not run), False if allowed
        """
        exclusion_rules = job.get("exclusion_rules", [])

        if not exclusion_rules:
            return False  # No exclusions

        # Check each exclusion rule
        for rule in exclusion_rules:
            try:
                if rule.get("type") == "cron":
                    # Check if current time matches exclusion cron pattern
                    exclusion_cron = croniter(rule["pattern"], current_time)
                    prev_exclusion = exclusion_cron.get_prev(datetime)

                    # If exclusion time is very recent, exclude this execution
                    if (current_time - prev_exclusion).total_seconds() <= 60:
                        observability.observe(
                            event_type=observability.ConversationEvents.SCHEDULED_JOB_EXCLUDED,
                            level=observability.EventLevel.INFO,
                            data={
                                "job_id": job["id"],
                                "exclusion_rule": rule["pattern"],
                                "reason": rule.get("description", "No description"),
                            },
                            description=f"Job {job['id']} excluded by rule: {rule.get('description')}",
                        )
                        return True

                elif rule.get("type") == "complex_date":
                    # Check complex date patterns
                    if self._check_complex_date_exclusion(rule["pattern"], current_time):
                        observability.observe(
                            event_type=observability.ConversationEvents.SCHEDULED_JOB_EXCLUDED,
                            level=observability.EventLevel.INFO,
                            data={
                                "job_id": job["id"],
                                "exclusion_rule": rule["pattern"],
                                "reason": rule.get("description", "No description"),
                            },
                            description=f"Job {job['id']} excluded by complex date rule: {rule.get('description')}",
                        )
                        return True

            except Exception as e:
                observability.observe(
                    event_type=observability.ErrorEvents.WARNING,
                    level=observability.EventLevel.WARNING,
                    data={"job_id": job["id"], "exclusion_rule": rule, "error": str(e)},
                    description=f"Failed to evaluate exclusion rule for job {job['id']}: {e}",
                )

        return False  # Not excluded

    def _check_complex_date_exclusion(self, pattern: str, current_time: datetime) -> bool:
        """
        Check if current time matches a complex date pattern.

        Args:
            pattern: Complex date pattern (e.g., "last_friday_of_month")
            current_time: Current time to check

        Returns:
            True if current date matches the pattern (should exclude)
        """
        try:
            # Get current date info in formation timezone
            tz = pytz.timezone(self.formation_timezone)
            local_time = current_time.astimezone(tz)

            pattern_lower = pattern.lower()

            # Parse different pattern types
            if pattern_lower.startswith("first_") and pattern_lower.endswith("_of_month"):
                # first_monday_of_month, first_friday_of_month, etc.
                weekday_name = pattern_lower[6:-9]  # Extract weekday name
                return self._is_nth_weekday_of_month(local_time, 1, weekday_name)

            elif pattern_lower.startswith("last_") and pattern_lower.endswith("_of_month"):
                # last_friday_of_month, last_monday_of_month, etc.
                weekday_name = pattern_lower[5:-9]  # Extract weekday name
                return self._is_last_weekday_of_month(local_time, weekday_name)

            elif pattern_lower.startswith("nth_weekday:"):
                # nth_weekday:3:tuesday
                parts = pattern_lower.split(":")
                if len(parts) == 3:
                    n = int(parts[1])
                    weekday_name = parts[2]
                    return self._is_nth_weekday_of_month(local_time, n, weekday_name)

            elif pattern_lower.startswith("nth_day:"):
                # nth_day:15 (15th day of month)
                day = int(pattern_lower[8:])
                return local_time.day == day

            elif pattern_lower.startswith("last_day_minus:"):
                # last_day_minus:2 (2 days before end of month)
                days_before = int(pattern_lower[15:])
                return self._is_n_days_before_month_end(local_time, days_before)

        except Exception as e:
            observability.observe(
                event_type=observability.ErrorEvents.WARNING,
                level=observability.EventLevel.WARNING,
                data={"pattern": pattern, "error": str(e)},
                description=f"Failed to evaluate complex date pattern: {e}",
            )

        return False

    def _is_nth_weekday_of_month(self, dt: datetime, n: int, weekday_name: str) -> bool:
        """Check if date is the Nth occurrence of a weekday in the month."""
        weekday_map = {
            "monday": 0,
            "tuesday": 1,
            "wednesday": 2,
            "thursday": 3,
            "friday": 4,
            "saturday": 5,
            "sunday": 6,
        }

        target_weekday = weekday_map.get(weekday_name.lower())
        if target_weekday is None:
            return False

        # Check if current date is the target weekday
        if dt.weekday() != target_weekday:
            return False

        # Calculate which occurrence this is
        occurrence = (dt.day - 1) // 7 + 1
        return occurrence == n

    def _is_last_weekday_of_month(self, dt: datetime, weekday_name: str) -> bool:
        """Check if date is the last occurrence of a weekday in the month."""
        weekday_map = {
            "monday": 0,
            "tuesday": 1,
            "wednesday": 2,
            "thursday": 3,
            "friday": 4,
            "saturday": 5,
            "sunday": 6,
        }

        target_weekday = weekday_map.get(weekday_name.lower())
        if target_weekday is None:
            return False

        # Check if current date is the target weekday
        if dt.weekday() != target_weekday:
            return False

        # Check if adding 7 days would be in the next month
        next_week = dt + timedelta(days=7)
        return next_week.month != dt.month

    def _is_n_days_before_month_end(self, dt: datetime, n: int) -> bool:
        """Check if date is N days before the end of the month."""
        # Get last day of current month
        if dt.month == 12:
            last_day = datetime(dt.year + 1, 1, 1, tzinfo=dt.tzinfo) - timedelta(days=1)
        else:
            last_day = datetime(dt.year, dt.month + 1, 1, tzinfo=dt.tzinfo) - timedelta(days=1)

        # Check if current date is N days before last day
        target_date = last_day - timedelta(days=n)
        return dt.date() == target_date.date()

    async def _execute_due_jobs(self, due_jobs: List[Dict[str, Any]]):
        """
        Execute due jobs respecting concurrency limits.

        Args:
            due_jobs: List of jobs to execute
        """
        for job in due_jobs:
            # Check concurrency limit
            if len(self._active_executions) >= self.max_concurrent_jobs:
                break

            # Execute job asynchronously
            asyncio.create_task(self._execute_single_job(job))

    async def _execute_single_job(self, job: Dict[str, Any]):
        """
        Execute a single scheduled job.

        Args:
            job: Job data dict
        """
        job_id = job["id"]
        session_id = f"job_{job_id}"
        self._active_executions.add(session_id)  # Track by session_id, not job_id

        try:
            # Update job execution tracking (start)
            await self.job_manager.mark_job_execution_start(job_id)

            # Execute job through overlord with session_id
            execution_prompt = job["execution_prompt"]

            observability.observe(
                event_type=observability.ConversationEvents.SCHEDULED_JOB_STARTED,
                level=observability.EventLevel.INFO,
                data={
                    "job_id": job_id,
                    "user_id": job["user_id"],
                    "session_id": session_id,
                    "title": job["title"],
                },
                description=f"Starting execution of scheduled job: {job['title']}",
            )

            # Record scheduled task feature usage in telemetry
            from ..telemetry import get_telemetry

            telemetry = get_telemetry()
            if telemetry:
                telemetry.record_feature("scheduled_task")

            # Execute through overlord
            if self.overlord:
                # Get webhook URL from formation configuration
                webhook_url = None
                if hasattr(self.overlord, "formation_config"):
                    webhook_url = self.overlord.formation_config.get("async", {}).get("webhook_url")

                # Fallback to overlord's attribute if not in config
                if not webhook_url:
                    webhook_url = getattr(self.overlord, "async_webhook_url", None)

                response = await self.overlord.chat(
                    message=execution_prompt,
                    user_id=job["user_id"],
                    session_id=session_id,
                    use_async=True,  # CRITICAL: Must be async since user is not waiting
                    webhook_url=webhook_url,  # Required for async execution
                    stream=False,  # No streaming needed for scheduled jobs
                )

                # Log that async execution has been initiated
                observability.observe(
                    event_type=observability.ConversationEvents.SCHEDULED_JOB_ASYNC_INITIATED,
                    level=observability.EventLevel.INFO,
                    data={
                        "job_id": job_id,
                        "session_id": session_id,
                        "response_id": (
                            response.id if hasattr(response, "id") else str(response)[:50]
                        ),
                    },
                    description=f"Async execution initiated for job: {job['title']}",
                )

                # DO NOT mark as complete here - wait for webhook
                # The completion will be handled by complete_job_from_webhook
            else:
                raise Exception("No overlord available for job execution")

        except Exception as e:
            # Immediate failure - never made it to async
            self._active_executions.discard(session_id)
            # Mark job as failed
            await self.job_manager.mark_job_execution_failure(job_id, str(e))

            # Check if job should be auto-paused
            consecutive_failures = await self.job_manager.get_consecutive_failures(job_id)
            if consecutive_failures >= self.max_failures_before_pause:
                await self.job_manager.pause_job(job_id)

                observability.observe(
                    event_type=observability.ConversationEvents.SCHEDULED_JOB_PAUSED,
                    level=observability.EventLevel.WARNING,
                    data={
                        "job_id": job_id,
                        "consecutive_failures": consecutive_failures,
                        "max_failures": self.max_failures_before_pause,
                    },
                    description=f"Job {job_id} auto-paused after {consecutive_failures} consecutive failures",
                )

            observability.observe(
                event_type=observability.ConversationEvents.SCHEDULED_JOB_FAILED,
                level=observability.EventLevel.ERROR,
                data={
                    "job_id": job_id,
                    "user_id": job["user_id"],
                    "error": str(e),
                    "error_type": type(e).__name__,
                    "consecutive_failures": consecutive_failures,
                },
                description=f"Scheduled job failed: {job['title']} - {e}",
            )

        finally:
            # Don't discard from active_executions here - wait for webhook
            pass

    # API Methods for job management

    async def create_job(
        self,
        user_id: str,
        title: str,
        original_prompt: str,
        schedule: str,
        exclusions: List[str] = None,
    ) -> str:
        """
        Create a new scheduled job.

        Args:
            user_id: User ID who owns the job
            title: Human-readable job title
            original_prompt: Original natural language request
            schedule: Natural language schedule description
            exclusions: Optional list of exclusion descriptions

        Returns:
            Job ID of created job
        """
        # Parse schedule (returns either cron expression or dict for one-off job)
        parse_result = await self.schedule_parser.parse_schedule(schedule, self.formation_timezone)

        # Check if it's a one-off job (dict) or recurring (string cron expression)
        cron_expression = None
        scheduled_for = None

        if isinstance(parse_result, dict):
            # One-off job - extract the scheduled datetime
            if parse_result.get("job_type") == "one_time":
                scheduled_for = parse_result.get("scheduled_for")
                # For one-off jobs, we don't use a cron expression
                cron_expression = None
            else:
                # Unexpected dict format
                raise ValueError(f"Unexpected parse result format: {parse_result}")
        else:
            # Recurring job - parse_result is the cron expression
            cron_expression = parse_result

        # Generate dynamic exclusion rules
        exclusion_rules = []
        if exclusions:
            exclusion_rules = await self.schedule_parser.generate_exclusion_rules(exclusions)

        # Rewrite prompt for execution
        execution_prompt = await self.prompt_rewriter.rewrite_for_execution(original_prompt)

        # Create job (handles both one-off and recurring)
        if scheduled_for:
            # Create one-off job
            job_id = await self.job_manager.create_job(
                user_id=user_id,
                title=title,
                original_prompt=original_prompt,
                execution_prompt=execution_prompt,
                cron_expression=None,
                scheduled_for=scheduled_for,
                is_recurring=False,
                exclusion_rules=exclusion_rules,
            )
        else:
            # Create recurring job
            job_id = await self.job_manager.create_job(
                user_id=user_id,
                title=title,
                original_prompt=original_prompt,
                execution_prompt=execution_prompt,
                cron_expression=cron_expression,
                scheduled_for=None,
                is_recurring=True,
                exclusion_rules=exclusion_rules,
            )

        observability.observe(
            event_type=observability.ConversationEvents.SCHEDULED_JOB_CREATED,
            level=observability.EventLevel.INFO,
            data={
                "job_id": job_id,
                "user_id": user_id,
                "title": title,
                "cron_expression": cron_expression,
                "scheduled_for": scheduled_for.isoformat() if scheduled_for else None,
                "is_recurring": cron_expression is not None,
            },
            description=f"Scheduled job created: {title}",
        )

        return job_id

    async def complete_job_from_webhook(
        self, session_id: str, success: bool, result: str = None, error: str = None
    ) -> bool:
        """
        Called by webhook handler to complete a job execution.

        Args:
            session_id: Session ID (format: "job_{job_id}")
            success: Whether the execution was successful
            result: Result text if successful
            error: Error message if failed

        Returns:
            bool: True if this was a scheduled job, False otherwise
        """
        if session_id not in self._active_executions:
            return False

        # Extract job_id from session_id (format: "job_{job_id}")
        if not session_id.startswith("job_"):
            return False

        job_id = session_id[4:]  # Remove "job_" prefix
        self._active_executions.discard(session_id)

        # Get job details for logging
        job = await self.job_manager.get_job(job_id)
        if not job:
            return False

        if success:
            await self.job_manager.mark_job_execution_success(
                job_id, result[:1000] if result else ""
            )

            # For one-time jobs, mark as completed
            if not job.get("is_recurring", True):
                await self.job_manager.complete_onetime_job(job_id)

                observability.observe(
                    event_type=observability.ConversationEvents.ONETIME_JOB_COMPLETED,
                    level=observability.EventLevel.INFO,
                    data={
                        "job_id": job_id,
                        "session_id": session_id,
                        "result_length": len(result) if result else 0,
                    },
                    description=f"One-time job completed via webhook: {job['title']}",
                )
            else:
                observability.observe(
                    event_type=observability.ConversationEvents.SCHEDULED_JOB_COMPLETED,
                    level=observability.EventLevel.INFO,
                    data={
                        "job_id": job_id,
                        "session_id": session_id,
                        "result_length": len(result) if result else 0,
                    },
                    description=f"Recurring job completed via webhook: {job['title']}",
                )
        else:
            await self.job_manager.mark_job_execution_failure(
                job_id, error[:1000] if error else "Unknown error"
            )

            # Check if job should be auto-paused
            consecutive_failures = await self.job_manager.get_consecutive_failures(job_id)
            if consecutive_failures >= self.max_failures_before_pause:
                await self.job_manager.pause_job(job_id)

                observability.observe(
                    event_type=observability.ConversationEvents.SCHEDULED_JOB_PAUSED,
                    level=observability.EventLevel.WARNING,
                    data={
                        "job_id": job_id,
                        "consecutive_failures": consecutive_failures,
                        "max_failures": self.max_failures_before_pause,
                    },
                    description=f"Job auto-paused after {consecutive_failures} consecutive failures",
                )

            observability.observe(
                event_type=observability.ConversationEvents.SCHEDULED_JOB_FAILED,
                level=observability.EventLevel.ERROR,
                data={
                    "job_id": job_id,
                    "session_id": session_id,
                    "error": error[:500] if error else "Unknown error",
                    "consecutive_failures": consecutive_failures,
                },
                description=f"Job failed via webhook: {job['title']}",
            )

        # Log webhook received
        observability.observe(
            event_type=observability.ConversationEvents.SCHEDULED_JOB_WEBHOOK_RECEIVED,
            level=observability.EventLevel.INFO,
            data={
                "job_id": job_id,
                "session_id": session_id,
                "success": success,
            },
            description=f"Webhook received for scheduled job: {job_id}",
        )

        return True

    async def pause_job(self, job_id: str, user_id: str = "0") -> bool:
        """
        Pause a scheduled job.

        Args:
            job_id: Job ID to pause
            user_id: User requesting the pause (for audit trail)

        Returns:
            True if successful, False otherwise
        """
        success = await self.job_manager.pause_job(job_id, user_id=user_id)

        if success:
            observability.observe(
                event_type=observability.ConversationEvents.SCHEDULED_JOB_PAUSED,
                level=observability.EventLevel.INFO,
                data={"job_id": job_id},
                description=f"Scheduled job paused: {job_id}",
            )

        return success

    async def resume_job(self, job_id: str, user_id: str = "0") -> bool:
        """
        Resume a paused scheduled job.

        Args:
            job_id: Job ID to resume
            user_id: User requesting the resume (for audit trail)

        Returns:
            True if successful, False otherwise
        """
        success = await self.job_manager.resume_job(job_id, user_id=user_id)

        if success:
            observability.observe(
                event_type=observability.ConversationEvents.SCHEDULED_JOB_RESUMED,
                level=observability.EventLevel.INFO,
                data={"job_id": job_id},
                description=f"Scheduled job resumed: {job_id}",
            )

        return success

    async def delete_job(self, job_id: str, user_id: str = "0") -> bool:
        """
        Delete a scheduled job.

        Args:
            job_id: Job ID to delete
            user_id: User requesting the deletion (for audit trail)

        Returns:
            True if successful, False otherwise
        """
        success = await self.job_manager.delete_job(job_id, user_id=user_id)

        if success:
            observability.observe(
                event_type=observability.ConversationEvents.SCHEDULED_JOB_DELETED,
                level=observability.EventLevel.INFO,
                data={"job_id": job_id},
                description=f"Scheduled job deleted: {job_id}",
            )

        return success

    async def list_user_jobs(self, user_id: str) -> List[Dict[str, Any]]:
        """
        List all jobs for a user.

        Args:
            user_id: User ID to list jobs for

        Returns:
            List of job data dicts
        """
        return await self.job_manager.get_user_jobs(user_id)

    # Performance monitoring methods

    async def get_performance_stats(self) -> Dict[str, Any]:
        """
        Get current performance statistics.

        Returns:
            Dictionary with performance metrics
        """
        # Get cache statistics
        cache_stats = self.cache.get_cache_stats() if self.cache else {}

        # Get circuit breaker stats
        circuit_stats = self.llm_circuit_breaker.get_stats() if self.llm_circuit_breaker else {}

        # Get job count
        active_jobs_count = await self.job_manager.count_active_jobs()

        # Calculate LLM calls saved
        llm_calls_saved = cache_stats.get("hits", 0)

        return {
            "scheduler_stats": {
                "cycles_completed": self._performance_stats["cycles_completed"],
                "jobs_processed": self._performance_stats["jobs_processed"],
                "batch_processing_time": self._performance_stats["batch_processing_time"],
                "active_jobs": active_jobs_count,
                "batch_size": self.batch_processor.get_batch_size(),
            },
            "cache_stats": cache_stats,
            "circuit_breaker_stats": circuit_stats,
            "performance_improvements": {
                "llm_calls_saved": llm_calls_saved,
                "estimated_cost_savings": llm_calls_saved * 0.002,  # Rough estimate
                "cache_hit_rate": cache_stats.get("hit_rate", 0.0),
                "memory_efficient": True,  # Using batch processing
            },
        }

    async def cleanup_performance_caches(self) -> Dict[str, int]:
        """
        Clean up expired cache entries and old job records.

        Returns:
            Dictionary with cleanup statistics
        """
        # Clean expired cache entries
        cache_cleaned = self.cache.cleanup_expired() if self.cache else 0

        # Clean old job records
        retention_days = self._config.get("retention_days", 30)
        jobs_cleaned = await self.batch_processor.cleanup_old_jobs(retention_days)

        return {"cache_entries_removed": cache_cleaned, "old_jobs_removed": jobs_cleaned}

    def reset_circuit_breaker(self) -> None:
        """
        Manually reset the LLM circuit breaker.

        Useful for recovering from extended LLM outages.
        """
        if self.llm_circuit_breaker:
            self.llm_circuit_breaker.reset()
