"""Crawler orchestration and runtime management."""

import asyncio
import os
import signal
import threading
from datetime import datetime

from anystore.functools import weakref_cache as cache
from anystore.logging import get_logger
from anystore.types import SDict, Uri
from anystore.util import ensure_uri, ensure_uuid, path_from_uri
from ftm_lakehouse import get_entities
from openaleph_procrastinate.manage import cancel_jobs
from openaleph_procrastinate.settings import OpenAlephSettings
from procrastinate.jobs import DeleteJobCondition

from memorious.core import get_tags, settings
from memorious.model import CrawlerConfig, CrawlerStage
from memorious.operations import resolve_operation


@cache
def get_crawler(uri: Uri) -> "Crawler":
    """Load and cache a crawler from a YAML configuration file.

    Args:
        uri: Path or URI to the crawler YAML config file.

    Returns:
        Cached Crawler instance.
    """
    return Crawler(uri)


class Crawler:
    """A processing graph that constitutes a crawler.

    Wraps CrawlerConfig with runtime state and operations.
    """

    def __init__(self, source_file: Uri):
        self.source_file = ensure_uri(source_file)

        # Load and parse config using anystore's from_yaml_uri
        self.config = CrawlerConfig.from_yaml_uri(self.source_file)

        # Build stage objects
        self.stages = {}
        for stage_name, stage_config in self.config.pipeline.items():
            self.stages[stage_name] = CrawlerStage(self, stage_name, stage_config)

        # Configure logging
        self.log = get_logger(__name__, crawler=self.config.name, config=source_file)

    # Delegate to config properties
    @property
    def name(self) -> str:
        return self.config.name

    @property
    def description(self) -> str:
        return self.config.title or self.name

    @property
    def init_stage(self) -> str:
        return self.config.init

    @property
    def delay(self) -> int:
        return self.config.delay

    @property
    def expire(self) -> int:
        """Expire time in seconds."""
        expire_days = self.config.expire or settings.expire
        return expire_days * 86400

    @property
    def max_runtime(self) -> int:
        """Max runtime in seconds (0 = unlimited)."""
        return self.config.max_runtime or settings.max_runtime

    @property
    def stealthy(self) -> bool:
        return self.config.stealthy

    @property
    def aggregator_config(self) -> dict:
        if self.config.aggregator:
            return {
                "method": self.config.aggregator.method,
                "params": self.config.aggregator.params,
            }
        return {}

    @property
    def aggregator_method(self):
        if not self.config.aggregator:
            return None
        base_path = path_from_uri(self.source_file).parent
        return resolve_operation(self.config.aggregator.method, base_path)

    def aggregate(self, context):
        if self.aggregator_method:
            self.log.info("Running aggregator", dataset=self.name)
            params = self.config.aggregator.params if self.config.aggregator else {}
            self.aggregator_method(context, params)
        self.entities_flush()

    def set_run_started(self, run_id: str) -> None:
        """Record the start time for a run.

        Args:
            run_id: Unique run identifier.
        """
        tags = get_tags(self.name)
        tags.set(f"runs/{run_id}/started", datetime.now())

    def get_run_started(self, run_id: str) -> datetime | None:
        """Get the start timestamp for a run.

        Args:
            run_id: Unique run identifier.

        Returns:
            The datetime when the run started, or None if not found.
        """
        return _get_run_started(self.name, run_id)

    def is_run_expired(self, run_id: str) -> bool:
        """Check if a run has exceeded its max_runtime.

        Args:
            run_id: Unique run identifier.

        Returns:
            True if max_runtime > 0 and the run has exceeded it.
        """
        if self.max_runtime <= 0:
            return False
        started_ts = self.get_run_started(run_id)
        if started_ts is None:
            return False
        elapsed = (datetime.now() - started_ts).total_seconds()
        return elapsed > self.max_runtime

    def flush(self):
        """Delete all run-time data generated by this crawler."""
        self.flush_tags()

    def flush_tags(self):
        tags = get_tags(self.name)
        tags.delete(prefix=self.name)

    def entities_flush(self):
        """Flush the entity write-ahead journal to the lakehouse."""
        entities = get_entities(self.name)
        entities.flush()

    def cancel(self):
        """Cancel all pending/running jobs for this crawler."""
        if OpenAlephSettings().in_memory_db:
            # remove pending jobs from in-memory queue
            from memorious.tasks import app

            for job in app.job_manager.list_jobs():
                if job.id:
                    try:
                        app.job_manager.cancel_job_by_id(job.id)
                    except Exception:  # another worker might have already deleted
                        pass
        else:
            # a real sql db
            cancel_jobs(dataset=self.name)

        self.entities_flush()

    def stop(self):
        """Stop crawler execution by cancelling jobs and sending SIGTERM.

        This is used for hard stops on errors when continue_on_error=False.
        Unlike cancel(), this also terminates the current worker process.
        """
        # self.cancel()
        self.log.warning("Stopping crawler, sending SIGTERM")
        os.kill(os.getpid(), signal.SIGTERM)

    def start(
        self, incremental=None, run_id=None, continue_on_error=None, clear_runs=True
    ):
        """
        Queue the initial stage for execution by external workers.

        This only defers the first task - external workers must be running
        to pick up and process the jobs.

        Args:
            incremental: Skip already-processed items (default: from settings).
            run_id: Unique run identifier (default: auto-generated UUID).
            continue_on_error: Continue on errors (default: from settings).
            clear_runs: Cancel previous runs before starting (default: True).
        """
        # Cancel previous runs if requested
        if clear_runs:
            self.cancel()

        # Generate run_id if not provided
        run_id = run_id or ensure_uuid()

        # Record run start time for max_runtime tracking
        self.set_run_started(run_id)

        # Defer the initial stage
        self.defer(
            self.config.init,
            data={},
            incremental=incremental,
            run_id=run_id,
            continue_on_error=continue_on_error,
        )

    def run(
        self,
        incremental=None,
        run_id=None,
        continue_on_error=None,
        concurrency: int = 1,
        wait: bool | None = None,
        clear_runs: bool = True,
        idle_timeout: int | None = None,
    ):
        """
        Run the crawler.

        Defers the initial stage and starts a worker to process jobs.

        Args:
            incremental: Skip already-processed items (default: from settings).
            run_id: Unique run identifier (default: auto-generated UUID).
            continue_on_error: Continue on errors (default: from settings).
            concurrency: Number of concurrent jobs to process. Use >1 for I/O-bound
                crawlers to improve throughput (default: 1).
            wait: If True, block until worker is stopped by signal. If False,
                process all queued jobs and return. Default: False for concurrency=1,
                True for concurrency>1 (required for async execution).
            clear_runs: Cancel previous runs before starting (default: True).
            idle_timeout: Seconds of queue inactivity before auto-stopping workers.
                Auto-enabled (30s) when concurrency>1. Set to 0 to disable.

        Note:
            If max_runtime is set, a timer will send SIGTERM to stop the worker
            after the specified duration. This is useful for CI environments
            with time limits (e.g., GitHub Actions 6h limit).
        """
        from memorious.tasks import app

        # Concurrency > 1 requires wait=True for async execution
        if not wait:
            wait = concurrency > 1

        # Auto-enable idle_timeout for concurrent crawlers
        if idle_timeout is None and concurrency > 1:
            idle_timeout = 30

        # Generate run_id if not provided (needed for aggregate context)
        run_id = run_id or ensure_uuid()

        # Defer the initial stage
        self.start(
            incremental=incremental,
            run_id=run_id,
            continue_on_error=continue_on_error,
            clear_runs=clear_runs,
        )

        # Set up timeout timer if max_runtime is configured
        timer = None
        if self.max_runtime > 0:

            def _timeout_handler():
                self.log.warning(
                    "Max runtime exceeded, sending SIGTERM to stop worker(s)",
                    max_runtime=self.max_runtime,
                )
                os.kill(os.getpid(), signal.SIGTERM)

            timer = threading.Timer(self.max_runtime, _timeout_handler)
            timer.daemon = True
            timer.start()

        try:
            # Run worker with idle monitoring if enabled (only applies when wait=True)
            if wait and idle_timeout and idle_timeout > 0:
                self._run_with_idle_monitor(
                    app=app,
                    concurrency=concurrency,
                    idle_timeout=idle_timeout,
                )
            else:
                # Run worker without idle monitoring
                # (wait=False already exits after processing all jobs)
                app.run_worker(
                    wait=wait,
                    concurrency=concurrency,
                    delete_jobs=DeleteJobCondition.SUCCESSFUL,
                )
        finally:
            # Cancel timeout timer if still running
            if timer is not None:
                timer.cancel()

            # Run aggregator (if configured) and flush entity journal
            self.aggregate(self._make_aggregate_context(run_id))

    def _run_with_idle_monitor(
        self,
        app,
        concurrency: int,
        idle_timeout: int,
    ):
        """Run worker with idle monitor for auto-stopping.

        Args:
            app: Procrastinate App instance.
            concurrency: Number of concurrent jobs.
            idle_timeout: Seconds of inactivity before stopping.
        """
        from memorious.logic.idle_monitor import IdleMonitor

        async def _run_async():
            async with app.open_async():
                # Create worker
                worker = app._worker(
                    wait=True,
                    concurrency=concurrency,
                    delete_jobs=DeleteJobCondition.SUCCESSFUL,
                )

                # Create idle monitor
                monitor = IdleMonitor(
                    app=app,
                    worker=worker,
                    queue="memorious",
                    idle_timeout=idle_timeout,
                )

                # Run both concurrently
                await asyncio.gather(
                    worker.run(),
                    monitor.run(),
                    return_exceptions=True,
                )

        asyncio.run(_run_async())

    def _make_aggregate_context(self, run_id: str | None):
        """Create a minimal context for running the aggregator.

        Args:
            run_id: Run identifier for the context.

        Returns:
            A BaseContext suitable for aggregator operations.
        """
        from memorious.logic.context import BaseContext

        return BaseContext(
            dataset=self.name,
            run_id=run_id,
            stealthy=self.stealthy,
        )

    def defer(
        self,
        stage: str,
        data: SDict,
        incremental=None,
        run_id=None,
        continue_on_error=None,
    ):
        """
        Emit (defer) a new job for this crawler
        """
        # Import here to avoid circular import with tasks.py
        from memorious.tasks import defer_stage

        run_id = run_id or ensure_uuid()
        incr = incremental if incremental is not None else settings.incremental
        coe = (
            continue_on_error
            if continue_on_error is not None
            else settings.continue_on_error
        )
        defer_stage(
            dataset=self.name,
            stage=stage,
            run_id=run_id,
            config_file=self.source_file,
            data=data,
            incremental=incr,
            continue_on_error=coe,
        )

    def get(self, name: str) -> CrawlerStage | None:
        return self.stages.get(name)

    def __str__(self) -> str:
        return self.name

    def __iter__(self):
        return iter(self.stages.values())

    def __repr__(self) -> str:
        return f"<Crawler({self.name})>"


@cache
def _get_run_started(name: str, run_id: str) -> datetime:
    tags = get_tags(name)
    return tags.get(f"runs/{run_id}/started")
