import time
import logging
import traceback

from datetime import datetime, timedelta, timezone
from concurrent.futures import ThreadPoolExecutor, as_completed

from wurker.hive import Hive
from wurker.settings import conf
from wurker.bee import Bee, PyBee
from wurker.util import init_logger
from wurker.db import update_status, make_session
from wurker.cli.commands.admin.status import StatusCommand
from wurker.models import WurkerBeeState, WurkerControl, WurkerBee, WurkerCron


# Using globals, so let's be up-front... here they are.
class Globals:
    STOP = False
    PAUSE = False


class Wurkman:
    """The controller class."""

    logger = None

    # Init settings
    app_id = ''

    db_conf = None
    session = None

    bee_ct = 0
    slug_map = None

    control = None
    hive = None
    bees = None


    def __init__(self, session=None, logger=None):
        self.app_id = conf.app_id
        self.app_root = conf.app_root
        self.logger = logger or init_logger()

        self._init_db(session=session)
        self._init_slug_map()
        self._init_status()


    def log(self, msg, exception=False, level=logging.INFO, prefix=''):
        """Adds Bee-specific metadata to log entries."""

        if exception:
            level = logging.ERROR
            msg += "\n{0}".format(traceback.format_exc())

        prefix = prefix or "Wurkman"
        self.logger.log(msg, level=level, prefix=prefix)


    def _init_db(self, session=None):
        """Initialize the configuration and local DB connections."""

        self.db_conf = conf.RemoteDB
        self.log(f"Connecting to config DB: '{self.db_conf.database}'...")
        self.session = session or make_session(self.db_conf)

        # And then load the hive
        self.hive = Hive(app_root=self.app_root, logger=self.logger)


    def _init_slug_map(self):
        """Initialize ID-slug maps for crons and bees. Used by hive logic."""

        slug_map = {"bee":{}, "cron":{None:"ondemand"}}

        with self.session() as db:
            for bee in db.query(WurkerBee).filter(WurkerBee.app_id == self.app_id).all():
                slug_map["bee"][bee.id] = bee.slug

            for cron in db.query(WurkerCron).filter(WurkerCron.app_id == self.app_id).all():
                slug_map["cron"][cron.id] = cron.slug

        self.slug_map = slug_map


    def _init_status(self):
        """Initialize wurker_status table from local settings and state."""

        # Get the status data from the 'status' subcommand
        status_cmd = StatusCommand()
        status = status_cmd.run(return_result=True)

        # Use the util to update DB to validate 1st
        try:
            update_status(self.db_conf, self.app_id, status)
        except Exception as ex:
            raise Exception(
                f"Failed to update 'wurker_status' on host '{self.db_conf.hostname}' due to exception: {ex}"
            )


    @property
    def Now(self):
        return datetime.now(timezone.utc)


    def rm_bee(self, wurk_type, cron, bee):
        """Delete a bee from self.bees (for run_once CLI logic)"""

        if not self.bees or wurk_type not in self.bees:
            return

        cron_slug = cron if isinstance(cron, str) else cron.slug
        bee_slug = bee if isinstance(bee, str) else bee.slug

        if cron_slug not in self.bees[wurk_type] or bee_slug not in self.bees[wurk_type][cron_slug]:
            return

        self.log(f"Removing Wurker Bee '{bee_slug}' from '{wurk_type}' cache", level=logging.DEBUG)

        del self.bees[wurk_type][cron_slug][bee_slug]

        # We also have to remove ondemand bees from the queue table
        if wurk_type == "ondemand":
            self.log(f"Removing Wurker Bee '{bee_slug}' from 'ondemand' queue",
                     level=logging.DEBUG)

            with self.session() as db:
                if isinstance(bee, str):
                    bee = db.query(WurkerBee).filter(
                        WurkerBee.slug == bee_slug,
                        WurkerBee.app_id == self.app_id
                    ).first()

                # No queue? Weird! But OK...
                if not len(bee.queue):
                    self.log(f"Wurker Bee '{bee_slug}' not queued!", level=logging.WARN)
                    return

                # Delete the bee from the queue
                queue = bee.queue[0]
                db.delete(queue)
                db.flush()
                db.commit()


    def get_bees(
            self,
            wurk_type:str='',
            slug_type:str='',
            slug:str='',
            show_cmd:bool=False,
            any_bees:bool=True
    ):
        """
        Return scheduled Wurker Bees (jobs). Looks confusing, but basically, it boils down to:
        - Bee only: dict with [bee.slug] for keys
        - Cron only: dict with [cron.slug][bee.slug] for keys
        - Scheduled/Supervised:
        """

        if wurk_type and wurk_type not in ["scheduled", "supervised", "ondemand",]:
            raise Exception(f"Invalid wurk type: '{wurk_type}'")

        self.log("Obtaining Wurker Bees from DB", level=logging.DEBUG)

        cron_slug = None
        bee_slug = None
        if slug_type == "cron" and slug:
            cron_slug = slug
        elif slug_type == "bee" and slug:
            bee_slug = slug

        # Let's decide what 'bees' response should look like...
        bees = {"scheduled": {}, "supervised": {}, "ondemand": {}, "orphaned": {}, "disabled": {}}
        bee_ct = 0

        with self.session() as db:
            query = db.query(WurkerBee).filter(WurkerBee.app_id == self.app_id)
            if cron_slug:
                query = query.filter(WurkerBee.crons.any(slug=cron_slug))
                bees = {} if show_cmd else bees
            elif bee_slug:
                query = query.filter(WurkerBee.slug == bee_slug)
                bees = {} if show_cmd else bees

            result = query.all()
            for dbee in result:
                # Note we load ondemand separately and first
                if len(dbee.queue) and "ondemand" in bees:
                    if "ondemand" not in bees["ondemand"]:
                        bees["ondemand"]["ondemand"] = {}

                    bee = (
                        PyBee(self.session, dbee, args=dbee.queue[0].args, logger=self.logger)
                        if dbee.command.is_module else
                        Bee(self.session, dbee, args=dbee.queue[0].args, logger=self.logger)
                    )
                    bees["ondemand"]["ondemand"][dbee.slug] = bee.AsDict if show_cmd else bee
                    bee_ct += 1

                    if not show_cmd and wurk_type == "ondemand":
                        continue

                # Bee has no crons? Use --any if you want orphaned bees and childless crons
                if not len(dbee.crons) and wurk_type != "ondemand":
                    if "orphaned" in bees:
                        if "orphaned" not in bees["orphaned"]:
                            bees["orphaned"]["orphaned"] = {}

                        bee = (
                            PyBee(self.session, dbee, logger=self.logger)
                            if dbee.command.is_module else
                            Bee(self.session, dbee, logger=self.logger)
                        )

                        bees["orphaned"]["orphaned"][dbee.slug] = bee.AsDict if show_cmd else bee

                    continue

                for cron in dbee.crons:
                    bee = (
                        PyBee(self.session, dbee, cron, logger=self.logger)
                        if dbee.command.is_module else
                        Bee(self.session, dbee, cron, logger=self.logger)
                    )

                    if not bee.enabled:
                        if "disabled" in bees:
                            if bee.cron_slug not in bees["disabled"]:
                                bees["disabled"][bee.cron_slug] = {}
                            bees["disabled"][bee.cron_slug][bee.slug] = bee.AsDict if show_cmd else bee

                        continue

                    # Just the 'show' command? Let's get 'er done...
                    if show_cmd and (bee_slug or cron_slug):
                        if bee_slug:
                            bees[bee.slug] = bee.AsDict if show_cmd else bee
                        else:
                            if bee.cron_slug not in bees:
                                bees[bee.cron_slug] = {}
                            bees[bee.cron_slug][bee.slug] = bee.AsDict if show_cmd else bee

                        bee_ct += 1
                        continue

                    # Supervised?
                    if bee.cron is None:
                        if bee.cron_slug not in bees["supervised"]:
                            bees["supervised"][bee.cron_slug] = {}
                        bees["supervised"][bee.cron_slug][bee.slug] = bee.AsDict if show_cmd else bee
                        bee_ct += 1
                        continue

                    # Or Scheduled?
                    if bee.cron_slug not in bees["scheduled"]:
                        bees["scheduled"][bee.cron_slug] = {}
                    bees["scheduled"][bee.cron_slug][bee.slug] = bee.AsDict if show_cmd else bee
                    bee_ct += 1

        # Is wurk_type set? Then just return that
        if wurk_type:
            bee_ct = sum(len(b) for b in bees[wurk_type].values())
            return bees[wurk_type], bee_ct

        if any_bees:
            return bees, bee_ct

        # Not --any so only return scheduled Bees
        if "orphaned" in bees:
            del bees["orphaned"]
        if "disabled" in bees:
            del bees["disabled"]

        return bees, bee_ct


    def get_hive(self, incl_duration=True):
        """Get Hive data from local DB, formatted for display."""

        # For 'show' subcommand, we trick get_bees() to get real bees
        bees, bee_ct = self.get_bees()

        flat = {}
        for section, crons in bees.items():
            for cron_slug, cron in crons.items():
                if cron_slug not in flat:
                    flat[cron_slug] = {}

                for bee_slug, bee in cron.items():
                    if bee_slug not in flat[cron_slug]:
                        flat[cron_slug][bee_slug] = bee

        hive = {}
        for cron_id, cron in self.hive.hive.items():
            cron_slug = "ondemand" if cron_id is None else self.slug_map["cron"][cron_id]
            if cron_slug not in flat:
                continue

            for bee_id, item in cron.items():
                bee_slug = self.slug_map["bee"][bee_id]
                bee = flat[cron_slug][bee_slug]

                if cron_slug not in hive:
                    hive[cron_slug] = {}

                if incl_duration and item['timestamp']:
                    item['duration'] = str(self.Now - item['timestamp'])

                hive[cron_slug][bee_slug] = item

        return hive


    def get_control(self):
        """Get wurker_control record."""

        self.log("Obtaining Wurker Control from DB", level=logging.DEBUG)

        with self.session() as db:
            control = db.query(WurkerControl).filter(
                WurkerControl.app_id == self.app_id
            ).one()

        if control is None:
            raise Exception("Unable to obtain wurker_control record")

        return control


    def set_control(self, reload:bool=None, pause:bool=None):
        """Nones will not be updated. Only strict booleans."""

        if not isinstance(reload, bool) and not isinstance(pause, bool):
            return

        self.log("Setting Wurker Control in DB", level=logging.DEBUG)

        with self.session() as db:
            control = db.query(WurkerControl).filter(
                WurkerControl.app_id == self.app_id
            ).one()

            if control is None:
                raise Exception("Unable to obtain wurker_control record")

            if isinstance(reload, bool):
                control.reload = reload
            if isinstance(pause, bool):
                control.pause = pause

            db.flush()
            db.commit()


    def run_once(self, wurk_type:str='', slug_type:str='', slug:str=''):
        """Run the bees once through for any the CLI options."""

        bees, _ = self.get_bees(wurk_type=wurk_type, slug_type=slug_type, slug=slug)
        if not len(bees):
            return "Wurker Bee(s) not found"

        hive = {"ondemand":{}, "supervised":{}}
        if wurk_type == "ondemand":
            hive["ondemand"] = bees["ondemand"]
        elif wurk_type in ["scheduled", "supervised"]:
            hive["supervised"] = bees[wurk_type]
        else:
            hive["supervised"] = bees["supervised"]
            hive["supervised"].update(bees["scheduled"])
            hive["supervised"].update(bees["orphaned"])
            hive["ondemand"] = bees["ondemand"]

        bee_ct = 0
        for wurk_type, wurk_cfg in hive.items():
            for cron_slug, bee_cfg in wurk_cfg.items():
                for bee_slug, bee in bee_cfg.items():
                    bee.run_once = True
                    bee.cron = None
                    bee_ct += 1

        self.bees = hive
        self.bee_ct = bee_ct
        if not self.bee_ct:
            return f"No Wurker Bee(s) found matching criteria"

        self.log(f"Running ({self.bee_ct}) Wurker Bee(s) one time")

        errno = None
        try:
            while not Globals.STOP:
                if self.bees["supervised"]:
                    self._run_supervised(run_once=True)
                if self.bees["ondemand"]:
                    self._run_ondemand(run_once=True)
                time.sleep(1)

        except KeyboardInterrupt:
            errno = "\nKeyboard interrupt detected!"
        except Exception as ex:
            traceback.print_exc()
            errno = ex
        finally:
            self.kill()

        self.log("Wurk is COMPLETED")


    def run_forever(self):
        """Run the bees forever."""

        self.log("Running Wurker forever", level=logging.DEBUG)

        errno = None

        # Clear the Hive before running forever
        self.hive.clear()

        # Initial gets. We'll get again in case we need to reload
        control = self.get_control()
        if control.pause:
            errno = f"Wurker Control ({control.app_id}) sent PAUSE signal!"
            self.log(errno, level=logging.ERROR)
            return errno

        self.bees, self.bee_ct = self.get_bees()
        try:
            init_time = curr_time = time.time()

            while not Globals.STOP:
                if control.pause:
                    self.log(f"Wurker Control ({control.app_id}) sent PAUSE signal!", level=logging.ERROR)

                    Globals.PAUSE = True
                    self.kill()
                    control = self.get_control()

                elif control.reload:
                    self.log(f"Wurker Control ({control.app_id}) sent RELOAD signal!", level=logging.ERROR)

                    delay = 0
                    self.kill()
                    self.bees, self.bee_ct = self.get_bees()
                    self.set_control(reload=False)
                    control = self.get_control()

                elif curr_time - init_time >= 30:
                    self.log(f"Checking for Control and Ondemand events", level=logging.DEBUG)

                    init_time = time.time()
                    control = self.get_control()
                    self._run_ondemand()

                else:
                    self._run_scheduled()
                    self._run_supervised()

                curr_time = time.time()

        except KeyboardInterrupt:
            errno = "\nKeyboard interrupt detected!"
        except Exception as ex:
            traceback.print_exc()
            errno = ex
        finally:
            self.kill()
            self.hive.clear()
            self.log("Unexpected exit!", level=logging.ERROR)

        return errno


    def _run_scheduled(self):
        """Optimized method to categorize and handle scheduled Bees concurrently."""

        if not "scheduled" in self.bees or not self.bees["scheduled"]:
            return

        offset = 30
        tmp_now = self.Now.replace(second=0, microsecond=0)
        win_beg = tmp_now - timedelta(seconds=offset)
        win_end = tmp_now + timedelta(seconds=offset)

        to_run, to_kill, to_poll, to_overrun = set(), set(), set(), set()

        # Step 1: Categorize Bees into action sets
        for cron_slug, bee_cfg in self.bees.get("scheduled", {}).items():
            for bee_slug, bee in bee_cfg.items():
                if not bee.enabled:
                    continue

                # Is there a job process already?
                if bee.proc is not None:

                    # There's a process, but is the job done?
                    if bee.poll() is not None:
                        # Job done, so we'll complete it poll_bee()
                        to_poll.add(bee)

                    elif tmp_now - bee.CurrRun > timedelta(seconds=bee.timeout):
                        # Job still running and timing out. Overrun allowed?
                        if not bee.overrun or bee.runcount >= bee.overrun:
                            # No overrun, so kill_bee()
                            to_kill.add(bee)

                    continue

                # Time to run the job?
                if tmp_now - timedelta(seconds=offset) <= bee.NextRun < tmp_now + timedelta(seconds=offset):
                    # It's time, but is the job still running?
                    if bee.proc is not None and bee.poll() is None:
                        # Job still running, but can we let it overrun?
                        if bee.overrun and bee.runcount <= bee.overrun:
                            # We can let it overrun
                            to_overrun.add(bee)
                        else:
                            # Overrun not allowed. Kill it
                            to_kill.add(bee)

                        continue

                    # OK to run this bee!
                    to_run.add(bee)

        # Step 2: Define functions to run, poll, kill, and complete bees with retries
        def run_bee_with_retries(bee, retries=3):
            """Attempt to run a bee with retries on failure."""
            attempts = 0
            while attempts < retries:
                try:
                    bee.run()
                    bee.set_next()  # Schedule next run
                    self.hive.add(bee)  # Add to active hive
                    self.log(f"[{bee.PID}] SCHEDULED '{bee.slug}' job STARTED", level=logging.INFO)
                    return
                except Exception as e:
                    attempts += 1
                    self.log(f"Run attempt {attempts}/{retries} failed for bee '{bee.slug}': {e}",
                             level=logging.ERROR)

                    if attempts >= retries:
                        bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": [str(e)]})
                        self.log(f"Bee '{bee.slug}' failed after {retries} attempts", level=logging.ERROR)

        def poll_bee(bee):
            """Check if a running bee has completed."""
            try:
                if bee.poll() is not None:  # Bee completed
                    errno = bee.poll()
                    level = logging.ERROR if errno != 0 else logging.INFO
                    self.log(f"[{bee.PID}] SCHEDULED '{bee.slug}' job COMPLETED with exit code ({errno})",
                             level=level)

                    bee.kill()  # Clean up process
                    self.hive.delete(bee)  # Remove from hive
                    bee.reset_proc()  # Explicitely reset PID and process to None
                    if bee in to_poll:
                        to_poll.discard(bee)  # Ensure it's removed from polling set
                    if bee in to_kill:
                        to_kill.discard(bee)  # Ensure it's removed from killing set
                    if bee.errors:
                        self.log(f"[{bee.PID}] '{bee.slug}' job errors:\n{bee.errors}", level=logging.ERROR)

            except Exception as e:
                self.log(f"Error polling bee '{bee.slug}': {e}", level=logging.ERROR)
                bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": [str(e)]})

        def overrun_bee(bee):
            """Handle an overrun situation for a Bee."""
            try:
                bee.runcount += 1  # Increment the run count to track overruns
                bee.set_report(status=WurkerBeeState.OVERRUN)
                self.hive.set_runcount(bee)  # Update the hive with the new runcount

                self.log(f"[{bee.PID}] SCHEDULED '{bee.slug}' job allowed to overrun "
                    f"({bee.runcount}/{bee.overrun})", level=logging.WARNING)

            except Exception as e:
                self.log(f"Error handling overrun for bee '{bee.slug}': {e}", level=logging.ERROR)
                bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": [str(e)]})

        def kill_bee(bee):
            """Kill a bee if it's exceeded its timeout."""

            # Check if PID is already cleared, indicating bee is already completed
            if bee.pid is None:
                self.log(f"[{bee.slug}] has no PID; skipping kill as it should be completed.", level=logging.INFO)

                to_kill.discard(bee)
                return

            self.log(f"[{bee.PID}] SCHEDULED '{bee.slug}' job TIMEOUT. KILLING it...", level=logging.ERROR)

            try:
                bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": ["Timeout exceeded"]})
                bee.kill()
                self.hive.delete(bee)
                bee.reset_proc()  # Reset PID and proc
                to_kill.discard(bee)  # Remove from killing set to avoid repeated timeout actions
                if bee.errors:
                    self.log(f"[{bee.PID}] '{bee.slug}' job errors:\n{bee.errors}", level=logging.ERROR)

            except Exception as e:
                self.log(f"Error killing bee '{bee.slug}': {e}", level=logging.ERROR)
                bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": [str(e)]})

        # Step 3: Execute actions concurrently with ThreadPoolExecutor and handle errors
        completed_bees = set()  # Track bees that are fully completed
        with ThreadPoolExecutor() as executor:
            futures = {
                executor.submit(run_bee_with_retries, bee): bee for bee in to_run
            }
            futures.update({
                executor.submit(poll_bee, bee): bee for bee in to_poll
            })
            futures.update({
                executor.submit(kill_bee, bee): bee for bee in to_kill
            })
            futures.update({
                executor.submit(overrun_bee, bee): bee for bee in to_overrun
            })

            for future in as_completed(futures):
                bee = futures[future]
                try:
                    future.result()  # Raises exception if the task failed
                except Exception as e:
                    self.log(f"Unhandled error for bee '{bee.slug}': {e}", level=logging.ERROR)
                    bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": [str(e)]})


    def _run_supervised(self, run_once=False):
        """Run supervised (always on) bees."""

        if Globals.PAUSE:
            return

        if not "supervised" in self.bees or not self.bees["supervised"]:
            return

        rm_bees = []
        supervised = self.bees["supervised"]
        bee_ct = sum(len(b) for b in supervised.values())
        if not bee_ct:
            if run_once:
                Globals.STOP = True
            return

        for cron_slug, bee_cfg in supervised.items():
            for bee_slug, bee in bee_cfg.items():
                if not bee.enabled:
                    continue

                action = None
                level = None

                if not bee.proc:
                    level, action = (logging.INFO, "STARTED")
                elif bee.poll() is not None:
                    self.hive.delete(bee)

                    if run_once:
                        rm_bees.append((cron_slug, bee_slug))
                    elif not bee.quiet_restart:
                        self.log(f"[{bee.PID}] SUPER '{bee.slug}' job DIED", level=logging.ERROR)
                        level, action = (logging.ERROR, "RESTARTED")
                    else:
                        level, action = (None, "QUIET RESTART")

                if action:
                    try:
                        bee.run()
                    except Exception as ex:
                        self.log(f"SUPER '{bee.slug}' job FAILED to run due to exception: {ex}", exception=True)
                        bee.kill()
                        continue

                    if not bee.proc:
                        self.log(f"SUPER '{bee.slug}' job FAILED to run", level=logging.ERROR)

                        pid = bee.pid
                        bee.kill()
                        if bee.errors:
                            self.log(f"[{pid}] SUPER '{bee.slug}' job errors:\n{bee.errors}", level=logging.ERROR)

                    elif level:
                        self.log(f"[{bee.PID}] SUPER '{bee.slug}' job {action}", level=level)
                        self.hive.add(bee)

        # If run_once then we can delete the bees outside the loop
        for pair in rm_bees:
            self.rm_bee("supervised", *pair)


    def _run_ondemand(self, run_once=False):
        """Run ondemand (once and done) bees. Reports back to the wurker_queue table."""

        self.log("ONDEMAND check", level=logging.DEBUG)

        if Globals.PAUSE:
            self.log("ONDEMAND PAUSED", level=logging.WARN)
            return

        # Update ondemand Bees from DB
        ondemand, bee_ct = self.get_bees("ondemand")

        if bee_ct and "ondemand" in ondemand:
            if not "ondemand" in self.bees:
                self.bees["ondemand"] = {}
            if not "ondemand" in self.bees["ondemand"]:
                self.bees["ondemand"]["ondemand"] = {}

            for bee_slug, bee in ondemand["ondemand"].items():
                if bee_slug not in self.bees["ondemand"]["ondemand"]:
                    self.bees["ondemand"]["ondemand"][bee_slug] = bee

        # No ondemand? Then nothing to do...
        if not "ondemand" in self.bees or not self.bees["ondemand"]:
            return

        rm_bees = []
        tmp_now = self.Now
        ondemand = self.bees["ondemand"]
        bee_ct = sum(len(b) for b in ondemand.values())

        # No ondemand bees left? Then nothing to do...
        if not bee_ct or not "ondemand" in ondemand:
            if run_once:
                Globals.STOP = True
            return

        # OK loop over ondemand Bees...
        for bee_slug, bee in ondemand["ondemand"].items():
            if not bee.enabled:
                self.log(f"ONDEMAND '{bee.slug}' job is disabled", level=logging.DEBUG)
                continue

            if bee.queue is None:
                self.log(f"ONDEMAND '{bee.slug}' job is not queued", level=logging.DEBUG)
                continue

            # Is there a job process already?
            if bee.proc is not None:

                # There's a process. Is the job done?
                if bee.poll() is not None:
                    errno = bee.poll()

                    level = logging.ERROR if errno != 0 else logging.INFO
                    self.log(f"[{bee.PID}] ONDEMAND '{bee.slug}' job COMPLETED with exit code ({errno})", level=level)

                    pid = bee.pid
                    bee.kill()
                    self.hive.delete(bee)
                    if bee.errors:
                        self.log(f"[{pid}] ONDEMAND '{bee.slug}' job errors:\n{bee.errors}", level=logging.ERROR)

                    rm_bees.append(bee)

                # Job still running... is it timing out?
                elif bee.timeout and tmp_now - bee.start_dt > timedelta(seconds=bee.timeout):
                    self.log(f"[{bee.PID}] ONDEMAND '{bee.slug}' job TIMEOUT. KILLED", level=logging.ERROR)

                    bee.set_report(status=WurkerBeeState.ERROR, status_info={
                        "errors":["Timeout after {bee.timeout} seconds"]
                    })

                    pid = bee.pid
                    if not bee.kill(error_status=WurkerBeeState.TIMEOUT):
                        self.log(f"[{bee.PID}] SCHED '{bee.slug}' job could not be killed!", level=logging.ERROR)
                        continue

                    self.hive.delete(bee)

                    if bee.errors:
                        self.log(f"[{pid}] ONDEMAND '{bee.slug}' job errors:\n{bee.errors}", level=logging.ERROR)

                    rm_bees.append(bee)

            # No process, so run the job
            else:
                try:
                    bee.run()
                except Exception as ex:
                    self.log(f"ONDEMAND '{bee.slug}' run FAILED with exception: {ex}", exception=True)
                    bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": ["Failed to start"]})
                    bee.kill()
                    continue

                if not bee.proc:
                    self.log(f"ONDEMAND '{bee.slug}' job FAILED", level=logging.ERROR)
                    bee.set_report(status=WurkerBeeState.ERROR, status_info={"errors": ["Failed to start"]})
                    bee.kill()
                else:
                    self.log(f"[{bee.PID}] ONDEMAND '{bee.slug}' job STARTED")
                    self.hive.add(bee)

        # OK now that we're not in the loop we can unset any expired bees
        for bee in rm_bees:
            self.rm_bee("ondemand", "ondemand", bee.slug)


    def kill(self, bees:dict={}):
        """Kill some or all of the busy bees."""

        bees = bees or self.bees or {}
        if not len(bees):
            return "Wurker Bee(s) not found"

        for wurk_type, wurk_cfg in bees.items():
            for cron_slug, bee_cfg in wurk_cfg.items():
                for bee_slug, bee in bee_cfg.items():
                    pid = bee.pid
                    bee.kill()
                    self.hive.delete(bee)

                    if pid:
                        level, action = (
                            (logging.ERROR, "KILLED") if not bee.run_once
                            else (logging.INFO, "COMPLETED")
                        )
                        self.log(f"[{pid}] '{bee.slug}' job {action}", level=level)

                        if bee.errors:
                            self.log(f"[{pid}] '{bee.slug}' job errors:\n{bee.errors}", level=logging.ERROR)

        # Finally, clear the hive
        self.hive.clear()
