"""
This module's job is to run code, resp. to keep it running.


"""

from __future__ import annotations

import anyio
import time
from contextlib import AsyncExitStack, asynccontextmanager, suppress
from weakref import ref

import psutil
from asyncactor import AuthPingEvent, NodeList, PingEvent, TagEvent, UntagEvent

from moat.util import (
    NotGiven,
    P,
    Path,
    attrdict,
    combine_dict,
    create_queue,
    digits,
    logger_for,
    spawn,
)

from .actor import (
    ActorState,
    BrokenState,
    ClientActor,
    CompleteState,
    DetachedState,
    PartialState,
)
from .exceptions import ServerError
from .obj import AttrClientEntry, ClientRoot, MirrorRoot

from collections.abc import Mapping

try:
    ClosedResourceError = anyio.exceptions.ClosedResourceError
except AttributeError:
    ClosedResourceError = anyio.ClosedResourceError

import logging

logger = logging.getLogger(__name__)

QLEN = 10


class NotSelected(RuntimeError):
    """
    This node has not been selected for a very long time. Something is amiss.
    """

    pass


class ErrorRecorded(RuntimeError):  # noqa: D101
    pass


class RunnerMsg(ActorState):
    """Superclass for runner-generated messages.

    Not directly instantiated.

    This message and its descendants take one opaque parameter: ``msg``.
    """

    pass


class ChangeMsg(RunnerMsg):
    """A message telling your code that some entry has been updated.

    Subclass this and use it as `CallAdmin.watch`'s ``cls`` parameter for easier
    disambiguation.

    The runner sets ``path`` and ``value`` attributes.
    """

    pass


class MQTTmsg(RunnerMsg):
    """A message transporting some MQTT data.

    `value` is the MsgPack-decoded content. If that doesn't exist the
    message is not decodeable.

    The runner also sets the ``path`` attribute.
    """

    pass


class ReadyMsg(RunnerMsg):
    """
    This message is queued when the last watcher has read all data.
    """

    pass


class TimerMsg(RunnerMsg):
    """
    A message telling your code that a timer triggers.

    Subclass this and use it as `CallAdmin.timer`'s ``cls`` parameter for easier
    disambiguation.
    """

    pass


_CLASSES = attrdict()
for _c in (
    DetachedState,
    PartialState,
    CompleteState,
    ActorState,
    BrokenState,
    TimerMsg,
    ReadyMsg,
    ChangeMsg,
    MQTTmsg,
    RunnerMsg,
    ErrorRecorded,
):
    _CLASSES[_c.__name__] = _c

_CLASSES["NotGiven"] = NotGiven  # ellipsis


class CallAdmin:
    """
    This class collects some standard tasks which async MoaT-KV-embedded
    code might want to do.
    """

    _taskgroup = None
    _stack = None
    _restart = False
    _n_watch: int = None
    _n_watch_seen: int = None

    def __init__(self, runner, state, data):
        self._runner = runner
        self._state = state
        self._data = data
        self._client = runner.root.client
        self._err = runner.root.err
        self._q = runner._q  # noqa: SLF001
        self._path = runner._path  # noqa: SLF001
        self._subpath = runner.subpath
        self._logger = runner._logger  # noqa: SLF001

    async def _run(self, code, data):
        while True:
            self._n_watch = 0
            self._n_watch_seen = 0

            res = await self._run2(code, data)
            if self._restart:
                self._restart = False
            else:
                return res

    async def _run2(self, code, data):
        """Called by the runner to actually execute the code."""
        self._logger.debug("Start %s with %s", self._runner._path, self._runner.code)  # noqa: SLF001
        async with (
            anyio.create_task_group() as tg,
            AsyncExitStack() as stack,
        ):
            self._stack = stack
            self._taskgroup = tg
            self._runner.scope = sc = tg.cancel_scope

            data["_self"] = self

            oka = getattr(self._runner, "ok_after", 0)
            if oka > 0:

                async def is_ok(oka):
                    await anyio.sleep(oka)
                    await self.setup_done()

                tg.start_soon(is_ok, oka)
            tg.start_soon(self._changed_code, code)

            await self._runner.send_event(ReadyMsg(0))
            res = code(**data)
            if code.is_async is not None:
                res = await res

            sc.cancel()
            return res

    async def _pinger(self):
        t = self._runner.ok_after or 10
        await anyio.sleep(t / 2)
        while True:
            await self._runner.state.ping()
            await anyio.sleep(t * 5)

    async def _changed_code(self, code):
        """
        Kill the job if the underlying code has changed
        """
        await code.reload_event.wait()
        self._restart = True
        self.cancel()

    def cancel(self):
        """
        Cancel the running task
        """
        self._taskgroup.cancel_scope.cancel()

    async def spawn(self, proc, *a, **kw):
        """
        Start a background subtask.

        The task is auto-cancelled when your code ends.

        Returns: an `anyio.abc.CancelScope` which you can use to cancel the
            subtask.
        """
        return await spawn(self._taskgroup, proc, *a, **kw)

    async def setup_done(self, **kw):
        """
        Call this when your code has successfully started up.
        """
        self._state.backoff = 0
        await self._state.save()
        await self._err.record_working("run", self._runner._path, **kw)  # noqa: SLF001

    async def error(self, path=None, **kw):
        """
        Record that an error has occurred. This function records specific
        error data, then raises `ErrorRecorded` which the code is not
        supposed to catch.

        See `moat.kv.errors.ErrorRoot.record_error` for keyword details. The
        ``path`` argument is auto-filled to point to the current task.
        """
        if path is None:
            path = self._path
        r = await self._err.record_error("run", path, **kw)
        await self._err.root.wait_chain(r.chain)
        raise ErrorRecorded

    async def open_context(self, ctx):  # noqa: D102
        return await self._stack.enter_async_context(ctx)

    async def watch(self, path, cls=ChangeMsg, **kw):
        """
        Create a watcher. This path is monitored as per `moat.kv.client.Client.watch`;
        messages are encapsulated in `ChangeMsg` objects.
        A `ReadyMsg` will be sent when all watchers have transmitted their
        initial state.

        By default a watcher will only monitor a single entry. Set
        ``max_depth`` if you also want child entries.

        By default a watcher will not report existing entries. Set
        ``fetch=True`` if you want them.
        """

        class Watcher:
            """Helper class for watching an entry"""

            # pylint: disable=no-self-argument

            def __init__(slf, admin, runner, client, cls, path, kw):
                kw.setdefault("fetch", True)

                slf.admin = admin
                slf.runner = runner
                slf.client = client
                slf.path = path
                slf.kw = kw
                slf.cls = cls
                slf.scope = None

            async def run(slf):
                @asynccontextmanager
                async def _watch(path, kw):
                    if path.mark == "r":
                        async with slf.client.msg_monitor(path, **kw) as watcher:
                            yield watcher
                    elif not path.mark:
                        async with slf.client.watch(path, **kw) as watcher:
                            yield watcher
                    else:
                        raise RuntimeError(f"What should I do with a path marked {path.mark!r}?")

                with anyio.CancelScope() as sc:
                    slf.scope = sc
                    async with _watch(path, kw) as watcher:
                        async for msg in watcher:
                            if "path" in msg:
                                chg = cls(msg)
                                try:
                                    chg.value = (  # pylint:disable=attribute-defined-outside-init
                                        msg.value
                                    )
                                except AttributeError:
                                    pass
                                chg.path = (  # pylint:disable=attribute-defined-outside-init
                                    msg.path
                                )
                                await slf.runner.send_event(chg)

                            elif msg.get("state", "") == "uptodate":
                                slf.admin._n_watch_seen += 1  # noqa: SLF001
                                if slf.admin._n_watch_seen == slf.admin._n_watch:  # noqa: SLF001
                                    await slf.runner.send_event(ReadyMsg(slf.admin._n_watch_seen))  # noqa: SLF001

            def cancel(slf):
                if slf.scope is None:
                    return False
                sc, slf.scope = slf.scope, None
                sc.cancel()

        if isinstance(path, (tuple, list)):
            path = Path.build(path)
        elif not isinstance(path, Path):
            raise TypeError(f"You didn't pass in a path: {path!r}")

        kw.setdefault("max_depth", 0)
        if kw.setdefault("fetch", True):
            self._n_watch += 1

        w = Watcher(self, self._runner, self._client, cls, path, kw)
        self._taskgroup.start_soon(w.run)
        return w

    async def send(self, path, value=NotGiven, raw=None):
        """
        Publish an MQTT message.

        Set either ``value`` or ``raw``.
        """
        if isinstance(path, (tuple, list)):
            path = Path.build(path)
        elif not isinstance(path, Path):
            raise TypeError(f"You didn't pass in a path: {path!r}")

        await self._client.msg_send(topic=path, data=value, raw=raw)

    async def set(self, path, value, chain=NotGiven):
        """
        Set a MoaT-KV value.
        """
        if path.mark == "r":
            return await self.send(path, value)
        elif path.mark:
            raise RuntimeError(f"What should I do with a path marked {path.mark!r}")

        if isinstance(path, (tuple, list)):
            path = Path.build(path)
        elif not isinstance(path, Path):
            raise TypeError(f"You didn't pass in a path: {path!r}")

        return await self._client.set(path, value=value, chain=chain)

    async def get(self, path, value):
        """
        Get a MoaT-KV value.
        """
        if isinstance(path, (tuple, list)):
            path = Path.build(path)
        elif not isinstance(path, Path):
            raise TypeError(f"You didn't pass in a path: {path!r}")

        res = await self._client.get(path, value=value)  # TODO chain

        if "value" in res:
            return res.value
        else:
            return KeyError(path)

    async def monitor(self, path, cls=MQTTmsg, **kw):
        """
        Create an MQTT monitor.
        Messages are encapsulated in `MQTTmsg` objects.

        By default a monitor will only monitor a single entry. You may use
        MQTT wildcards.

        The message is decoded and stored in the ``value`` attribute unless
        it's either undecodeable or ``raw`` is set, in which case it's
        stored in ``.msg``. The topic the message was sent to is in
        ``topic``.
        """

        class Monitor:
            """Helper class for reading an MQTT topic"""

            def __init__(self, admin, runner, client, cls, path, kw):
                self.admin = admin
                self.runner = runner
                self.client = client
                self.path = path
                self.kw = kw
                self.cls = cls
                self.scope = None

            async def run(self):
                with anyio.CancelScope() as sc:
                    self.scope = sc
                    async with self.client.msg_monitor(path, **kw) as watcher:
                        async for msg in watcher:
                            if "topic" in msg:
                                # pylint:disable=attribute-defined-outside-init
                                chg = cls(msg.get("raw", None))
                                with suppress(AttributeError):
                                    chg.value = msg.data
                                chg.path = Path.build(msg.topic)
                                await self.runner.send_event(chg)

            async def cancel(self):
                if self.scope is None:
                    return False
                sc, self.scope = self.scope, None
                sc.cancel()

        if isinstance(path, (tuple, list)):
            path = Path.build(path)
        elif not isinstance(path, Path):
            raise TypeError(f"You didn't pass in a path: {path!r}")

        w = Monitor(self, self._runner, self._client, cls, path, kw)
        self._taskgroup.start_soon(w.run)
        return w

    async def timer(self, delay, cls=TimerMsg):  # noqa: D102
        class Timer:
            def __init__(self, runner, cls, tg):
                self.runner = runner
                self.cls = cls
                self.scope = None
                self._taskgroup = tg
                self.delay = None

            async def _run(self):
                with anyio.CancelScope() as sc:
                    self.scope = sc
                    await anyio.sleep(self.delay)
                    self.scope = None
                    await self.runner.send_event(self.cls(self))

            def cancel(self):
                if self.scope is None:
                    return False
                sc, self.scope = self.scope, None
                sc.cancel()
                return True

            async def run(self, delay):
                self.cancel()
                self.delay = delay
                if self.delay > 0:
                    self._taskgroup.start_soon(t._run)

        t = Timer(self._runner, cls, self._taskgroup)
        await t.run(delay)
        return t


class RunnerEntry(AttrClientEntry):
    """
    An entry representing some hopefully-running code.

    The code will run some time after ``target`` has passed.
    On success, it will run again ``repeat`` seconds later (if >0).
    On error, it will run ``delay`` seconds later (if >0), multiplied by 2**backoff.

    Arguments:
        code (list): pointer to the code that's to be started.
        data (dict): additional data for the code.
        delay (float): time before restarting the job on error.
            Default 100.
        repeat (float): time before restarting on success.
            Default: zero: no restart.
        target (float): time the job should be started at.
            Default: zero: don't start.
        ok_after (float): the job is marked OK if it has run this long.
            Default: zero: the code will do that itself.
        backoff (float): Exponential back-off factor on errors.
            Default: 1.1.

    The code runs with these additional keywords::

        _self: the `CallEnv` object, which the task can use to actually do things.
        _client: the MoaT-KV client connection.
        _info: a queue which the task can use to receive events. A message of
            ``None`` signals that the queue was overflowing and no further
            messages will be delivered. Your task should use that as its
            mainloop.
        _P: build a path from a string
        _Path: build a path from its arguments

    Some possible messages are defined in :mod:`moat.kv.actor`.
    """

    ATTRS = "code data delay ok_after repeat backoff target".split()

    delay = 100  # timedelta, before restarting
    repeat = 0
    target = 0
    backoff = 1.1
    ok_after = 0

    code = ()  # what to execute
    data = None
    scope = None  # scope to kill off
    _comment = None  # used for error entries, i.e. mainly Cancel
    _q = None  # send events to the running task. Async tasks only.
    _running = False  # .run is active. Careful with applying updates.
    _task = None
    retry = None

    def __init__(self, *a, **k):
        self.data = {}  # local data

        super().__init__(*a, **k)

        self._logger = logger_for(self._path)

    def __repr__(self):
        return f"<{self.__class__.__name__} {self.subpath!r}:{self.code!r}>"

    @property
    def state(self):  # noqa: D102
        return self.root.state.follow(self.subpath, create=None)

    async def run(self):  # noqa: D102
        if self.code is None:
            return  # nothing to do here

        state = self.state
        try:
            self._running = True
            try:
                self._logger.debug("Start")
                if state.node is not None:
                    raise RuntimeError(f"already running on {state.node}")
                code = self.root.code.follow(self.code, create=False)
                data = combine_dict(
                    self.data or {},
                    {} if code.value is NotGiven else code.value.get("default", {}),
                    deep=True,
                )

                if code.is_async:
                    data["_info"] = self._q = create_queue(QLEN)
                data["_client"] = self.root.client
                data["_cfg"] = self.root.client._cfg  # noqa: SLF001
                data["_cls"] = _CLASSES
                data["_P"] = P
                data["_Path"] = Path
                data["_log"] = self._logger
                data["_digits"] = digits

                state.started = time.time()
                state.node = state.root.name

                await state.save(wait=True)
                if state.node != state.root.name:
                    raise RuntimeError("Rudely taken away from us.", state.node, state.root.name)

                data["_self"] = calls = CallAdmin(self, state, data)
                res = await calls._run(code, data)  # noqa: SLF001

            except BaseException as exc:
                self._logger.info("Error: %r", exc)
                raise
            else:
                self._logger.debug("End")
            finally:
                self.scope = None
                self._q = None
                t = time.time()

        except ErrorRecorded:
            # record_error() has already been called
            self._comment = None
            state.backoff = min(state.backoff + 1, 20)

        except BaseException as exc:
            c, self._comment = self._comment, None
            with anyio.move_on_after(5, shield=True):
                r = await self.root.err.record_error(
                    "run",
                    self._path,
                    exc=exc,
                    data=self.data,
                    comment=c,
                )
                if r is not None:
                    await self.root.err.wait_chain(r.chain)
            if isinstance(exc, Exception):
                state.backoff = min(state.backoff + 1, 20)
            else:
                raise

        else:
            state.result = res
            state.backoff = 0
            await self.root.err.record_working("run", self._path)

        finally:
            with anyio.fail_after(2, shield=True):
                if state.node == state.root.name:
                    state.node = None
                self._running = False
                state.stopped = t

                if state.backoff > 0:
                    self.retry = t + (self.backoff**state.backoff) * self.delay
                else:
                    self.retry = None

                try:
                    await state.save()
                except ClosedResourceError:
                    pass
                except ServerError:
                    logger.exception("Could not save")

                await self.root.trigger_rescan()

    async def send_event(self, evt):
        """Send an event to the running process."""
        if self._q is None:
            if self._running:
                self._logger.info("Discarding %r", evt)
        elif self._q.qsize() < QLEN - 1:
            self._logger.debug("Event: %r", evt)
            await self._q.put(evt)
        elif self._q.qsize() == QLEN - 1:
            self._logger.warning("Queue full")
            await self._q.put(None)
            self._q = None
            self._logger.info("Discarding %r", evt)

    async def seems_down(self):  # noqa: D102
        state = self.state
        state.node = None
        if not self._running:
            await state.save(wait=True)

    async def set_value(self, value):
        """Process incoming value changes"""
        c = self.code
        await super().set_value(value)

        # Check whether running code needs to be killed off
        if self.scope is not None:
            if value is NotGiven or c is not self.code:
                # The code changed.
                self._comment = "Cancel: Code changed"
                self.scope.cancel()
            elif not getattr(self, "target", None):
                self._comment = "Cancel: target zeroed"
                self.scope.cancel()

        await self.root.trigger_rescan()

    async def seen_value(self):  # noqa: D102
        await super().seen_value()
        await self.root.trigger_rescan()

    async def run_at(self, t: float):
        """Next run at this time."""
        self.target = t
        if not self._running:
            await self.save()

    def should_start(self):
        """Tell whether this job might want to be started.

        Returns:
          ``False``: No, it's running (or has run and doesn't restart).
          ``0``: No, it should not start
          ``>0``: timestamp at which it should start, or should have started

        """

        state = self.state
        if self.code is None:
            return False, "no code"
        if state.node is not None:
            # not our responsibility!
            return None, "node set"
        if state.started and not state.stopped:
            raise RuntimeError("Running! should not be called")

        if self.target is None:
            return False, "no target"
        elif self.target > state.started:
            return self.target, "target > started"
        elif state.backoff:
            return state.stopped + self.delay * (self.backoff**state.backoff), "backoff"
        elif self.repeat:
            return state.stopped + self.repeat, "repeat"
        elif state.started and state.started > state.stopped:
            return False, "is started"
        else:
            return 0, "no target"

    def __hash__(self):
        return hash(self.subpath)

    def __eq__(self, other):
        other = getattr(other, "subpath", other)
        return self.subpath == other

    def __lt__(self, other):
        other = getattr(other, "subpath", other)
        return self.subpath < other

    @property
    def age(self):  # noqa: D102
        return time.time() - self.state.started


class RunnerNode:
    """
    Represents all nodes in this runner group.

    This is used for load balancing and such. TODO.
    """

    seen = 0
    load = 999

    def __new__(cls, root, name):  # noqa: D102
        try:
            self = root._nodes[name]  # noqa: SLF001
        except KeyError:
            self = object.__new__(cls)
            self.root = root
            self.name = name
            root._nodes[name] = self  # noqa: SLF001
        return self

    def __init__(self, *a, **k):
        pass


class StateEntry(AttrClientEntry):
    """
    This is the actual state associated with a RunnerEntry.
    It must only be managed by the node that actually runs the code.

    Arguments:
      started (float): timestamp when the job was last started
      stopped (float): timestamp when the job last terminated
      pinged (float): timestamp when the state was last verified by the runner
      result (Any): the code's return value
      node (str): the node running this code
      backoff (float): on error, the multiplier to apply to the restart timeout
      computed (float): computed start time
      reason (str): reason why (not) starting
    """

    ATTRS = "started stopped pinged computed reason result node backoff".split()

    started = 0  # timestamp
    stopped = 0  # timestamp
    pinged = 0  # timestamp
    node = None  # on which the code is currently running
    result = NotGiven
    backoff = 0
    computed = 0  # timestamp
    reason = ""

    @property
    def runner(self):  # noqa: D102
        return self.root.runner.follow(self.subpath, create=False)

    async def startup(self):  # noqa: D102
        try:
            self.runner  # noqa:B018
        except KeyError:
            # The code entry doesn't exist any more.
            await self.delete()
            return

        if self.node is None:
            return
        if self.node != self.root.name:
            self.root.runner.get_node(self.node)
            return

        self.stopped = time.time()
        self.node = None
        self.backoff = min(20, self.backoff + 1)
        await self.root.runner.err.record_error(
            "run",
            self.runner._path,  # noqa: SLF001
            message="Runner restarted",
        )
        await self.save()

    async def ping(self):  # noqa: D102
        if self.node != self.root.name:
            raise RuntimeError("Not our state!")
        self.pinged = time.time()
        await self.save()

    async def stale(self):  # noqa: D102
        self.stopped = time.time()
        node, self.node = self.node, None
        self.backoff = min(20, self.backoff + 2)
        await self.root.runner.err.record_error(
            "run",
            self.runner._path,  # noqa: SLF001
            message="Runner killed: {node} {state}",
            data={"node": node, "state": "offline" if self.stopped else "stale"},
        )
        await self.save()

    async def set_value(self, value):  # noqa: D102
        n = self.node

        await super().set_value(value)
        if not self.root.runner_ready:
            return
        try:
            run = self.runner
        except KeyError:
            return

        # side effect: add to the global node list
        if n is not None:
            run.root.get_node(n)

        # Check whether running code needs to be killed off
        if run.scope is None:
            return
        if self.node is not None and self.node == n:
            # Nothing changed.
            return
        elif self.node is None or n == self.root.runner.name:
            # Owch. Our job got taken away from us.
            run._comment = f"Cancel: Node set to {self.node!r}"  # noqa: SLF001
            run.scope.cancel()
        elif n is not None:
            logger.warning(
                "Runner %s at %r: running but node is %s",
                self.root.name,
                self.subpath,
                n,
            )

        await run.root.trigger_rescan()


class StateRoot(MirrorRoot):
    """Base class for handling the state of entries.

    This is separate from the RunnerRoot hierarchy because the latter may
    be changed by anybody while this subtree may only be affected by the
    actual runner. Otherwise we get interesting race conditions.
    """

    _runner = None

    @classmethod
    def child_type(cls, name):  # noqa: ARG003, D102
        return StateEntry

    @property
    def name(self):  # noqa: D102
        return self.client.client_name

    @property
    def runner(self):  # noqa: D102
        return self._runner()

    @property
    def runner_ready(self):  # noqa: D102
        r = self._runner
        if r is None:
            return False
        r = r()
        if r is None:
            return False
        return r.ready

    def set_runner(self, runner):  # noqa: D102
        self._runner = ref(runner)

    async def runner_running(self):  # noqa: D102
        for n in self.all_children:
            await n.startup()

    async def kill_stale_nodes(self, names):
        """States with node names in the "names" set are stale. Kill them."""
        for s in self.all_children:
            if s.node in names:
                await s.stale()

    _last_t = 0

    async def ping(self):  # noqa: D102
        t = time.time()
        if t - self._last_t >= abs(self._cfg["ping"]):
            self._last_t = t
            if self._cfg["ping"] > 0:
                val = self.value_or({}, Mapping)
                val["alive"] = t
                await self.update(val)
            else:
                await self.client.msg_send(
                    "run",
                    {"group": self.runner.group, "time": t, "node": self.name},
                )


class _BaseRunnerRoot(ClientRoot):
    """common code for AnyRunnerRoot and SingleRunnerRoot"""

    _trigger: anyio.abc.Event = None
    _run_now_task: anyio.abc.CancelScope = None
    _tagged: bool = True

    err = None
    code = None
    ready = False
    _run_next = 0
    node_history = None
    _start_delay = None
    state = None
    _act = None

    CFG = "runner"
    SUB = None

    def __init__(self, *a, _subpath, err=None, code=None, nodes=0, **kw):
        super().__init__(*a, **kw)
        self.err = err
        self.code = code
        self._nodes = {}
        self.n_nodes = nodes
        self._trigger = anyio.Event()
        self._x_subpath = _subpath

    @classmethod
    def child_type(cls, name):  # noqa: ARG003
        return RunnerEntry

    @classmethod
    async def as_handler(cls, client, subpath, cfg=None, **kw):  # pylint: disable=arguments-differ
        assert cls.SUB is not None
        if cfg is None:
            cfg_ = client._cfg["runner"]  # noqa: SLF001
        else:
            cfg_ = combine_dict(cfg, client._cfg["runner"])  # noqa: SLF001
        return await super().as_handler(client, subpath=subpath, _subpath=subpath, cfg=cfg_, **kw)

    async def run_starting(self):
        from .code import CodeRoot  # noqa: PLC0415
        from .errors import ErrorRoot  # noqa: PLC0415

        if self.err is None:
            self.err = await ErrorRoot.as_handler(self.client)
        if self.code is None:
            self.code = await CodeRoot.as_handler(self.client)

        await self._state_runner()
        self.state.set_runner(self)

        self.node_history = NodeList(0)
        self._start_delay = self._cfg["start_delay"]

        await super().run_starting()

    async def _state_runner(self):
        self.state = await StateRoot.as_handler(
            self.client,
            cfg=self._cfg,
            subpath=self._x_subpath,
            key="state",
        )

    @property
    def name(self):
        """my node name"""
        return self.client.client_name

    def get_node(self, name):  # noqa: ARG002
        """
        If the runner keeps track of "foreign" nodes, allocate them
        """
        return None

    async def running(self):
        self._tg.start_soon(self._run_actor)

        # the next block needs to be atomic
        self.ready = True

        await self.state.runner_running()
        await super().running()

    async def _run_actor(self):
        """
        This method is started as a long-lived task of the root as soon as
        the subtree's data are loaded.

        Its job is to control which tasks are started.
        """
        raise RuntimeError("You want to override me.")

    async def trigger_rescan(self):
        """Tell the _run_actor task to rescan our job list prematurely"""
        if self._trigger is not None:
            self._trigger.set()

    async def _run_now(self, evt=None):
        t_next = self._run_next
        with anyio.CancelScope() as sc:
            self._run_now_task = sc
            if evt is not None:
                evt.set()
            t = time.time()
            while True:
                if t_next > t:
                    with anyio.move_on_after(t_next - t):
                        await self._trigger.wait()
                        self._trigger = anyio.Event()

                t = time.time()
                t_next = t + 999
                for j in self.all_children:
                    d, r = j.should_start()
                    if d is None:
                        continue
                    if not d or d > t:
                        j.state.computed = d
                        j.state.reason = r
                        if self._tagged:
                            try:
                                await j.state.save()
                            except anyio.ClosedResourceError:  # owch
                                logger.error("Could not update state %r %r", j, j.state)
                                return
                        if d and t_next > d:
                            t_next = d
                        continue
                    self._tg.start_soon(j.run)
                    await anyio.sleep(self._start_delay)

    async def notify_actor_state(self, msg=None):
        """
        Notify all running jobs about a possible change in active status.
        """
        ac = len(self.node_history)
        if ac == 0 or msg is None:
            ac = BrokenState
        elif self.name in self.node_history and ac == 1:
            ac = DetachedState
        elif self._act is not None and ac >= self.n_nodes:  # TODO configureable
            ac = CompleteState
        else:
            ac = PartialState

        logger.debug("State %r %r", ac, msg)

        ac = ac(msg)
        for n in self.all_children:
            await n.send_event(ac)


class AnyRunnerRoot(_BaseRunnerRoot):
    """
    This class represents the root of a code runner. Its job is to start
    (and periodically restart, if required) the entry points stored under it.

    :class:`AnyRunnerRoot` tries to ensure that the code in question runs
    on one single cluster member. In case of a network split, the code will
    run once in each split areas until the split is healed.
    """

    SUB = "group"

    _stale_times = None
    tg = None
    seen_load = None
    _tagged: bool = False

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self.group = (
            P(self.client.config.server["root"]) + P(self._cfg["name"]) | "any" | self._path[-1]
        )

    def get_node(self, name):  # noqa: D102
        return RunnerNode(self, name)

    @property
    def max_age(self):
        """Timeout after which we really should have gotten another go"""
        # allow one go-around without being in the list; happens when the
        # bus is slow
        return self._act.cycle_time_max * (self._act.history_size + 2.5)

    async def _run_actor(self):
        """
        Monitor the Actor state, run a :meth:`_run_now` subtask whenever we're 'it'.
        """
        async with ClientActor(
            self.client,
            self.name,
            topic=self.group,
            cfg=self._cfg["actor"],
        ) as act:
            self._act = act

            age_q = create_queue(10)
            await self.spawn(self._age_killer, age_q)

            psutil.cpu_percent(interval=None)
            await act.set_value(0)
            self.seen_load = None

            async for msg in act:
                logger.debug("Actor %r", msg)
                if isinstance(msg, PingEvent):
                    await act.set_value(100 - psutil.cpu_percent(interval=None))

                    node = self.get_node(msg.node)
                    node.load = msg.value
                    node.seen = time.time()
                    if self.seen_load is not None:
                        self.seen_load += msg.value
                    self.node_history += node

                elif isinstance(msg, TagEvent):
                    self._tagged = True
                    load = 100 - psutil.cpu_percent(interval=None)
                    await act.set_value(load)
                    if self.seen_load is not None:
                        pass  # TODO

                    self.get_node(self.name).seen = time.time()
                    self.node_history += self.name
                    evt = anyio.Event()
                    await self.spawn(self._run_now, evt)
                    await age_q.put(None)
                    await evt.wait()

                    await self.state.ping()

                elif isinstance(msg, UntagEvent):
                    self._tagged = False
                    await act.set_value(100 - psutil.cpu_percent(interval=None))
                    self.seen_load = 0

                    self._run_now_task.cancel()
                    # TODO if this is a DetagEvent, kill everything?

                await self.notify_actor_state(msg)

            pass  # end of actor task

    async def find_stale_nodes(self, cur):
        """
        Find stale nodes (i.e. last seen < cur) and clean them.
        """
        if self._stale_times is None:
            self._stale_times = []
        self._stale_times.append(cur)
        if self._stale_times[0] > cur - self.max_age:
            return
        if len(self._stale_times) <= 2 * self._cfg["actor"]["n_hosts"] + 1:
            return
        cut = self._stale_times.pop(0)

        dropped = []
        for node in self._nodes.values():
            if node.seen < cut:
                dropped.append(node)
        names = set()
        for node in dropped:
            names.add(node.name)
            del self._nodes[node.name]
        if names:
            await self.state.kill_stale_nodes(names)

    async def _age_killer(self, age_q):
        """
        Subtask which cleans up stale tasks

        TODO check where to use time.monotonic
        """
        t0 = None
        t1 = time.time()
        while True:
            with anyio.move_on_after(self.max_age):
                await age_q.get()
                t1 = time.time()
            t2 = time.time()
            if t1 + self.max_age < t2:
                raise NotSelected(self.max_age, t2, t1)
            t0 = t1
            t1 = t2
            if t0 is not None:
                await self.find_stale_nodes(t0)


class SingleRunnerRoot(_BaseRunnerRoot):
    """
    This class represents the root of a code runner. Its job is to start
    (and periodically restart, if required) the entry points stored under it.

    While :class:`AnyRunnerRoot` tries to ensure that the code in question runs
    on any cluster member, this class runs tasks on a single node.
    The code is able to check whether any and/or all of the cluster's main
    nodes are reachable; this way, the code can default to local operation
    if connectivity is lost.

    Local data (dict):

    Arguments:
      cores (tuple): list of nodes whose reachability may determine
        whether the code uses local/emergency/??? mode.

    Config file:

    Arguments:
      path (tuple): the location this entry is stored at. Defaults to
        ``('.moat', 'kv', 'process')``.
      name (str): this runner's name. Defaults to the client's name plus
        the name stored in the root node, if any.
      actor (dict): the configuration for the underlying actor. See
        ``asyncactor`` for details.
    """

    SUB = "single"

    err = None
    code = None
    state = None
    tg = None

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self.group = (
            P(self.client.config.server["root"]) + P(self._cfg["name"])
            | "single"
            | self._path[-2]
            | self._path[-1]
        )

    async def set_value(self, value):  # noqa: D102
        await super().set_value(value)
        try:
            cores = value["cores"]
        except (TypeError, KeyError):
            if self._act is not None:
                await self._act.disable(0)
        else:
            if self.name in cores:
                await self._act.enable(len(cores))
            else:
                await self._act.disable(len(cores))

    @property
    def max_age(self):
        """Timeout after which we really should have gotten another ping"""
        return self._act.cycle_time_max * 1.5

    async def _run_actor(self):
        async with anyio.create_task_group() as tg:
            age_q = create_queue(1)

            async with ClientActor(self.client, self.name, topic=self.group, cfg=self._cfg) as act:
                self._act = act
                tg.start_soon(self._age_notifier, age_q)
                await self.spawn(self._run_now)
                await act.set_value(0)

                async for msg in act:
                    if isinstance(msg, AuthPingEvent):
                        # Some node, not necessarily us, is "it".
                        # We're the SingleNode runner: we manage our jobs
                        # when triggered by this, no matter whether we're
                        # "it" or not.
                        self.node_history += msg.node
                        await age_q.put(None)
                        await self.notify_actor_state(msg)

                        await self.state.ping()

                pass  # end of actor task

    async def _age_notifier(self, age_q):
        """
        This background job triggers :meth:`notify_actor_state` when too much
        time has passed without an AuthPing.
        """
        while True:
            try:
                with anyio.fail_after(self.max_age):
                    await age_q.get()
            except TimeoutError:
                await self.notify_actor_state()


class AllRunnerRoot(SingleRunnerRoot):
    """
    This class represents the root of a code runner. Its job is to start
    (and periodically restart, if required) the entry points stored under it.

    This class behaves like `SingleRunner`, except that it runs tasks on all nodes.
    """

    SUB = "all"

    err = None
    _act = None
    code = None

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self.group = (
            P(self.client.config.server["root"]) + P(self._cfg["name"]) | "all" | self._path[-1]
        )

    async def _state_runner(self):
        self.state = await StateRoot.as_handler(
            self.client,
            cfg=self._cfg,
            subpath=self._x_subpath + (self.client.client_name,),
            key="state",
        )
