"""
This module's job is to run code, resp. to keep it running.


"""

import anyio
from weakref import ref
from asyncactor import NodeList
from asyncactor import PingEvent, TagEvent, UntagEvent, AuthPingEvent
from copy import deepcopy
import psutil
import time
from collections.abc import Mapping
try:
    from contextlib import AsyncExitStack, asynccontextmanager
except ImportError:
    from async_exit_stack import AsyncExitStack, asynccontextmanager

from .actor import ClientActor
from .actor import DetachedState, PartialState, CompleteState, ActorState, BrokenState
from .util import NotGiven, combine_dict, attrdict

from .exceptions import ServerError
from .obj import AttrClientEntry, ClientRoot

import logging

logger = logging.getLogger(__name__)

QLEN = 10

class NotSelected(RuntimeError):
    """
    This node has not been selected for a very long time. Something is amiss.
    """

    pass


class ErrorRecorded(RuntimeError):
    pass


class RunnerMsg(ActorState):
    """Superclass for runner-generated messages.

    Not directly instantiated.
    """
    pass

class ChangeMsg(RunnerMsg):
    """A message telling your code that some entry has been updated.

    Subclass this and use it as `CallAdmin.watch`'s ``cls`` parameter for easier
    disambiguation.
    """
    pass


class ReadyMsg(RunnerMsg):
    """
    This message is queued when the last watcher has read all data.
    """
    pass

class TimerMsg(RunnerMsg):
    """
    A message telling your code that a timer triggers.

    Subclass this and use it as `CallAdmin.timer`'s ``cls`` parameter for easier
    disambiguation.
    """
    pass


_CLASSES = attrdict()
for _c in (DetachedState, PartialState, CompleteState, ActorState,
        BrokenState, NotGiven, TimerMsg,ReadyMsg,ChangeMsg,RunnerMsg,ErrorRecorded):
    _CLASSES[_c.__name__] = _c


class CallAdmin:
    """
    This class collects some standard tasks which async DistKV-embedded
    code might want to do.
    """
    def __init__(self, runner, state, data):
        self._runner = runner
        self._state = state
        self._data = data
        self._client = runner.root.client
        self._err = runner.root.err
        self._q = runner._q
        self._path = runner._path
        self._subpath = runner.subpath
        self._n_watch = 0

    async def _run(self, code, data):
        """Called by the runner to actually execute the code."""
        logger.debug("Start %r with %r", self._runner._path, self._runner.code)
        async with anyio.create_task_group() as tg:
            self._taskgroup = tg
            async with AsyncExitStack() as stack:
                self._stack = stack
                sc = tg.cancel_scope

                self._taskgroup = tg
                self._runner.scope = sc
                data["_self"] = self

                oka = getattr(self._runner, "ok_after", 0)
                if oka > 0:
                    async def is_ok(oka):
                        await anyio.sleep(oka)
                        await self.setup_done()
                    await tg.spawn(is_ok, oka)

                res = code(**data)
                if code.is_async is not None:
                    res = await res

                await sc.cancel()
                return res

    async def spawn(self, proc, *a, **kw):
        """
        Start a background subtask.

        The task is auto-cancelled when your code ends.

        Returns: an `anyio.abc.CancelScope` which you can use to cancel the
            subtask.
        """
        async def _spawn(evt,proc,a,kw):
            nonlocal scope
            async with anyio.open_cancel_scope() as scope:
                await evt.set()
                await proc(*a,**kw)

        scope = None
        evt = anyio.create_event()
        await self._taskgroup.spawn(_spawn,evt,proc,a,kw)
        await evt.wait()
        return scope

    async def setup_done(self, **kw):
        """
        Call this when your code has successfully started up.
        """
        self._state.backoff = 0
        await self._state.save()
        await self._err.record_working("run", *self._runner._path, **kw)

    async def error(self, **kw):
        """
        Record that an error has occurred. This function records specific
        error data, then raises `ErrorRecorded` which the code is not
        supposed to catch.

        See `distkv.errors.ErrorRoot.record_error` for keyword details.
        """
        r = await self._err.record_error("run", *self._path, **kw)
        await self._err.root.wait_chain(r.chain)
        raise ErrorRecorded()

    async def open_context(self, ctx):
        return await self._stack.enter_async_context(ctx)

    async def watch(self, *path, cls=ChangeMsg, **kw):
        """
        Create a watcher. This path is monitored as per `distkv.client.Client.watch`;
        messages are encapsulated in `ChangeMsg` objects.
        A `ReadyMsg` will be sent when all watchers have transmitted their
        initial state.

        By default a watcher will only monitor a single entry. Set
        ``max_depth`` if you also want child entries.
        """
        class Watcher:
            def __init__(self, admin, runner, client, cls, path, kw):
                self.admin = admin
                self.runner = runner
                self.client = client
                self.path = path
                self.kw = kw
                self.cls = cls
                self.scope = None

            async def run(self):
                async with anyio.open_cancel_scope() as sc:
                    self.scope = sc
                    async with self.client.watch(*path,**kw) as watcher:
                        async for msg in watcher:
                            if 'path' in msg:
                                await self.runner.send_event(cls(msg))
                            elif msg.get('state','') == "uptodate":
                                self.admin._n_watch -= 1
                                if not self.admin._n_watch:
                                    await self.runner.send_event(ReadyMsg(msg))

            async def cancel(self):
                if self.scope is None:
                    return False
                sc, self.scope = self.scope, None
                await sc.cancel()

        if kw.setdefault('fetch', True):
            self._n_watch += 1

        w = Watcher(self, self._runner, self._client, cls, path, kw)
        await self._taskgroup.spawn(w.run)
        return w

    async def timer(self, delay, cls=TimerMsg):
        class Timer:
            def __init__(self, runner, delay, cls):
                self.runner = runner
                self.delay = delay
                self.cls = cls
                self.scope = None

            async def run(self):
                async with anyio.open_cancel_scope() as sc:
                    self.scope = sc
                    await anyio.sleep(delay)
                    await self.runner.send_event(self.cls())
                    self.scope = None

            async def cancel(self):
                if self.scope is None:
                    return False
                sc, self.scope = self.scope, None
                await sc.cancel()
                return True
        t = Timer(self._runner, delay, cls)
        await self._taskgroup.spawn(t.run)
        return t


class RunnerEntry(AttrClientEntry):
    """
    An entry representing some hopefully-running code.

    The code will run some time after ``target`` has passed.
    On success, it will run again ``repeat`` seconds later (if >0).
    On error, it will run ``delay`` seconds later (if >0), multiplied by 2**backoff.

    Arguments:
        code (list): pointer to the code that's to be started.
        data (dict): additional data for the code.
        delay (float): time before restarting the job on error.
            Default 100.
        repeat (float): time before restarting on success.
            Default: zero: no restart.
        target (float): time the job should be started at.
            Default: zero: don't start.
        ok_after (float): the job is marked OK if it has run this long.
            Default: zero: the code will do that itself.
        backoff (float): Exponential back-off factor on errors.
            Default: 1.4.

    The code runs with these additional keywords::

        _self: the `CallEnv` object, which the task can use to actually do things.
        _client: the DistKV client connection.
        _info: a queue which the task can use to receive events. A message of
            ``None`` signals that the queue was overflowing and no further
            messages will be delivered. Your task should use that as its
            mainloop.

    Some possible messages are defined in :mod:`distkv.actor`.
    """

    ATTRS = "code data delay ok_after repeat backoff target".split()

    delay = 100  # timedelta, before restarting
    repeat = 0
    target = 0
    backoff = 1.4
    ok_after = 0

    code = ()  # what to execute
    data = None
    scope = None  # scope to kill off
    _comment = None  # used for error entries, i.e. mainly Cancel
    _q = None  # send events to the running task. Async tasks only.
    _running = False  # .run is active. Careful with applying updates.
    _task = None
    retry = None

    def __init__(self, *a, **k):
        self.data = {}  # local data

        super().__init__(*a, **k)

    def __repr__(self):
        return "<%s %r:%r>" % (self.__class__.__name__, self.subpath, self.code)

    @property
    def state(self):
        return self.root.state.follow(*self.subpath, create=None)

    async def run(self):
        if self.code is None:
            return  # nothing to do here

        state = self.state
        try:
            self._running = True
            try:
                if state.node is not None:
                    raise RuntimeError("already running on %s" % (state.node,))
                code = self.root.code.follow(*self.code, create=False)
                data = self.data
                if data is None:
                    data = {}
                else:
                    data = deepcopy(data)

                if code.is_async:
                    data["_info"] = self._q = anyio.create_queue(QLEN)
                data["_client"] = self.root.client
                data["_cfg"] = self.root.client._cfg
                data["_cls"] = _CLASSES

                state.started = time.time()
                state.node = state.root.name

                await state.save(wait=True)
                if state.node != state.root.name:
                    raise RuntimeError(
                        "Rudely taken away from us.", state.node, state.root.name
                    )

                data["_self"] = calls = CallAdmin(self,state,data)
                res = await calls._run(code, data)

            finally:
                logger.debug("End %r", self._path)
                self._taskgroup = None
                self.scope = None
                self._q = None
                t = time.time()

        except ErrorRecorded:
            # record_error() has already been called
            self._comment = None
            state.backoff += 1

        except BaseException as exc:
            c, self._comment = self._comment, None
            async with anyio.move_on_after(2, shield=True):
                r = await self.root.err.record_error(
                    "run",
                    *self._path,
                    message="Exception",
                    exc=exc,
                    data=self.data,
                    comment=c
                )
                await self._err.root.wait_chain(r.chain)
            state.backoff += 1
        else:
            state.result = res
            state.backoff = 0
            await self.root.err.record_working("run", *self._path)
        finally:
            async with anyio.fail_after(2, shield=True):
                if state.node == state.root.name:
                    state.node = None
                self._running = False
                state.stopped = t

                if state.backoff > 0:
                    self.retry = t + (self.backoff ** state.backoff) * self.delay
                else:
                    self.retry = None
                async with anyio.move_on_after(2, shield=True):
                    try:
                        await state.save()
                    except anyio.exceptions.ClosedResourceError:
                        pass
                    except ServerError:
                        logger.exception("Could not save")
                    await self.root.trigger_rescan()

    async def send_event(self, evt):
        """Send an event to the running process."""
        if self._q is None:
            return
        if self._q.qsize() < QLEN - 1:
            await self._q.put(evt)
        elif self._q.qsize() == QLEN - 1:
            await self._q.put(None)
            self._q = None

    async def seems_down(self):
        state = self.state
        state.node = None
        if not self._running:
            await state.save(wait=True)

    async def set_value(self, value):
        """Process incoming value changes"""
        c = self.code
        await super().set_value(value)

        # Check whether running code needs to be killed off
        if self.scope is not None:
            if value is NotGiven or c is not self.code:
                # The code changed.
                self._comment = "Cancel: Code changed"
                await self.scope.cancel()
            elif not getattr(self, "target", None):
                self._comment = "Cancel: target zeroed"
                await self.scope.cancel()

        await self.root.trigger_rescan()

    async def seen_value(self):
        await super().seen_value()
        await self.root.trigger_rescan()

    async def run_at(self, t: float):
        """Next run at this time.
        """
        self.target = t
        if not self._running:
            await self.save()

    def should_start(self):
        """Tell whether this job might want to be started.

        Returns:
          ``False``: No, it's running (or has run and doesn't restart).
          n>0: wait for n seconds before thinking again.
          n<0: should have been started n seconds ago, do something!

        """

        state = self.state
        if self.code is None:
            return False
        if state.node is not None:
            return False
        if state.started and not state.stopped:
            raise RuntimeError("Running! should not be called")

        if self.target > state.started:
            return self.target
        elif state.backoff:
            return state.stopped + self.delay * (self.backoff ** state.backoff)
        elif self.repeat:
            return state.stopped + self.repeat
        elif state.started:
            return False
        else:
            return 0

    def __hash__(self):
        return hash(self.subpath)

    def __eq__(self, other):
        other = getattr(other, "subpath", other)
        return self.subpath == other

    def __lt__(self, other):
        other = getattr(other, "subpath", other)
        return self.subpath < other

    @property
    def age(self):
        return time.time() - self.state.started


class RunnerNode:
    """
    Represents all nodes in this runner group.

    This is used for load balancing and such. TODO.
    """

    seen = 0
    load = 999

    def __new__(cls, root, name):
        try:
            self = root._nodes[name]
        except KeyError:
            self = object.__new__(cls)
            self.root = root
            self.name = name
            root._nodes[name] = self
        return self

    def __init__(self, *a, **k):
        pass


class StateEntry(AttrClientEntry):
    """
    This is the actual state associated with a RunnerEntry.
    It must only be managed by the node that actually runs the code.

    Arguments:
      started (float): timestamp when the job was last started
      stopped (float): timestamp when the job last terminated
      result (Any): the code's return value
      node (str): the node running this code
      backoff (float): on error, the multiplier to apply to the restart timeout
    """

    ATTRS = "started stopped result node backoff".split()

    started = 0  # timestamp
    stopped = 0  # timestamp
    node = None  # on which the code is currently running
    result = NotGiven
    backoff = 0

    @property
    def runner(self):
        return self.root.runner.follow(*self.subpath, create=False)

    async def startup(self):
        try:
            self.runner
        except KeyError:
            # The code entry doesn't exist any more.
            await self.delete()
            return

        if self.node is None or self.node != self.root.name:
            return

        self.stopped = time.time()
        self.node = None
        self.backoff += 1
        await self.root.runner.err.record_error(
            "run", *self.runner._path, message="Runner restarted"
        )
        await self.save()

    async def stale(self):
        self.stopped = time.time()
        self.node = None
        self.backoff += 2
        await self.root.runner.err.record_error(
            "run",
            *self.runner._path,
            client_name=self.node,
            message="Runner {node} {state}",
            data={"node": self.node, "state": "offline" if self.stopped else "stale"}
        )
        await self.save()

    async def set_value(self, value):
        n = self.node

        await super().set_value(value)
        if not self.root.runner.ready:
            return
        try:
            run = self.runner
        except KeyError:
            return

        # Check whether running code needs to be killed off
        if run.scope is None:
            return
        if self.node is not None and self.node == n:
            # Nothing changed.
            return
        elif self.node is None or n == self.root.runner.name:
            # Owch. Our job got taken away from us.
            run._comment = "Cancel: Node set to %r" % (self.node,)
            await run.scope.cancel()
        elif n is not None:
            logger.warning(
                "Runner %s at %r: running but node is %s",
                self.root.name,
                self.subpath,
                n,
            )

        await run.root.trigger_rescan()


class StateRoot(ClientRoot):
    """Base class for handling the state of entries.

    This is separate from the RunnerRoot hierarchy because the latter may
    be changed by anybody while this subtree may only be affected by the
    actual runner. Otherwise we get interesting race conditions.
    """

    _runner = None

    @classmethod
    def child_type(cls, name):
        return StateEntry

    @property
    def name(self):
        return self.client.client_name

    @property
    def runner(self):
        return self._runner()

    def set_runner(self, runner):
        self._runner = ref(runner)

    async def runner_running(self):
        for n in self.all_children:
            await n.startup()

    async def kill_stale_nodes(self, names):
        """States with node names in the "names" set are stale. Kill them."""
        for s in self.all_children:
            if s.node in names:
                await s.stale()

    _last_t = 0

    async def ping(self):

        t = time.time()
        if t - self._last_t >= abs(self._cfg["ping"]):
            self._last_t = t
            if self._cfg["ping"] > 0:
                val = self.value_or({}, Mapping)
                val["alive"] = t
                await self.update(val)
            else:
                await self.client.msg_send(
                    "run", {"group": self.runner.group, "time": t, "node": self.name}
                )


class _BaseRunnerRoot(ClientRoot):
    """common code for AnyRunnerRoot and SingleRunnerRoot"""

    _trigger: anyio.abc.Event = None
    _run_now_task: anyio.abc.CancelScope = None

    err = None
    code = None
    ready = False
    _run_next = 0
    node_history = None
    _start_delay = None
    state = None

    CFG = "runner"
    SUB = None

    def __init__(self, *a, _subpath, err=None, code=None, **kw):
        super().__init__(*a, **kw)
        self.err = err
        self.code = code
        self._nodes = {}
        self._trigger = anyio.create_event()
        self._x_subpath = _subpath

    @classmethod
    def child_type(cls, name):
        return RunnerEntry

    @classmethod
    async def as_handler(
        cls, client, subpath, cfg=None, **kw
    ):  # pylint: disable=arguments-differ
        assert cls.SUB is not None
        if cfg is None:
            cfg_ = client._cfg["runner"]
        else:
            cfg_ = combine_dict(cfg, client._cfg["runner"])
        return await super().as_handler(
            client, subpath=subpath, _subpath=subpath, cfg=cfg_, **kw
        )

    async def run_starting(self):
        from .errors import ErrorRoot
        from .code import CodeRoot

        if self.err is None:
            self.err = await ErrorRoot.as_handler(self.client)
        if self.code is None:
            self.code = await CodeRoot.as_handler(self.client)

        await self._state_runner()
        self.state.set_runner(self)

        self.node_history = NodeList(0)
        self._start_delay = self._cfg["start_delay"]

        await super().run_starting()

    async def _state_runner(self):
        self.state = await StateRoot.as_handler(
            self.client, cfg=self._cfg, subpath=self._x_subpath, key="state"
        )

    @property
    def name(self):
        """my node name"""
        return self.client.client_name

    def get_node(self, name):  # pylint: disable=unused-argument
        """
        If the runner keeps track of "foreign" nodes, allocate them
        """
        return None

    async def running(self):
        await self._tg.spawn(self._run_actor)

        # the next block needs to be atomic
        for n in self.all_children:
            print(n)
        self.ready = True

        await self.state.runner_running()
        await super().running()

    async def _run_actor(self):
        """
        This method is started as a long-lived task of the root as soon as
        the subtree's data are loaded.

        Its job is to control which tasks are started.
        """
        raise RuntimeError("You want to override me." "")

    async def trigger_rescan(self):
        """Tell the _run_actor task to rescan our job list prematurely"""
        if self._trigger is not None:
            await self._trigger.set()

    async def _run_now(self, evt=None):
        t_next = self._run_next
        async with anyio.open_cancel_scope() as sc:
            self._run_now_task = sc
            if evt is not None:
                await evt.set()
            t = time.time()
            while True:
                if t_next > t:
                    async with anyio.move_on_after(t_next - t):
                        await self._trigger.wait()
                        self._trigger = anyio.create_event()

                t = time.time()
                t_next = t + 99999
                for j in self.all_children:
                    d = j.should_start()
                    if d is False:
                        continue
                    if d <= t:
                        await self._tg.spawn(j.run)
                        await anyio.sleep(self._start_delay)
                    elif t_next > d:
                        t_next = d

    async def notify_actor_state(self, msg=None):
        """
        Notify all running jobs about a possible change in active status.
        """
        ac = len(self.node_history)
        if ac == 0 or msg is None:
            ac = BrokenState
        elif self.name in self.node_history and ac == 1:
            ac = DetachedState
        elif ac >= self._act.n_nodes:
            ac = CompleteState
        else:
            ac = PartialState

        logger.debug("State %r %r",ac,msg)

        ac = ac(msg)
        for n in self.all_children:
            await n.send_event(ac)



class AnyRunnerRoot(_BaseRunnerRoot):
    """
    This class represents the root of a code runner. Its job is to start
    (and periodically restart, if required) the entry points stored under it.

    :class:`AnyRunnerRoot` tries to ensure that the code in question runs
    on one single cluster member. In case of a network split, the code will
    run once in each split areas until the split is healed.
    """

    SUB = "group"

    _stale_times = None
    _act = None
    tg = None
    seen_load = None

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self.group = "run." + self._name

    def get_node(self, name):
        return RunnerNode(self, name)

    @property
    def max_age(self):
        """Timeout after which we really should have gotten another go"""
        return self._act.cycle_time_max * (self._act.history_size + 1.5)

    async def _run_actor(self):
        """
        Monitor the Actor state, run a :meth:`_run_now` subtask whenever we're 'it'.
        """
        async with anyio.create_task_group() as tg:
            async with ClientActor(
                self.client, self.name, topic=self.group, cfg=self._cfg["actor"]
            ) as act:
                self._act = act

                age_q = anyio.create_queue(10)
                await self.spawn(self._age_killer, age_q)

                psutil.cpu_percent(interval=None)
                await act.set_value(0)
                self.seen_load = None

                async for msg in act:
                    logger.debug("Actor %r", msg)
                    if isinstance(msg, PingEvent):
                        await act.set_value(100 - psutil.cpu_percent(interval=None))

                        node = self.get_node(msg.node)
                        node.load = msg.value
                        node.seen = time.time()
                        if self.seen_load is not None:
                            self.seen_load += msg.value
                        self.node_history += node

                    elif isinstance(msg, TagEvent):
                        load = 100 - psutil.cpu_percent(interval=None)
                        await act.set_value(load)
                        if self.seen_load is not None:
                            pass  # TODO

                        self.node_history += self.name
                        evt = anyio.create_event()
                        await self.spawn(self._run_now, evt)
                        await age_q.put(None)
                        await evt.wait()

                        await self.state.ping()

                    elif isinstance(msg, UntagEvent):
                        await act.set_value(100 - psutil.cpu_percent(interval=None))
                        self.seen_load = 0

                        await self._run_now_task.cancel()
                        # TODO if this is a DetagEvent, kill everything?

                    await self.notify_actor_state(msg)

                pass  # end of actor task

    async def find_stale_nodes(self, cur):
        """
        Find stale nodes (i.e. last seen < cur) and clean them.
        """
        if self._stale_times is None:
            self._stale_times = []
        self._stale_times.append(cur)
        if self._stale_times[0] > cur - self.max_age:
            return
        if len(self._stale_times) < 5:
            return
        cut = self._stale_times.pop(0)

        dropped = []
        for node in self._nodes.values():
            if node.seen < cut:
                dropped.append(node)
        names = set()
        for node in dropped:
            names.add(node.name)
            del self._nodes[node.name]
        if names:
            await self.state.kill_stale_nodes(names)

    async def _age_killer(self, age_q):
        """
        Subtask which cleans up stale tasks

        TODO check where to use time.monotonic
        """
        t1 = time.time()
        while age_q is not None:
            await self.find_stale_nodes(t1)
            async with anyio.move_on_after(self.max_age):
                await age_q.get()
                t1 = time.time()
                continue
            t2 = time.time()
            if t1 + self.max_age < t2:
                raise NotSelected(self.max_age, t2, t1)
            t1 = t2

    async def _cleanup_nodes(self):
        t = time.time()
        while len(self.node_history) > 1:
            node = self.get_node(self.node_history[-1])
            if t - node.seen < self.max_age:
                break
            assert node.name == self.node_history.pop()
            for j in self.all_children:
                if j.node == node.name:
                    await j.seems_down()


class SingleRunnerRoot(_BaseRunnerRoot):
    """
    This class represents the root of a code runner. Its job is to start
    (and periodically restart, if required) the entry points stored under it.

    While :class:`AnyRunnerRoot` tries to ensure that the code in question runs
    on any cluster member, this class runs tasks on a single node.
    The code is able to check whether any and/or all of the cluster's main
    nodes are reachable; this way, the code can default to local operation
    if connectivity is lost.

    Local data (dict):

    Arguments:
      cores (tuple): list of nodes whose reachability may determine
        whether the code uses local/emergency/??? mode.

    Config file:

    Arguments:
      path (tuple): the location this entry is stored at. Defaults to
        ``('.distkv', 'process')``.
      name (str): this runner's name. Defaults to the client's name plus
        the name stored in the root node, if any.
      actor (dict): the configuration for the underlying actor. See
        ``asyncactor`` for details.
    """

    SUB = "single"

    err = None
    _act = None
    code = None
    state = None
    tg = None

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self.group = "run." + self._path[-2]

    async def set_value(self, value):
        await super().set_value(value)
        try:
            cores = value["cores"]
        except (TypeError, KeyError):
            if self._act is not None:
                await self._act.disable(0)
        else:
            if self.name in cores:
                await self._act.enable(len(cores))
            else:
                await self._act.disable(len(cores))

    @property
    def max_age(self):
        """Timeout after which we really should have gotten another ping"""
        return self._act.cycle_time_max * 1.5

    async def _run_actor(self):
        async with anyio.create_task_group() as tg:
            age_q = anyio.create_queue(1)

            async with ClientActor(
                self.client, self.name, topic=self.group, cfg=self._cfg
            ) as act:
                self._act = act
                await tg.spawn(self._age_notifier, age_q)
                await self.spawn(self._run_now)
                await act.set_value(0)

                async for msg in act:
                    if isinstance(msg, AuthPingEvent):
                        # Some node, not necessarily us, is "it".
                        # We're the SingleNode runner: we manage our jobs
                        # when triggered by this, no matter whether we're
                        # "it" or not.
                        self.node_history += msg.node
                        await age_q.put(None)
                        await self.notify_actor_state(msg)

                        await self.state.ping()

                pass  # end of actor task

    async def _age_notifier(self, age_q):
        """
        This background job triggers :meth:`notify_actor_state` when too much
        time has passed without an AuthPing.
        """
        while True:
            try:
                async with anyio.fail_after(self.max_age):
                    await age_q.get()
            except TimeoutError:
                await self.notify_actor_state()


class AllRunnerRoot(SingleRunnerRoot):
    """
    This class represents the root of a code runner. Its job is to start
    (and periodically restart, if required) the entry points stored under it.

    This class behaves like `SingleRunner`, except that it runs tasks on all nodes.
    """

    SUB = "all"

    err = None
    _act = None
    code = None

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self.group = "run." + self.name

    async def _state_runner(self):
        self.state = await StateRoot.as_handler(
            self.client,
            cfg=self._cfg,
            subpath=self._x_subpath + (self.client.client_name,),
            key="state",
        )
