"""
The MoaT-Link client
"""

from __future__ import annotations

import anyio
import logging
import os
import platform
import sys
import time
from contextlib import asynccontextmanager, nullcontext, suppress
from contextvars import ContextVar
from functools import partial
from traceback import format_exception

import outcome
from attrs import define, field

from moat.util import (
    CtxObj,
    NotGiven,
    ValueEvent,
    attrdict,
    ctx_as,
    gen_ident,
    merge,
    srepr,
    timed_ctx,
    to_attrdict,
    ungroup,
)
from moat.lib.config import CFG
from moat.lib.mqtt import QoS, RetainHandling
from moat.lib.path import (
    P,
    Path,
    PathLongener,
    Root,
)
from moat.lib.rpc import Caller, MsgSender
from moat.util.random import al_unique

from .auth import AnonAuth, TokenAuth
from .common import CmdCommon
from .conn import TCPConn, UnixConn
from .exceptions import AuthError, ClientCancelledError
from .hello import Hello
from .meta import MsgMeta
from .node import Node

from typing import TYPE_CHECKING, overload

try:
    from .schema import schema_path, validate_instance
except ImportError:
    _HAVE_SCHEMA = False

    def schema_path(_path: Path) -> Path:
        raise RuntimeError("Schema support is not installed")

    def validate_instance(_schema: object, _data: object) -> None:
        raise RuntimeError("Schema support is not installed")

else:
    _HAVE_SCHEMA = True

if TYPE_CHECKING:
    from contextlib import AbstractAsyncContextManager

    from moat.lib.path import PathElem
    from moat.lib.rpc import Msg
    from moat.link.code.run import Code
    from moat.link.node.codec import CodecNode

    from .auth import AuthMethod
    from .backend import Backend, Message

    from collections.abc import AsyncIterator, Awaitable, Callable
    from typing import Any, Literal, Protocol

    class DataSender(Protocol):
        "Protocol for `d.*` RPC sender helpers."

        get: Callable[[Path], Awaitable[Any]]
        search: Callable[[Path], Awaitable[Any]]
        set: Callable[[Path, Any, MsgMeta], Awaitable[Any]]
        walk: Callable[..., Caller]

    class ErrSender(Protocol):
        "Protocol for `e.*` RPC sender helpers."

        exc: Callable[..., Awaitable[Any]]
        info: Callable[..., Awaitable[Any]]
        ack: Callable[..., Awaitable[Any]]
        ok: Callable[..., Awaitable[Any]]
        mon: Callable[..., AbstractAsyncContextManager[Any]]


NotGivenType = type(NotGiven)


class _Requeue(Exception):
    pass


__all__ = [
    "BasicLink",
    "ClientCaller",
    "CodeCaller",
    "Link",
    "LinkCommon",
    "LinkSender",
    "get_link",
]

_the_link = ContextVar("_the_link")

local_addrs = {"localhost", "127.0.0.1", "::1"}


def get_link() -> Link | None:
    """
    Return the global Link instance, if one exists.
    """
    return _the_link.get(None)


class TS:
    "A wrapper to TaskStatus that swallows successive calls"

    def __init__(self, ts):
        self.ts = ts

    def started(self, data=None):
        "call .started once"
        self.ts.started(data)
        self.ts = anyio.TASK_STATUS_IGNORED


class BasicCmd:
    """
    A simple command that doesn't require streaming.

    The command can thus be repeated if a connection dies while it's
    running.
    """

    def __init__(self, a, kw):
        self.a = a
        self.kw = kw
        self._result = None
        self._evt = anyio.Event()

    async def run(self, link):
        "run the command"
        try:
            res = await link.cmd(*self.a, **self.kw)
        except anyio.get_cancelled_exc_class():
            raise
        except Exception as exc:
            self._result = outcome.Error(exc)
        else:
            self._result = outcome.Value(res)
        finally:
            self._evt.set()

    @property
    async def result(self):
        "retreve the result"
        await self._evt.wait()
        if self._result is None:
            raise _Requeue
        return self._result.unwrap()


class LinkCommon(CmdCommon):
    """
    Common code for links to MQTT only (BasicLink) and to a MoaT-Link server (Link).
    """

    protocol_version: int = -1
    name: str
    _id: str
    server_name: str | None = None
    is_server: bool = False
    _sender: MsgSender
    _hello: Hello | None = None
    _port: dict[str, Any] | str | None = None
    sdr: LinkSender
    announced: set[Path]
    backend: Backend
    tg: Any
    current_server: MsgSender | None

    def __init__(self, cfg, name: str | None = None):
        super().__init__(cfg)
        CFG.maybe_redo()

        if name is not None:
            self.is_server = True
        else:
            name = cfg.get("client_id")
            if name is None:
                name = "_" + gen_ident(12, alphabet=al_unique)
        self._id = name
        self.name = name

        self._cmdq_w, self._cmdq_r = anyio.create_memory_object_stream(5)
        self.logger = logging.getLogger(f"moat.link.client.{name}")
        self._sender = MsgSender(self)

    @property
    def sender(self) -> MsgSender:
        "The MsgSender that forwards to our server"
        return self._sender

    async def handle(self, msg, rcmd):
        """
        Message handler that intercepts incoming commands
        while authorization has not completed
        """
        if self._hello is not None and self._hello.auth_data is None:
            return await self._hello.handle(msg, rcmd)

        if rcmd and rcmd[-1] == "d_":
            msg._kw = dict(msg.kw)  # noqa: SLF001
            # reversed path, without the local "d_" suffix
            msg._kw["p"] = Path.build(rcmd[-2::-1])  # noqa: SLF001
            return await msg.call_stream(self.sdr.stream_d_)

        return await super().handle(msg, rcmd)

    @property
    def id(self):  # noqa:D102
        return self._id

    async def _connected_port(self, *, task_status=anyio.TASK_STATUS_IGNORED):
        if self._port is None:
            raise RuntimeError("No configured port")
        async with self._connect_one(self._port) as hdl:
            task_status.started(hdl)
            await anyio.sleep_forever()

    def _state_changed(self):
        pass

    async def get_link(self) -> MsgSender:
        "Return the active remote sender."
        raise RuntimeError("No server link")

    @asynccontextmanager
    async def _connect_one(
        self, remote: dict[str, Any] | str, data: dict[str, Any] | None = None
    ) -> AsyncIterator[MsgSender]:
        auth_out: list[AuthMethod] = []
        rpc_auth_modes = ["anon"]
        rpc_auth_data: dict[str, Any] = {}
        if isinstance(remote, dict):
            if data is not None:
                with suppress(KeyError):
                    token = data["auth"]["token"]
                    auth_out.append(TokenAuth(token))
                    rpc_auth_modes.insert(0, "token")
                    rpc_auth_data["token"] = token
            conn_ = TCPConn(
                self,
                remote_host=remote["host"],
                remote_port=remote["port"],
                logger=self.logger,
            )
        else:
            conn_ = UnixConn(self, path=remote, logger=self.logger.debug)

        auth_out.append(AnonAuth())
        self._hello = Hello(
            me=self.name,
            me_server=self.is_server,
            auth_out=auth_out,
            rpc_auth_modes=tuple(rpc_auth_modes),
            rpc_auth_data=rpc_auth_data,
            rpc_auth_server=False,
        )
        yielded = False

        async with conn_ as conn:
            if self.logger.isEnabledFor(logging.INFO):
                self.logger.info("Connection %s to %s", self.name, srepr(remote))
            handler = MsgSender(conn)
            if (res := await self._hello.run(handler)) is False:
                raise AuthError("Initial handshake failed")

            if isinstance(res, dict):
                name = res.pop("name", None)
                if name is not None:
                    self.name = name
                if res:
                    self.logger.warning("Unknown auth reply: %r", res)

            if self._hello.me is not None:
                self.name = self._hello.me
            self.server_name = self._hello.them
            self.protocol_version = self._hello.protocol_version
            self._hello = None  # done with that
            self._state_changed()

            yielded = True
            yield handler

        if not yielded:
            raise EOFError


class ClientCaller(Caller):
    """
    This is a helper class that ensures the connection to the MoaT-Link server
    is up before attempting to talk to it.

    Don't instantiate this class yourself.
    """

    def __init__(self, sender, *a, **kw):
        self._link = sender._link  # noqa:SLF001
        super().__init__(sender, *a, **kw)

    async def _call(self):
        "helper for __await__ that calls the remote handler"
        link = await self._link.get_link()
        cmd, a, kw = self.data
        return await link.root._sender.cmd(cmd, *a, **kw)  # noqa:SLF001


@define
class _CodeWatch:
    """
    Shared watcher state for one code path.
    """

    code: Code
    users: int
    cancel_scope: anyio.CancelScope
    done: anyio.Event


def _code_path(path: Path) -> Path:
    "Build a code storage path below ``code.exec``."
    from moat.link.code import CODE_EXEC_ROOT  # noqa: PLC0415

    return CODE_EXEC_ROOT + Path.build(path)


class CodeCaller:
    """
    Helper returned by :meth:`LinkSender.code_at`.

    This object supports direct calls, ``await`` to get the current
    code wrapper, and async context usage to
    keep one shared wrapper updated in the background.
    """

    sender: LinkSender
    path: Path

    def __init__(self, sender: LinkSender, path: Path):
        self.sender = sender
        self.path = _code_path(path)

    def __await__(self):
        return self.sender._code_get(self.path).__await__()  # noqa: SLF001

    def __call__(self, **kw: Any):
        "Call the referenced code snippet."

        async def _run():
            code = await self
            return await code(**kw)

        return _run()

    async def _enter(self) -> Code:
        "Acquire (and maybe start) one shared code watcher."
        p = self.path
        while True:
            entry = self.sender._code_watch.get(p, NotGiven)  # noqa: SLF001
            if isinstance(entry, _CodeWatch):
                entry.users += 1
                return entry.code
            if entry is not NotGiven:
                await entry.wait()
                continue

            evt = anyio.Event()
            self.sender._code_watch[p] = evt  # noqa: SLF001
            try:
                from moat.link.code.run import Code  # noqa: PLC0415

                code = Code(self.sender, p)
                code.update(await self.sender.d_get(p))
                done = anyio.Event()
                scope = await self.sender._link.tg.start(self.sender._watch_code, p, code, done)  # noqa: SLF001
                self.sender._code_watch[p] = _CodeWatch(  # noqa: SLF001
                    code=code,
                    users=1,
                    cancel_scope=scope,
                    done=done,
                )
                evt.set()
                return code
            except BaseException:
                self.sender._code_watch.pop(p, None)  # noqa: SLF001
                evt.set()
                raise

    async def _exit(self) -> None:
        "Release one shared code watcher."
        p = self.path
        while True:
            entry = self.sender._code_watch.get(p, NotGiven)  # noqa: SLF001
            if isinstance(entry, _CodeWatch):
                if entry.users > 1:
                    entry.users -= 1
                    return
                evt = anyio.Event()
                self.sender._code_watch[p] = evt  # noqa: SLF001
                try:
                    entry.cancel_scope.cancel()
                    await entry.done.wait()
                finally:
                    self.sender._code_watch.pop(p, None)  # noqa: SLF001
                    evt.set()
                return
            if entry is not NotGiven:
                await entry.wait()
                continue
            if entry is NotGiven:
                return

    async def __aenter__(self) -> Code:
        return await self._enter()

    async def __aexit__(self, *tb):
        await self._exit()


class LinkSender(MsgSender):
    """
    This is the client-side front-end to a "standard" MoaT-Link connection.

    It contains various somewhat-high-level helpers. Most are named like
    the corresponding backend commands, with an underscore replacing the
    dot.

    Do not instantiate this class yourself; it's returned when you enter
    the context of a `Link`.
    """

    Caller_ = ClientCaller

    _codec_tree: CodecNode | None = None
    _codec_tree_wait: anyio.Event | None = None
    _code_watch: dict[Path, _CodeWatch | anyio.Event]
    d: DataSender
    d_: MsgSender
    e: ErrSender
    i: MsgSender
    cl: MsgSender

    def __init__(self, link: LinkCommon):
        self._link = link
        self.announced = self._link.announced
        self._code_watch = {}
        self._codec_tree_wait = None

    @property
    def root(self):
        "Self"
        return self

    @property
    def link(self):
        "Link"
        return self._link

    @property
    def name(self) -> str:
        "Link name"
        return self._link.name

    @property
    def cfg(self) -> attrdict:
        "Link config"
        return self._link.cfg

    @property
    def id(self) -> str:
        "Link ID"
        return self._link.id

    async def handle(self, msg: Msg, rcmd: list[PathElem]) -> None:
        """
        Standard handler, forwards to the remote side.
        """
        if rcmd and rcmd[-1] == "d_":
            return await self._link.handle(msg, rcmd)
        srv = await self._link.get_link()
        await srv.handle(msg, rcmd)

    def find_handler(self, path: Path, cmd: bool = False) -> tuple[MsgSender, Path]:
        """
        Standard sub-dispatcher redirector, no-op.
        """
        cmd  # noqa:B018
        return self, path

    @asynccontextmanager
    async def announcing(self, *a, **kw):
        """
        Frontend for `moat.link.announce.announcing`.
        """
        from moat.link.announce import announcing as ann  # noqa:PLC0415

        async with ann(self, *a, **kw) as res:
            yield res

    async def stream_d_(self, msg: Msg):
        """
        Simple Data Protocol on the client, possibly with subpath.
        """
        try:
            p = msg["p"]
        except KeyError:
            p = msg[0]
            off = 1
        else:
            off = 0

        p = Path.build(p)

        try:
            try:
                d = msg["d"]
            except KeyError:
                d = msg[off]
            # write

            await self.d_set(p, d)

        except (KeyError, IndexError):
            # read
            res = await self.d_get(p)
            await msg.result(res)

    @overload
    async def d_get(self, path: Path, meta: Literal[True]) -> tuple[Any, MsgMeta]: ...

    @overload
    async def d_get(self, path: Path) -> Any: ...

    async def d_get(self, path: Path, meta: bool = False) -> tuple[Any, MsgMeta]:
        """
        Data retrieval. Calls the server's ``d.get`` method.

        Returns a data+metadata tuple if @meta is True, otherwise just the
        data.
        """
        if len(path) and isinstance(path[0], Path):
            raise ValueError("Don't use a root-prefixed path here.")

        res = await self.d.get(path)
        if not meta:
            return res[0]
        return res[0], MsgMeta.restore(res[1:])

    @overload
    async def d_search(self, path: Path, meta: Literal[True]) -> tuple[Any, MsgMeta]: ...

    @overload
    async def d_search(self, path: Path) -> Any: ...

    async def d_search(self, path: Path, meta: bool = False) -> tuple[Any, MsgMeta]:
        """
        Search-based data retrieval. Calls the server's ``d.search`` method.

        Returns a data+metadata tuple if @meta is True, otherwise just the
        data.
        """
        if len(path) and isinstance(path[0], Path):
            raise ValueError("Don't use a root-prefixed path here.")

        res = await self.d.search(path)
        if not meta:
            return res[0]
        return res[0], MsgMeta.restore(res[1:])

    @overload
    async def d_set(
        self,
        path: Path,
        data: Any = NotGiven,
        t: float | None = None,
        meta: Literal[True] = True,
        retain: bool | None = None,
        verify: bool | None = False,
    ) -> bool | None: ...

    @overload
    async def d_set(
        self,
        path: Path,
        data: Any = NotGiven,
        t: float | None = None,
        meta: Literal[False] = False,
        retain: bool | None = None,
        verify: bool | None = False,
    ) -> bool | None: ...

    async def d_set(
        self,
        path: Path,
        data: Any = NotGiven,
        meta: MsgMeta | None = None,
        t: float | None = None,
        with_prev: bool = False,
        retain: bool | None = None,
        verify: bool | None | NotGivenType = NotGiven,
    ) -> bool | None | tuple[Any, MsgMeta | None]:
        """
        Data update.

        If a timestamp is passed in @t or the old value+metadata is
        requested via @with_prev, goes through the server. Otherwise posts to
        MQTT directly.
        """
        if path and isinstance(path[0], Path):
            raise ValueError("Don't use a root-prefixed path here.")

        if verify is NotGiven:
            verify = (
                "pytest" in sys.modules and getattr(self._link, "current_server", None) is not None
            )

        if verify is not False:
            if not _HAVE_SCHEMA:
                if verify is None:
                    self._link.logger.warning(
                        "Schema support is not installed, cannot verify %s", path
                    )
                else:
                    raise RuntimeError("Schema support is not installed")
            else:
                try:
                    schema = await self.d_search(schema_path(path))
                except KeyError:
                    pass
                except Exception as exc:
                    if verify is None:
                        self._link.logger.warning("Schema lookup failed for %s: %r", path, exc)
                    else:
                        raise
                else:
                    try:
                        validate_instance(schema, data)
                    except Exception as exc:
                        if verify is None:
                            self._link.logger.warning(
                                "Schema validation failed for %s: %r", path, exc
                            )
                        else:
                            raise

        if meta is None:
            meta = MsgMeta(origin=self._link.name)
        if t is None and not with_prev:
            if retain is None:
                retain = len(path) == 0 or path[0] != "run"
            await self.send(Root.get() + path, data=data, meta=meta, retain=retain)
            return True
        tt = {} if t is None else {"t": t}
        old: tuple[Any, MsgMeta | None] | None = None
        if with_prev:
            try:
                old = await self.d_get(path, meta=True)
            except KeyError:
                old = (NotGiven, None)
        res = await self.d.set(path, data, meta, **tt)
        if not isinstance(res, bool) and res is not None:
            res = res[0]
        if with_prev:
            assert old is not None
            return old
        return res

    @asynccontextmanager
    async def d_walk(
        self,
        path: Path,
        meta: bool = False,
        min_ts: float = 0,
        min_depth: int = 0,
        max_depth: int = 255,
    ) -> AsyncIterator[tuple[str, Any, MsgMeta]]:
        """
        Fetch a (limited) subtree.
        """
        args = [path, min_ts, min_depth, max_depth]
        if args[-1] == 255:
            args.pop()
            if args[-1] == 0:
                args.pop()
                if args[-1] == 0:
                    args.pop()

        async with self.d.walk(*args).stream_in() as mon:
            yield Walker(mon, meta=meta)

    async def stream_watch(self, msg: Msg):
        """
        A hook for reading data. Used mainly by `moat.lib.rpc.app.link.Cmd`.
        """
        if msg.can_stream:
            async with msg.stream_out() as ms, self.d_watch(msg[0], **msg.kw) as mr:
                async for d in mr:
                    await ms.send(d)
        else:
            d = await self.d_get(msg[0], **msg.kw)
            await msg.result(d)

    # No use marking as Any|None if @mark is set
    @overload
    def d_watch(
        self,
        path: Path,
        mark: bool = False,
        meta: Literal[False] = False,
        subtree: Literal[False] = False,
        state: bool | NotGivenType | None = None,
    ) -> AbstractAsyncContextManager[AsyncIterator[Any]]: ...

    @overload
    def d_watch(
        self,
        path: Path,
        mark: Literal[False] = False,
        meta: Literal[True] = True,
        subtree: Literal[False] = False,
        state: bool | NotGivenType | None = None,
    ) -> AbstractAsyncContextManager[AsyncIterator[tuple[Any, MsgMeta]]]: ...

    @overload
    def d_watch(
        self,
        path: Path,
        mark: Literal[False],
        meta: Literal[True],
        subtree: Literal[True],
        state: bool | NotGivenType | None = None,
    ) -> AbstractAsyncContextManager[AsyncIterator[tuple[Path, Any, MsgMeta]]]: ...

    @overload
    def d_watch(
        self,
        path: Path,
        mark: Literal[False] = False,
        meta: Literal[False] = False,
        subtree: Literal[True] = True,
        state: bool | NotGivenType | None = None,
    ) -> AbstractAsyncContextManager[AsyncIterator[tuple[Path, Any]]]: ...

    @overload
    def d_watch(
        self,
        path: Path,
        mark: Literal[True],
        meta: Literal[True],
        subtree: Literal[False] = False,
        state: bool | NotGivenType | None = None,
    ) -> AbstractAsyncContextManager[AsyncIterator[None | tuple[Any, MsgMeta]]]: ...

    @overload
    def d_watch(
        self,
        path: Path,
        mark: Literal[True],
        meta: Literal[True],
        subtree: Literal[True],
        state: bool | NotGivenType | None = None,
    ) -> AbstractAsyncContextManager[AsyncIterator[None | tuple[Path, Any, MsgMeta]]]: ...

    @overload
    def d_watch(
        self,
        path: Path,
        mark: Literal[True],
        meta: Literal[False] = False,
        subtree: Literal[True] = True,
        state: bool | NotGivenType | None = None,
    ) -> AbstractAsyncContextManager[AsyncIterator[None | tuple[Path, Any]]]: ...

    def d_watch(
        self,
        path: Path,
        meta: bool = False,
        subtree: bool = False,
        state: bool | None | NotGivenType = None,
        age: float | None = None,
        mark: bool = False,
        min_length: int | None = None,
        max_length: int | None = None,
        cls: type[Node] = Node,
    ):
        """
        Monitor a node or subtree.

        Args:
            path: what to examine
            meta: flag whether to return metadata too
            subtree: flag whether to watch a subtree, not just this node
            mark: yield `None` when the initial state has been transmitted
            state: send the current state (True), updates (False), both with
                    current data from the server (None; this is the
                    default), or both via MQTT (NotGiven).
            age: skip nodes with data older than this many seconds.
            cls: type of root node (default `~moat.link.node.Node`)

        This method returns an async context manager which yields an async iterator.
        The iterator yields node data if neither ``subtree`` nor ``meta`` is set.
        Otherwise it yields tuples. The first item is the path if ``subtree``
        is set; the last item is the metadata if ``meta`` is set.

        ``state=NotGiven`` does not work with ``mark=True``, or when the same
        subscription is already active somewhere else.
        """
        return Watcher(self, path, meta, subtree, state, age, mark, cls, min_length, max_length)

    @asynccontextmanager
    async def as_dict(self, path: Path, default: attrdict | None = None):
        """
        This async context manager returns an attrdict that gets auto-updated
        when a configuration tree changes.

        The root also contains a ``wait_`` member which can be awaited,
        for re-sync after updates.

        Warning: The order of interpreting these values and their updates
        is not specified. Conflicts should be strictly avoided.
        """
        res = attrdict() if default is None else default
        evt = anyio.Event()

        class AWT:
            def __await__(self):
                return evt.wait().__await__()

        res.wait_ = AWT()

        async with (
            anyio.create_task_group() as tg,
            Watcher(self, path, False, True, None, None, True, Node, None, None) as mon,
        ):

            async def _run(task_status):
                nonlocal evt

                async for pd in mon:
                    if pd is None:
                        evt = anyio.Event()
                        task_status.started()
                        continue
                    p, d = pd
                    if type(d) is dict:
                        d = to_attrdict(d)
                    dl = list(p)
                    while dl:
                        n = dl.pop()
                        d = attrdict({n: d})
                    if not isinstance(d, dict):
                        self._link.logger.warning("Item at %r is %r, not a dict, ignoring", p, d)
                        continue
                    merge(res, d)
                    if evt is not None:
                        evt.set()
                        evt = anyio.Event()

            await tg.start(_run)
            yield res
            tg.cancel_scope.cancel()

    async def e_exc(self, path: Path, exc: Exception, **kw):
        """
        Report an exception, with backtrace.
        """
        exc = ungroup.one(exc)

        kw["bt"] = format_exception(exc)
        kw["exc"] = exc

        await self.e.exc(path, repr(exc), **kw)

    async def e_info(self, path: Path, txt: str, **kw):
        """
        Report a problem that's not an exception.
        """
        await self.e.info(path, txt, **kw)

    async def e_ack(self, path: Path, **kw):
        """
        Report that $USER has been notified of a problem.

        @ack can be True (don't notify any more), False (notify next
        time it happens), or a timestamp (notify if it happens again after
        $time).
        """
        await self.e.ack(path, **kw)

    async def e_ok(self, path: Path, **kw):
        """
        Report that something is working.
        """
        await self.e.ok(path, **kw)

    @asynccontextmanager
    async def e_wrap(self, path: Path, **kw):
        """
        An error-handling context manager. It opens a stream to the server
        and reports either success or failure over it.

        Streamed messages can be sent and will be stored as auxiliary
        information if the command should fail.

        This wrapper does *not* swallow the exception.
        """
        async with self.e.mon(path, **kw) as mon:
            yield mon
            await mon.send(True)

    def monitor(self, path: Path, *a, **kw) -> AbstractAsyncContextManager[AsyncIterator[Message]]:
        """
        Watch this path.

        This call forwards directly to the back-end.
        See `moat.link.backend.Backend.monitor` for call details.

        The caller is responsible for prefixing the Root path, if applicable.
        """
        return self._link.backend.monitor(path, *a, **kw)

    def send(self, path: Path, *a, **kw) -> Awaitable[None]:
        """
        Send to this path.

        This call forwards directly to the back-end.
        See `moat.link.backend.Backend.send` for call details

        The caller is responsible for prefixing the Root path, if applicable.
        """
        return self._link.backend.send(path, *a, **kw)

    async def i_sync(self):
        """
        This call tries to ensure that the server has processed all
        incoming MQTT messages.

        It does this by requesting a timestamp from the server, publishing
        the timestamp to MQTT, and then waiting for the server to
        acknowledge having received (at least) this number.

        This code assumes that MQTT messages are delivered in the order
        they are submitted. Depending on the server's multithreading setup
        this may or may not be true, but in practice it should be
        sufficiently true-ish to work out.

        It also assumes that if there's more than one server, they all have
        mostly-accurate time.
        """
        (st,) = await self.cmd(P("i.stamp"))
        await self.send(P(":R.run.service.main.stamp") / self._link.server_name, st, retain=False)
        await self.cmd(P("i.sync"), st)

    async def i_checkid(self, id: str) -> bool:
        """
        Check whether the given client ID is still valid.
        """
        (st,) = await self.cmd(P("i.checkid"), id)
        return st

    async def _get_tree(self, path, *, task_status, **kw):
        async with Watcher(
            self,
            path,
            bool(kw.get("meta", False)),
            bool(kw.get("subtree", False)),
            kw.get("state", None),
            kw.get("age", None),
            bool(kw.get("mark", False)),
            kw.get("cls", Node),
            kw.get("min_length", None),
            kw.get("max_length", None),
        ) as w:
            task_status.started(await w.get_node())
            await anyio.sleep_forever()

    async def get_codec_tree(self):
        """
        Returns a common subtree for codecs.
        """

        if self._codec_tree is None:
            if self._codec_tree_wait is not None:
                await self._codec_tree_wait.wait()
                assert self._codec_tree is not None
                return self._codec_tree
            evt = anyio.Event()
            self._codec_tree_wait = evt
            from moat.link.node.codec import CodecNode  # noqa: PLC0415

            tree = await self._link.tg.start(
                partial(
                    self._get_tree, P("codec"), subtree=True, state=None, meta=False, cls=CodecNode
                )
            )
            self._codec_tree = tree
            self._codec_tree_wait = None
            evt.set()
        assert self._codec_tree is not None
        return self._codec_tree

    async def get_service(self, srv: Path, wait: bool = False) -> MsgSender:
        """
        Retrieve a SubDispatcher that connects to the given announced
        service.
        """

        srv = P("run.host") + srv
        try:
            res = await self.d_get(srv)
        except KeyError:
            if not wait:
                raise
            async with self.d_watch(srv) as mon:
                async for res in mon:
                    if res not in (None, NotGiven):
                        break

        res2 = await self.d_get(P("run.id") / res["id"])

        return self.sub_at(Path.build(("srv", res2["srv"], "cl", res["id"])) + res["path"])

    async def _code_get(self, p: Path) -> Code:
        "Return cached code while monitored, otherwise a fresh wrapper."
        while True:
            entry = self._code_watch.get(p, NotGiven)
            if isinstance(entry, _CodeWatch):
                return entry.code
            if entry is not NotGiven:
                await entry.wait()
                continue
            if entry is NotGiven:
                from moat.link.code.run import Code  # noqa: PLC0415

                return Code(self, p)

    async def _watch_code(
        self, path: Path, code: Code, done: anyio.Event, *, task_status=anyio.TASK_STATUS_IGNORED
    ):
        "Background updater for one code path."
        with anyio.CancelScope() as scope:
            task_status.started(scope)
            try:
                async with self.d_watch(path, state=False) as mon:
                    async for data in mon:
                        code.update(data)
            finally:
                done.set()

    def code_at(self, path: Path) -> CodeCaller:
        """
        Return a code caller.

        The result can be called directly, awaited to retrieve a
        code wrapper, or used as an async
        context manager to keep one shared wrapper updated.
        """
        return CodeCaller(self, path)


class Link(LinkCommon, CtxObj):
    """
    This class combines the back-end link with a connection to a MoaT-Link server.

    If "common" is set, create global state.
    If "only" is set, connect only to the named server.
    """

    _server: ValueEvent = None
    _uptodate: bool = False
    _hello: Hello | None = None
    current_server: MsgSender | None = None
    _server_up: anyio.Event
    _last_link: Message | None = None
    _last_link_seen: anyio.Event
    _port: dict[str, Any] | str | None = None
    _state: str = "init"
    _common: bool = False
    _only: str | None = None
    announced: set[Path]
    sdr: LinkSender

    def __new__(cls, cfg, name: str | None = None, common: bool = False, only: str | None = None):  # noqa:D102
        cfg, name, only  # noqa:B018
        if common and (link := _the_link.get(NotGiven)) is not NotGiven:
            return link
        return super().__new__(cls)

    def __init__(
        self, cfg, name: str | None = None, common: bool = False, only: str | None = None
    ):
        if _the_link.get(NotGiven) is self:
            return
        super().__init__(cfg, name=name)
        self._retry_msgs: set[BasicCmd] = set()
        self._server_up = anyio.Event()
        self._state_change = anyio.Event()
        self._common = common
        self._only = only
        self.announced = set()
        with suppress(AttributeError):
            self._port = self.cfg.client.port

    async def set_state(self, state: str):
        """
        Set the state string for our ping message
        """
        if state != self._state:
            self._state = state
            self._state_changed()

    def _state_changed(self):
        self._state_change.set()

    async def get_link(self):
        """
        This method refreshes the link to the server if it happens to be down.
        """
        while self.current_server is None:
            await self._server_up.wait()
        return self.current_server

    @property
    def _ping_path(self):
        return P("run.ping.id") / self.id

    @property
    def _id_path(self):
        return P("run.id") / self.id

    async def _send_id(self):
        data = dict(
            host=platform.node(),
            pid=os.getpid(),
            argv=sys.argv,
        )
        if self.server_name is not None:
            data["srv"] = self.server_name
        await self.sdr.d_set(
            self._id_path,
            data,
            retain=True,
        )

    async def _send_ping(self, *, task_status=anyio.TASK_STATUS_IGNORED):
        """
        Periodially (or when our state changes) send a Ping.

        This also sends our ID message, after the first ping,
        but only if the task status isn't ignored.
        """
        path = Root.get() + self._ping_path
        sn = self.server_name
        while True:
            await self.backend.send(
                path,
                data=dict(
                    up=True,
                    state=self._state,
                ),
                retain=False,
                meta=False,
            )

            if task_status is not anyio.TASK_STATUS_IGNORED:
                # send initial run.id message, *after* the ping
                sn = self.server_name
                await self._send_id()
                task_status.started()
                task_status = anyio.TASK_STATUS_IGNORED

            elif sn != self.server_name:
                sn = self.server_name
                await self._send_id()

            if self._state == "init":
                self._state = "auto"

            with anyio.move_on_after(self.cfg.timeout.ping.every):
                await self._state_change.wait()
                self._state_change = anyio.Event()

    async def _monitor_ping(self, *, task_status=anyio.TASK_STATUS_IGNORED):
        path = Root.get() + self._ping_path
        async with self.backend.monitor(path) as mon:
            task_status.started()
            async for msg in mon:
                if msg.data is NotGiven:
                    raise ClientCancelledError(msg.meta)
                # anything else shall be ignored

    @asynccontextmanager
    async def _ctx(self):
        from .backend import get_backend  # noqa: PLC0415

        # clears our announcement on disconnect
        will = attrdict(
            data=dict(up=False, state="will"),
            topic=P(":R") + self._ping_path,
            retain=False,
            qos=QoS.AT_LEAST_ONCE,
        )
        async with (
            ctx_as(Root, self.cfg["root"]),
            get_backend(self.cfg, name=self.name, will=will) as self.backend,
        ):
            try:
                async with anyio.create_task_group() as self.tg:
                    if self._port is not None:
                        sdr = await self.tg.start(self._connected_port)
                    else:
                        if self.cfg.client.init_timeout:
                            # connect to the main server
                            await self.tg.start(self._run_server_link)
                        sdr = LinkSender(self)
                    sdr.add_sub("cl")
                    sdr.add_sub("d")
                    sdr.add_sub("d_")
                    sdr.add_sub("e")
                    sdr.add_sub("i")
                    try:
                        self.sdr = sdr
                        await self.tg.start(self._monitor_ping)
                        await self.tg.start(self._send_ping)

                        with ctx_as(_the_link, self) if self._common else nullcontext():
                            yield sdr
                    finally:
                        del self.sdr
                    self.tg.cancel_scope.cancel()

            finally:
                try:
                    with anyio.move_on_after(2, shield=True):
                        await self.backend.send(
                            Root.get() + self._ping_path,
                            data=dict(up=False, state="closed"),
                            retain=False,
                            meta=False,
                        )
                except Exception as exc:
                    self.logger.warning("Could not send Close message", exc_info=exc)

    def cancel(self):
        "Stop me"
        self.tg.cancel_scope.cancel()

    async def _run_server_link(self, *, task_status=anyio.TASK_STATUS_IGNORED):
        """
        This is the manager task for the server link channel.
        It starts a server connection (and tries to keep it alive).
        """
        task_status = TS(task_status)

        with anyio.fail_after(self.cfg.client.init_timeout):
            srv = await self.tg.start(self._read_server_link)

        tm = self.cfg.timeout.connect
        timeout = tm.initial
        while True:
            if srv is None:
                await self._last_link_seen.wait()
                srv = self._last_link
                self._last_link_seen = anyio.Event()
                continue
            try:
                await self._connect_server(srv, task_status=task_status)
            except Exception as exc:
                if srv.meta is None:
                    err_path = P("run.service.main.server")
                else:
                    err_path = P("run.service.main.server") / srv.meta.origin
                await self.backend.send_error(
                    err_path,
                    data=srv,
                    exc=exc,
                )
            finally:
                self.current_server = None
                if self._server_up.is_set():
                    self._server_up = anyio.Event()

                # TODO test if tasks from the remote side already get cancelled
                # TODO and if not, do that

            # Back off to either a new server announcement or the exponential timeout
            try:
                with anyio.fail_after(timeout):
                    await self._last_link_seen.wait()
            except TimeoutError:
                # try the last-reported connection again
                timeout = min(timeout * tm.factor, tm.max)
            else:
                # immediately use the new data
                srv = self._last_link
                self._last_link_seen = anyio.Event()

    async def _read_server_link(self, *, task_status=anyio.TASK_STATUS_IGNORED):
        # Background process that monitors the server link channel

        self._last_link_seen = anyio.Event()
        if self._only is not None:
            # Monitor the specific named server's announcement
            monitor_path = P(":R.run.service.main.server") / self._only
        else:
            # Monitor the main connection channel
            monitor_path = P(":R.run.service.main.conn")
        async with self.backend.monitor(
            monitor_path,
            retain_handling=RetainHandling.SEND_RETAINED,
        ) as mon:
            async for msg in mon:
                self._last_link = msg
                if task_status is None:
                    self._last_link_seen.set()
                else:
                    task_status.started(msg)
                    task_status = None

    async def _connect_run(self, rem, *, task_status=anyio.TASK_STATUS_IGNORED):
        # We're connected.
        self.current_server = rem
        self._server_up.set()

        # Check backend compatibility when connecting to a named server
        if self._only is not None:
            try:
                remote_backend = await rem.cmd(P("i.backend"))
                if remote_backend.kw != self.cfg.backend:
                    self.logger.info(
                        "Server %s uses different backend: %r (we use %r)",
                        self._only,
                        remote_backend.kw,
                        self.cfg.backend,
                    )
                    # Store the remote backend for potential reconnection
                    self._remote_backend = remote_backend.kw
            except Exception as exc:
                self.logger.warning("Could not query backend from %s: %r", self._only, exc)

        async def run_(cmd):
            await cmd.run(rem)

        async with anyio.create_task_group() as tg:
            for msg in self._retry_msgs:
                tg.start_soon(run_, msg)
            task_status.started()
            async for msg in self._cmdq_r:
                tg.start_soon(run_, msg)

    async def cmd(self, *a, _idem=False, **kw):
        """
        Queue and run a simple command.

        If @_idem is False, the command will error out if the connection
        dies while it's running. Otherwise (the default) it may be repeated.
        """
        cmd_ = BasicCmd(a, kw)
        try:
            self._retry_msgs.add(cmd_)
            while True:
                while self.current_server is None:
                    await self._server_up.wait()

                try:
                    with ungroup:
                        await self._cmdq_w.send(cmd_)
                        return await cmd_.result
                except _Requeue:
                    if _idem:
                        raise EOFError from None

        finally:
            self._retry_msgs.discard(cmd_)

    async def _connect_server(self, srv: Message, *, task_status=anyio.TASK_STATUS_IGNORED):
        task_status = TS(task_status)

        # Backend connection
        link = srv.data["link"]
        if isinstance(link, dict):
            link = (link,)

        local_ok = srv.data.get("node", "localhost")

        for remote in link:
            if local_ok:
                pass
            elif "host" not in remote:
                continue
            elif remote["host"] in local_addrs:
                continue
            try:
                async with timed_ctx(
                    self.cfg.client.init_timeout, self._connect_one(remote, srv.data)
                ) as rem:
                    await self._connect_run(rem, task_status=task_status)
            except OSError as exc:
                self.logger.warning("Link failed: %r (%r)", remote, exc)
            except Exception as exc:
                self.logger.warning("Link failed: %r", remote, exc_info=exc)


class BasicLink(LinkCommon, CtxObj):
    """
    Simple direct link to a MQTT server.
    """

    def __init__(self, cfg, name: str | None, data: dict, **kw):
        super().__init__(cfg, name=name, **kw)
        self.data = data

    @asynccontextmanager
    async def _ctx(self):
        link = self.data["link"]
        if isinstance(link, dict):
            link = (link,)

        err = None
        yielded = False
        for remote in link:
            try:
                async with self._connect_one(remote, self.data) as self._sender:
                    yielded = True
                    yield self._sender
                    return
            except Exception as exc:
                if yielded:
                    raise
                self.logger.warning(
                    "Link failed: %r",
                    remote,
                    exc_info=None if isinstance(exc, EnvironmentError) else exc,
                )
                if err is None:
                    err = exc
            else:
                if not yielded and err is None:
                    err = EOFError

        if not yielded:
            if err is None:
                raise ValueError(f"No links in {self.data!r}")
            raise err


@define(eq=False)
class Watcher(CtxObj):
    """
    Helper class for monitoring (and coalescint the data of) either-or-both
    * a MQTT subscription to a subtree of our MoaT-Link hierarchy
    * a MoaT-Link request to enumerate a subtree
    """

    link: LinkSender = field()
    path: Path = field()
    meta: bool = field()
    subtree: bool = field()
    state: bool | None | NotGivenType = field()
    age: float | None = field()
    mark: bool = field()
    node_cls: type[Node] = field()
    min_length: int | None = field()
    max_length: int | None = field()

    _qr = field(init=False, repr=False)
    _tg = field(init=False, repr=False)
    _node = field(init=False, default=None)

    _current_done: anyio.Event | None = field(init=False, default=None)

    def __attrs_post_init__(self):
        if self.mark and self.state is NotGiven:
            raise ValueError("MQTT doesn't send a mark. Sorry.")
        if self.age is not None:
            self.age = time.time() - self.age

    def _chk(self, p, m):
        if self.age and m.timestamp < self.age:
            return False
        if self.min_length is not None and len(p) < self.min_length:
            return False
        if self.max_length is not None and len(p) > self.max_length:
            return False
        return True

    async def _current(self, qw, *, task_status):
        "get the current state from the server"
        if self.subtree:
            pl = PathLongener(())
            args = [self.path, self.age, self.min_length, self.max_length]
            while args[-1] is None:
                args.pop()
            async with self.link.d.walk(*args) as mon:
                task_status.started()
                async for r in mon:
                    n, p, d, *m = r
                    p = pl.long(n, p)
                    m = MsgMeta.restore(m)
                    if self._chk(p, m):
                        await qw.send((p, d, m))
                if self.mark:
                    await qw.send(None)
        else:
            try:
                r = await self.link.d.get(self.path)
            except (KeyError, ValueError):
                task_status.started()
            else:
                task_status.started()
                p, d, m = Path(), r[0], MsgMeta.restore(r[1:])
                if self._chk(p, m):
                    await qw.send((p, d, m))
            if self.mark:
                await qw.send(None)

        await qw.aclose()
        if self._current_done is None:
            raise RuntimeError("Missing done event")
        self._current_done.set()

    async def _updates(self, qw, *, task_status):
        "get updates from MQTT"
        plen = len(self.path)  # ignores the root prefix
        async with self.link.monitor(
            Root.get() + self.path, subtree=self.subtree, retained=(self.state is NotGiven)
        ) as mon:
            task_status.started()
            async for msg in mon:
                p, d, m = Path.build(msg.topic[plen:]), msg.data, msg.meta
                if self._chk(p, m):
                    await qw.send((p, d, m))
        await qw.aclose()

    @asynccontextmanager
    async def _ctx(self):
        async with anyio.create_task_group() as tg:
            self._node = self.node_cls()
            self._current_done = anyio.Event()
            self._tg = tg
            qw, self._qr = anyio.create_memory_object_stream(10)
            if self.state is not True:
                await tg.start(self._updates, qw.clone())
            if self.state is True or self.state is None:
                await tg.start(self._current, qw.clone())
            else:
                self._current_done.set()
                if self.mark:
                    await qw.send(None)
            await qw.aclose()
            yield self
            tg.cancel_scope.cancel()
            self._node = None

    def __aiter__(self):
        return self

    @property
    @asynccontextmanager
    async def node(self) -> AsyncIterator[Node]:
        """
        Helper for fetching data in background.

        This is necessary if you start several watchers in a common context
        managers. If you then call :py.meth.`get_node` on one of them,
        the data from the other(s) might overwhelm the input queue.
        """
        if self._node is not None:
            raise RuntimeError("Use `await get_node()`")
        async with self:
            yield await self.get_node()

    async def get_node(self, background=True) -> Node:
        """
        Wait until fetching the static data is complete, then return the
        watched node.

        If @background is set (the default), start a background job that
        updates the node. In this case you cannot iterate the watcher any
        more.
        """
        if background:
            await self._tg.start(self._iter)
        if self._current_done is None:
            raise RuntimeError("Missing done event")
        await self._current_done.wait()
        return self._node

    async def _iter(self, *, task_status=anyio.TASK_STATUS_IGNORED):
        task_status.started()
        qr, self._qr = self._qr, None

        while True:
            try:
                msg = await qr.receive()
            except anyio.EndOfStream:
                return
            p, d, m = msg
            self._node.set(p, d, m)

    async def __anext__(self):
        while True:
            try:
                msg = await self._qr.receive()
            except anyio.EndOfStream:
                raise StopAsyncIteration from None
            if msg is None:
                return None
            p, d, m = msg
            if self._current_done is None:
                raise RuntimeError("Missing done event")
            if self._node.set(p, d, m, force=self._current_done.is_set()):
                if self.meta:
                    return (p, d, m) if self.subtree else (d, m)
                else:
                    return (p, d) if self.subtree else d


class Walker:
    """
    A trimmed-down watcher that retrieves a possibly-partial subtree
    from our MoaT-Link server.

    This differs from `Watcher` by not tracking updates, nor keeping a node
    tree in memory.
    """

    def __init__(self, mon, meta=False):
        self.mon = mon
        self.pl = PathLongener()
        self.meta = meta

    def __aiter__(self):
        return self

    async def __anext__(self):
        msg = await anext(self.mon)
        n, p, d, *m = msg.args
        p = self.pl.long(n, p)
        if self.meta:
            m = MsgMeta.restore(m)
            return p, d, m
        else:
            return p, d
