from __future__ import annotations

import fcntl
import gc
import io
import json
import logging
import os
import pathlib
import platform
import pwd
import queue
import re
import resource
import selectors
import signal
import socket
import sys
import threading
import time
import types
import typing as t
import uuid
from concurrent.futures import Future
from contextlib import contextmanager
from datetime import datetime
from http import HTTPStatus

import setproctitle
import yaml
from cachetools import TTLCache
from globus_compute_common.messagepack import pack
from globus_compute_common.messagepack.message_types import EPStatusReport
from globus_compute_common.pydantic_v1 import BaseModel
from globus_compute_endpoint import __version__
from globus_compute_endpoint.auth import get_globus_app_with_scopes
from globus_compute_endpoint.endpoint.config import ManagerEndpointConfig
from globus_compute_endpoint.endpoint.config.config import MINIMUM_HEARTBEAT
from globus_compute_endpoint.endpoint.config.utils import (
    load_user_config_schema,
    load_user_config_template,
    render_config_user_template,
    serialize_config,
)
from globus_compute_endpoint.endpoint.endpoint import Endpoint
from globus_compute_endpoint.endpoint.identity_mapper import (
    MappedPosixIdentity,
    PosixIdentityMapper,
)
from globus_compute_endpoint.endpoint.rabbit_mq import (
    CommandQueueSubscriber,
    ResultPublisher,
)
from globus_compute_endpoint.endpoint.utils import (
    _redact_url_creds,
    close_all_fds,
    is_privileged,
    send_endpoint_startup_failure_to_amqp,
    update_url_port,
)
from globus_compute_sdk import Client
from globus_compute_sdk.sdk.auth.auth_client import ComputeAuthClient
from globus_compute_sdk.sdk.compute_dir import ensure_compute_dir
from globus_sdk import GlobusAPIError, NetworkError

if t.TYPE_CHECKING:
    from pika.spec import BasicProperties


log = logging.getLogger(__name__)


class InvalidCommandError(Exception):
    pass


class InvalidUserError(Exception):
    pass


def _import_pyprctl():
    # Enable conditional import, and create a hook-point for testing to mock
    try:
        import pyprctl
    except AttributeError as e:
        raise ImportError("pyprctl is not supported on this system") from e

    return pyprctl


def _import_pam() -> types.ModuleType:
    # Enable conditional import, and create a hook-point for testing to mock
    from globus_compute_endpoint import pam

    return pam


class UserEndpointRecord(BaseModel):
    ep_name: str
    local_user_info: t.Optional[pwd.struct_passwd]
    arguments: str

    @property
    def uid(self) -> int:
        return self.local_user_info.pw_uid if self.local_user_info else -1

    @property
    def gid(self) -> int:
        return self.local_user_info.pw_gid if self.local_user_info else -1

    @property
    def uname(self) -> str:
        return self.local_user_info.pw_name if self.local_user_info else ""


T_CMD_START_ARGS = t.Tuple[
    MappedPosixIdentity, t.Optional[t.List[str]], t.Optional[t.Dict]
]


class EndpointManager:
    def __init__(
        self,
        conf_dir: pathlib.Path,
        endpoint_uuid: str | None,
        config: ManagerEndpointConfig,
        reg_info: dict | None = None,
    ):
        log.debug("Endpoint Manager initialization")

        self.conf_dir = conf_dir
        self._config = config

        self.user_config_template_path = (
            self._config.user_config_template_path
            or Endpoint.user_config_template_path(self.conf_dir)
        )
        self.user_config_schema_path = (
            self._config.user_config_schema_path
            or Endpoint.user_config_schema_path(self.conf_dir)
        )

        # UX - test conditional imports *now*, rather than when a request comes in;
        # this gives immediate feedback to an implementing admin if something is awry
        if config.pam.enable:
            _import_pam()
        else:
            _import_pyprctl()

        self._reload_requested = False
        self._time_to_stop = False

        self._heartbeat_period: float = max(MINIMUM_HEARTBEAT, config.heartbeat_period)

        self._children: dict[int, UserEndpointRecord] = {}

        self._wait_for_child = False

        self._command_queue: queue.SimpleQueue[tuple[int, BasicProperties, bytes]] = (
            queue.SimpleQueue()
        )
        self._command_stop_event = threading.Event()

        self._cached_cmd_start_args: TTLCache[int, T_CMD_START_ARGS] = TTLCache(
            maxsize=32768, ttl=config.mu_child_ep_grace_period_s
        )
        self._audit_pipes: dict[int, t.Any] = {}
        self._audit_log_handler_stop = not (
            self._config.high_assurance and bool(self._config.audit_log_path)
        )
        self._audit_buf_size = resource.getpagesize()
        self._audit_log_lock = threading.Lock()
        self._audit_selector = selectors.DefaultSelector()
        if self._audit_log_handler_stop:
            self._audit_selector.close()  # for mypy: closed, but always defined

        endpoint_uuid = Endpoint.get_endpoint_id(conf_dir) or endpoint_uuid

        self._mu_user = pwd.getpwuid(os.getuid())
        privileged = is_privileged(self._mu_user)

        self._allow_same_user = not privileged

        if not reg_info:
            try:
                gcc = Client(
                    local_compute_services=config.local_compute_services,
                    environment=config.environment,
                    app=get_globus_app_with_scopes(),
                )
                reg_info = gcc.register_endpoint(
                    name=conf_dir.name,
                    endpoint_id=endpoint_uuid,
                    metadata=self.get_metadata(config),
                    multi_user=privileged,
                    display_name=config.display_name,
                    allowed_functions=config.allowed_functions,
                    auth_policy=config.authentication_policy,
                    subscription_id=config.subscription_id,
                    public=config.public,
                    high_assurance=config.high_assurance,
                    admins=config.admins,
                )

                # Mostly to appease mypy, but also a useful text if it ever
                # *does* happen
                assert reg_info is not None, "Empty response from Compute API"

            except GlobusAPIError as e:
                blocked_msg = f"Endpoint registration blocked.  [{e.text}]"
                log.warning(blocked_msg)
                print(blocked_msg)
                if e.http_status in (
                    HTTPStatus.CONFLICT,
                    HTTPStatus.LOCKED,
                    HTTPStatus.NOT_FOUND,
                ):
                    sys.exit(os.EX_UNAVAILABLE)
                elif e.http_status in (
                    HTTPStatus.BAD_REQUEST,
                    HTTPStatus.UNPROCESSABLE_ENTITY,
                ):
                    sys.exit(os.EX_DATAERR)
                raise
            except NetworkError as e:
                log.exception("Network error while registering manager endpoint")
                log.critical(f"Network failure; unable to register endpoint: {e}")
                sys.exit(os.EX_TEMPFAIL)

        upstream_ep_uuid = reg_info.get("endpoint_id")
        if endpoint_uuid and upstream_ep_uuid != endpoint_uuid:
            log.error(
                "Unexpected response from server: mismatched endpoint id."
                f"\n  Expected: {endpoint_uuid}, received: {upstream_ep_uuid}"
            )
            sys.exit(os.EX_SOFTWARE)

        endpoint_uuid = str(upstream_ep_uuid)  # convenience, and satisfy mypy
        self._endpoint_uuid = uuid.UUID(endpoint_uuid)
        self._endpoint_uuid_str = endpoint_uuid

        self.identity_mapper: PosixIdentityMapper | None = None
        if not is_privileged(user_privs_only=True):
            # Test for uid-change privileges only because we don't want to enable
            # identity mapping unless the process UID has specifically these
            # privileges; else an unrelated permission (e.g., NET_BIND) would
            # allow identity mapping.
            if config.identity_mapping_config_path:
                msg = (
                    "`identity_mapping_config_path` specified, but process is not"
                    " privileged (e.g., not `root`) -- identity mapping configuration"
                    " will be ignored; only requests from identities that match the"
                    " identity that registered this endpoint will be honored."
                    f"\n    (ignored) '{config.identity_mapping_config_path}'"
                )
                log.warning(msg)
        elif config.identity_mapping_config_path:
            # Only map identities if possibility of *changing* uid; otherwise
            # we enforce that the identity of UEPs must match the
            # parent-process' authorization -- we do not want to allow an open
            # endpoint by a non-power user.
            try:
                self.identity_mapper = PosixIdentityMapper(
                    config.identity_mapping_config_path, self._endpoint_uuid_str
                )

            except PermissionError as e:
                msg = f"({type(e).__name__}) {e}"
                log.error(msg)
                print(msg, file=sys.stderr)
                sys.exit(os.EX_NOPERM)

            except Exception as e:
                msg = (
                    f"({type(e).__name__}) {e} -- Unable to read identity mapping"
                    f" configuration from: {config.identity_mapping_config_path}"
                )
                log.debug(msg, exc_info=e)
                log.error(msg)
                print(msg, file=sys.stderr)
                sys.exit(os.EX_CONFIG)

        try:
            cq_info = reg_info["command_queue_info"]
            _ = cq_info["connection_url"], cq_info["queue"]

            hbq_info = reg_info["heartbeat_queue_info"]
            _ = hbq_info["connection_url"], hbq_info["queue"]
            _ = hbq_info["queue_publish_kwargs"]
        except Exception as e:
            log_reg_info = _redact_url_creds(str(reg_info))
            log.debug("%s", log_reg_info)
            log.error(
                "Invalid or unexpected registration data structure:"
                f" ({e.__class__.__name__}) {e}"
            )
            sys.exit(os.EX_DATAERR)

        if config.amqp_port:
            cq_info["connection_url"] = update_url_port(
                cq_info["connection_url"], config.amqp_port
            )

        # sanitize passwords in logs
        log_reg_info = _redact_url_creds(repr(reg_info))
        log.debug(f"Registration information: {log_reg_info}")

        json_file = conf_dir / "endpoint.json"

        # `endpoint_id` key kept for backward compatibility when
        # globus-compute-endpoint list is called
        ep_info = {"endpoint_id": endpoint_uuid}
        json_file.write_text(json.dumps(ep_info))
        log.debug(f"Registration info written to {json_file}")

        # * == "manager endpoint"; not important until it is, so let it be subtle
        ptitle = f"Globus Compute Endpoint *({endpoint_uuid}, {conf_dir.name})"
        if config.environment:
            ptitle += f" - {config.environment}"
        ptitle += f" [{setproctitle.getproctitle()}]"
        setproctitle.setproctitle(ptitle)

        self._command = CommandQueueSubscriber(
            queue_info=cq_info,
            command_queue=self._command_queue,
            stop_event=self._command_stop_event,
            thread_name="CQS",
        )
        self._heartbeat_publisher = ResultPublisher(queue_info=hbq_info)

    def get_metadata(self, config: ManagerEndpointConfig) -> dict:
        user_config_template = load_user_config_template(self.user_config_template_path)
        user_config_schema = load_user_config_schema(self.user_config_schema_path)
        return {
            "endpoint_version": __version__,
            "python_version": platform.python_version(),
            "hostname": socket.getfqdn(),
            "local_user": pwd.getpwuid(os.getuid()).pw_name,
            "config": serialize_config(config),
            "endpoint_config": config.source_content,
            "user_config_template": user_config_template,
            "user_config_schema": user_config_schema,
        }

    def request_shutdown(self, sig_num, curr_stack_frame):
        self._time_to_stop = True

    def set_child_died(self, sig_num, curr_stack_fframe):
        self._wait_for_child = True

    def wait_for_children(self):
        try:
            self._wait_for_child = False
            wait_flags = os.WNOHANG
            pid, exit_status_ind = os.waitpid(-1, wait_flags)
            while pid > 0:
                try:
                    rc = os.waitstatus_to_exitcode(exit_status_ind)
                except ValueError:
                    rc = -127  # invalid signal number

                uep_record = self._children.pop(pid, None)

                proc_args = f" [{uep_record.arguments}]" if uep_record else ""
                if not rc:
                    log.info(f"Command stopped normally ({pid}){proc_args}")
                    cmd_start_args = self._cached_cmd_start_args.pop(pid, None)
                    if not self._time_to_stop and cmd_start_args is not None:
                        self._revive_child(uep_record, cmd_start_args)
                elif rc > 0:
                    log.warning(f"Command return code: {rc} ({pid}){proc_args}")
                elif rc == -127:
                    log.warning(f"Command unknown return code: ({pid}){proc_args}")
                else:
                    log.warning(
                        f"Command terminated by signal: {-rc} ({pid}){proc_args}"
                    )

                pid, exit_status_ind = os.waitpid(-1, wait_flags)

        except ChildProcessError:
            pass
        except Exception as e:
            log.exception(f"Failed to wait for a child process: {e}")

    def _revive_child(
        self, uep_record: UserEndpointRecord | None, cmd_start_args: T_CMD_START_ARGS
    ):
        ep_name = uep_record.ep_name if uep_record else "<unknown>"
        log.info(
            "User EP stopped within grace period; using cached arguments "
            f"to start a new instance (name: {ep_name})"
        )

        try:
            cached_identity, args, kwargs = cmd_start_args
            cur_local_user = pwd.getpwuid(cached_identity.local_user_record.pw_uid)
            cached_identity.local_user_record = cur_local_user
        except Exception as e:
            log.warning(
                "Unable to update local user information; user EP will not be revived."
                f"  ({e.__class__.__name__}) {e}"
            )
            return

        try:
            self.cmd_start_endpoint(cached_identity, args, kwargs)
        except Exception:
            log.exception(
                f"Unable to execute command: cmd_start_endpoint\n"
                f"    args: {args}\n"
                f"  kwargs: {kwargs}"
            )

    def _audit_log_impl(self):
        uid = os.getuid()
        pid = os.getpid()
        eid = self._endpoint_uuid_str
        try:
            with open(self._config.audit_log_path, "ab", buffering=0) as audit_f:
                nowtz = datetime.now().astimezone().isoformat()
                msg = f"{nowtz} uid={uid} pid={pid} eid={eid} Begin MEP session =====\n"
                audit_f.write(msg.encode())
                del msg, nowtz
                while not self._audit_log_handler_stop:
                    for key, _mask in self._audit_selector.select(timeout=3):
                        cb = key.data  # N.B.: _audit_log_write(), but general
                        cb(key.fd, audit_f)
                # thread stops after all UEPs have stopped, so perform one last round
                # to ensure we collect any outstanding messages still in kernel's buffer
                while self._audit_pipes:
                    for key, _mask in self._audit_selector.select(timeout=0.0001):
                        cb = key.data
                        cb(key.fd, audit_f)

                nowtz = datetime.now().astimezone().isoformat()
                msg = f"{nowtz} uid={uid} pid={pid} eid={eid} End MEP session -----\n"
                audit_f.write(msg.encode())
        finally:
            self._time_to_stop = True

    def _audit_log_close_reader(self, fd: int) -> None:
        try:
            with self._audit_log_lock:
                self._audit_pipes.pop(fd, None)
                os.close(fd)
                self._audit_selector.unregister(fd)
        except Exception as e:
            log.error(f"Failure unregistering audit pipe: ({type(e).__name__}) {e}")

    def _audit_log_write(self, fd: int, fpath: io.BytesIO):
        uep_audit_info = self._audit_pipes.get(fd)
        if not uep_audit_info:
            self._audit_log_close_reader(fd)
            return

        pid = uep_audit_info.get("pid")
        uid = uep_audit_info.get("uid")
        eid = uep_audit_info.get("endpoint_id")
        try:
            msg = (
                os.read(fd, self._audit_buf_size)
                .replace(b"\n", b" ")
                .replace(b"\r", b"")
                .replace(b"\0", b"")
            )
            if not msg:
                self._audit_log_close_reader(fd)
                return

            nowtz = datetime.now().astimezone().isoformat()
            header = f"{nowtz} uid={uid} pid={pid} uep={eid} "
            msgb = header.encode() + msg + b"\n"

            fpath.write(msgb)
        except Exception as e:
            # If unable to write audit log, then shutdown; don't run out of space,
            # Administrator.
            self._time_to_stop = True
            e_str = f"({type(e).__name__}) {e}"
            log.error(f"Failed to write audit log message: [{uid=}, {eid=}] - {e_str}")

    def _install_signal_handlers(self):
        signal.signal(signal.SIGTERM, self.request_shutdown)
        signal.signal(signal.SIGINT, self.request_shutdown)
        signal.signal(signal.SIGQUIT, self.request_shutdown)

        signal.signal(signal.SIGCHLD, self.set_child_died)

    def send_heartbeat(self, shutting_down=False) -> Future[None]:
        if not self._heartbeat_publisher.is_alive():
            _w = RuntimeWarning("Heartbeat requested, but publisher is not running")
            f: Future[None] = Future()
            f.set_exception(_w)
            return f

        def _heart_publish_done(pub_fut: Future):
            e = f.exception()
            if e:
                log.error(
                    f"Failed to send heartbeat to web-services"
                    f" -- ({type(e).__name__}) {e}"
                )

        global_state = {"heartbeat_period": self._heartbeat_period}
        if shutting_down:
            global_state["heartbeat_period"] = 0  # 0 == "shutting down now"

        message = EPStatusReport(
            endpoint_id=self._endpoint_uuid, global_state=global_state, task_statuses={}
        )
        f = self._heartbeat_publisher.publish(pack(message))
        f.add_done_callback(_heart_publish_done)
        return f

    def start(self):
        log.info(f"\n\n========== Endpoint Manager begins: {self._endpoint_uuid_str}")

        msg_out = None
        if sys.stdout.isatty():
            msg_out = sys.stdout
        elif sys.stderr.isatty():
            msg_out = sys.stderr

        if msg_out:
            # hide cursor, highlight color, reset
            hc, hl, r = "\033[?25l", "\033[104m", "\033[m"
            pld = f"{hl}{self._endpoint_uuid_str}{r}"
            print(f"{hc}        >>> Endpoint ID: {pld} <<<", file=msg_out)

        self._install_signal_handlers()

        audit_thr = None
        if not self._audit_log_handler_stop:
            audit_thr = threading.Thread(target=self._audit_log_impl, daemon=True)
            audit_thr.start()

        self._command.start()
        self._heartbeat_publisher.start()

        try:
            self._event_loop()
        except Exception:
            log.exception("Unhandled exception; shutting down endpoint master")

        ptitle = f"[shutdown in progress] {setproctitle.getproctitle()}"
        setproctitle.setproctitle(ptitle)
        self._command_stop_event.set()

        if self.identity_mapper:
            self.identity_mapper.stop_watching()

        try:
            f = self.send_heartbeat(shutting_down=True)
            f.result(10)  # Ensure heartbeat sent prior to thread shutdown
        except Exception as e:
            log.warning(f"Unable to send final heartbeat -- ({type(e).__name__}) {e}")

        self._heartbeat_publisher.stop(block=False)
        os.killpg(os.getpgid(0), signal.SIGTERM)

        proc_uid, proc_gid = os.getuid(), os.getgid()
        for msg_prefix, signum in (
            ("Signaling shutdown", signal.SIGTERM),
            ("Forcibly killing", signal.SIGKILL),
        ):
            for pid, rec in self._children.items():
                uid, gid, uname, proc_args = rec.uid, rec.gid, rec.uname, rec.arguments
                proc_ident = f"PID: {pid}, UID: {uid}, GID: {gid}, User: {uname}"
                log.info(f"{msg_prefix} of user endpoint ({proc_ident}) [{proc_args}]")
                try:
                    os.setresgid(gid, gid, -1)
                    os.setresuid(uid, uid, -1)
                    os.killpg(os.getpgid(pid), signum)
                except Exception as e:
                    log.warning(
                        f"User endpoint signal failed: {e} ({proc_ident}) [{proc_args}]"
                    )
                finally:
                    os.setresuid(proc_uid, proc_uid, -1)
                    os.setresgid(proc_gid, proc_gid, -1)

            deadline = time.monotonic() + 10
            while self._children and time.monotonic() < deadline:
                time.sleep(0.5)
                self.wait_for_children()

        self._audit_log_handler_stop = True
        self._command.join(5)
        self._heartbeat_publisher.join(5)
        if audit_thr:
            audit_thr.join(5)

        log.info(
            "Shutdown complete."
            f"\n---------- Endpoint Manager ends: {self._endpoint_uuid_str}\n\n"
        )
        if msg_out:
            # re-enable cursor visibility
            print("\033[?25h", end="", file=msg_out)

    def _event_loop(self):
        parent_identities: set[str] = set()
        if not (is_privileged() and self.identity_mapper):

            def _get_globus_identities():
                client_options = {
                    "local_compute_services": self._config.local_compute_services,
                    "environment": self._config.environment,
                    "app": get_globus_app_with_scopes(),
                }
                log.debug("Ascertaining user identity set (%s)", client_options)

                gcc = Client(**client_options)
                ac = ComputeAuthClient(app=gcc.app)
                idents: set[str] = set()
                try:
                    userinfo = ac.userinfo()
                    ids = userinfo["identity_set"]
                    idents.update(ident["sub"] for ident in ids)
                    log.debug(
                        "User-endpoint start requests are valid from identities: %s",
                        idents,
                    )
                    if not idents:
                        # Not a privileged user -- we require at least one identity
                        # against which to match start endpoint requests.
                        raise LookupError("No authorized identities found")

                except Exception as exc:
                    msg = "Failed to determine identity set; try `whoami` command?"
                    log.error(f"({type(exc).__name__}) {exc}\n    {msg}")
                    log.debug("Stopping; failed to determine identities", exc_info=exc)
                    self._time_to_stop = True

                return idents

            parent_identities.update(_get_globus_identities())
            del _get_globus_identities

        # Hopefully avoid apparent __del__-time spurious log messages after fork();
        # urllib3 connectionpool don't like us closing fds ahead of it, but we don't
        # have an avenue to tell it explicitly let go of them.  So, have Python do it
        # via the collector.
        gc.collect()

        last_heartbeat = 0.0
        valid_method_name_re = re.compile(r"^cmd_[A-Za-z][0-9A-Za-z_]{0,99}$")
        max_skew_s = 180  # 3 minutes; ignore commands with out-of-date timestamp
        while not self._time_to_stop:
            if self._wait_for_child:
                self.wait_for_children()

            if time.monotonic() - last_heartbeat >= self._heartbeat_period:
                self.send_heartbeat()
                last_heartbeat = time.monotonic()

            try:
                d_tag, props, body = self._command_queue.get(timeout=1.0)
                self._command.ack(d_tag)
                if props.headers and props.headers.get("debug", False):
                    body_log_b = _redact_url_creds(body, redact_user=False)
                    log.warning(
                        "Command debug requested:"
                        f"\n  Delivery Tag: {d_tag}"
                        f"\n  Properties: {props}"
                        f"\n  Body bytes: {body_log_b!r}"
                    )
            except queue.Empty:
                if self._command_stop_event.is_set():
                    self._time_to_stop = True
                if sys.stderr.isatty():
                    time_fmt = time.strftime("%c")
                    print(f"  ----> {time_fmt}\r", end="", flush=True, file=sys.stderr)
                continue

            try:
                server_cmd_ts = props.timestamp
                if props.content_type != "application/json":
                    raise ValueError("Invalid message type; expecting JSON")

                cmd_msg = json.loads(body)
                command = cmd_msg.get("command")
                command_args = cmd_msg.get("args", [])
                command_kwargs = cmd_msg.get("kwargs", {})
            except Exception as e:
                log.error(
                    "Unable to deserialize Globus Compute services command."
                    f"  ({e.__class__.__name__}) {e}"
                )
                continue

            now = round(time.time())
            if abs(now - server_cmd_ts) > max_skew_s:
                server_pp_ts = datetime.fromtimestamp(server_cmd_ts).strftime("%c")
                endp_pp_ts = datetime.fromtimestamp(now).strftime("%c")
                msg = (
                    "Ignoring command from server"
                    "\nCommand too old or skew between system clocks is too large."
                    f"\n  Command timestamp:  {server_cmd_ts} ({server_pp_ts})"
                    f"\n  Endpoint timestamp: {now} ({endp_pp_ts})"
                )
                log.warning(msg)
                self.send_failure_notice(command_kwargs, msg=msg)
                continue

            try:
                effective_identity = cmd_msg["globus_effective_identity"]
                identity_set = cmd_msg["globus_identity_set"]
                globus_username = cmd_msg["globus_username"]
            except Exception as e:
                msg = f"Invalid server command.  ({e.__class__.__name__}) {e}"
                log.error(msg)
                self.send_failure_notice(command_kwargs, msg=msg)
                continue

            identity_for_log = (
                f"\n  Globus effective identity: {effective_identity}"
                f"\n  Globus username: {globus_username}"
            )

            local_user_rec = None
            local_username = None
            mapped_idents = MappedPosixIdentity(
                local_user_record=self._mu_user,
                globus_identity_candidates=[],  # no mappings, initially
                matched_identity=None,
            )
            if not self.identity_mapper or parent_identities:
                # we are not a privileged user, so *only* allow the identity (or
                # linked identities) of the parent process auth'd to run tasks

                try:
                    cmd_identities = {ident["sub"] for ident in identity_set}
                except Exception as e:
                    log.debug(
                        "Invalid identity set: %s [({%s}) %s]",
                        identity_set,
                        type(e).__name__,
                        e,
                    )
                    cmd_identities = set()

                if not parent_identities.intersection(cmd_identities):
                    msg = (
                        "Ignoring start request for untrusted identity."
                        f"{identity_for_log}"
                    )
                    log.error(msg)
                    self.send_failure_notice(
                        command_kwargs, msg=msg, user_ident=identity_for_log
                    )
                    continue
                local_user_rec = self._mu_user
                local_username = self._mu_user.pw_name
                # in the no-idmap case, we did not run a mapper, so
                # `globus_identity_candidates` remains an empty list

            else:
                try:
                    idmaps = self.identity_mapper.map_identities(identity_set)
                    mapped_idents.globus_identity_candidates = idmaps
                    for mapped in idmaps:
                        if mapped:
                            first_found: dict = mapped[0]
                            ident, usernames = next(iter(first_found.items()))
                            local_username = usernames[0]
                            mapped_idents.matched_identity = ident
                            break

                    if not local_username:
                        raise LookupError()
                except LookupError as e:
                    msg = (
                        "Identity failed to map to a local user name."
                        f"  ({type(e).__name__}) {e}{identity_for_log}"
                    )
                    log.error(msg)
                    self.send_failure_notice(
                        command_kwargs, msg=msg, user_ident=identity_for_log
                    )
                    continue
                except Exception as e:
                    msg = "Unhandled error attempting to map to a local user name."
                    log.debug(f"{msg}{identity_for_log}", exc_info=e)
                    log.error(f"{msg}  ({type(e).__name__}) {e}{identity_for_log}")

                    fail_msg = f"{msg}{identity_for_log}"
                    self.send_failure_notice(
                        command_kwargs, msg=fail_msg, user_ident=identity_for_log
                    )
                    continue

                try:
                    local_user_rec = pwd.getpwnam(local_username)
                    mapped_idents.local_user_record = local_user_rec

                except Exception as e:
                    exc_type = type(e).__name__
                    msg = (
                        "  Identity mapped to a local user name, but local user does"
                        " not exist."
                        f"\n  Local user name: {local_username}{identity_for_log}"
                    )
                    log.error(f"({exc_type}) {e}\n{msg}")
                    fail_msg = f"({exc_type})\n{msg}"
                    self.send_failure_notice(
                        command_kwargs, msg=fail_msg, user_ident=identity_for_log
                    )
                    continue

            try:
                if not (command and valid_method_name_re.match(command)):
                    raise InvalidCommandError(f"Unknown or invalid command: {command}")

                command_func = getattr(self, command, None)
                if not command_func:
                    raise InvalidCommandError(f"Unknown or invalid command: {command}")

                command_func(mapped_idents, command_args, command_kwargs)
                log.info(
                    f"Command process successfully forked for '{local_username}'"
                    f" (Globus effective identity: {effective_identity})."
                )
            except (InvalidCommandError, InvalidUserError) as e:
                exc_type = type(e).__name__
                log.error(f"({exc_type}) {e}{identity_for_log}")
                msg = (
                    f"({exc_type}) unexpected error; this is due to either an endpoint"
                    " misconfiguration or a programming error.  If you are able to"
                    " recreate this error message at will, consider reaching out to the"
                    " endpoint administrator or the Globus Compute team."
                )
                self.send_failure_notice(
                    command_kwargs, msg=msg, user_ident=identity_for_log
                )

            except Exception:
                msg_a = _redact_url_creds(str(command_args), redact_user=False)
                msg_kw = _redact_url_creds(str(command_kwargs), redact_user=False)

                log.exception(
                    f"Unable to execute command: {command}\n"
                    f"    args: {msg_a}\n"
                    f"  kwargs: {msg_kw}{identity_for_log}"
                )
                self.send_failure_notice(command_kwargs, user_ident=identity_for_log)

    def send_failure_notice(
        self,
        kwargs: dict,
        msg: str | None = None,
        user_ident: str = "",
        fork: bool = True,
    ):
        """
        Given a set of AMQP credentials, send a message to the Compute web services
        that the given endpoint has failed to start up.

        This method conditionally forks (if ``fork == True``), but always exits.  The
        exit is always "clean" (exit code of 0) -- this is true even if there is an
        unhandled error as the assumption is that this is a last-ditch effort to be
        kind to the user (better UX).  If it fails, "oh well," and then it is time for
        the administrator to investigate the logs.

        :param kwargs: A structure containing an ``amqps_creds`` key.  This should
            match the structure as the web-service sends for a user-endpoint start
            command.  The credentials will be utilized to send a message to the AMQP
            service.

        :param msg: This parameter will be presented to the user via the SDK as the
            reason for failure (finishing any outstanding futures), so be mindful of
            values passed to this parameter (e.g., sensitive information).  If ``None``
            (as opposed to the empty string), then a default message will be sent.

        :param user_ident: utilized for logging purposes for the admin for when the
            forked process quits immediately after sending the message.
        """
        if fork:
            try:
                pid = os.fork()
            except Exception as e:
                log.error(f"Unable to fork child process: ({type(e).__name__}) {e}")
                raise

            if pid > 0:
                uep_info = [f"User endpoint name: {kwargs.get('name')}"]
                if user_ident:
                    uep_info.extend(i.strip() for i in user_ident.strip().split("\n"))
                info = "; ".join(uep_info)
                args = f"Temporary process to send failure message ({info})"
                self._children[pid] = UserEndpointRecord(
                    ep_name=f"{pid}", local_user_info=None, arguments=args
                )

                return

        try:
            send_endpoint_startup_failure_to_amqp(kwargs["amqp_creds"], msg=msg)
        except Exception:
            log.exception("Unable to send user endpoint start up failure")
        finally:
            sys.exit()

    @contextmanager
    def do_host_auth(self, username: str):
        def _affix_logd(prefix: str = "", suffix: str = ""):
            def _wrap(msg, *a, **k):
                log.debug(f"{prefix}{msg}{suffix}", *a, **k)

            return _wrap

        if not self._config.pam.enable:
            try:
                logd = _affix_logd(f"PRCTL ({username}): ")
                logd("Importing module")
                pyprctl = _import_pyprctl()
            except Exception:
                log.exception(f"({username}) Failed to import PRCTL library")
                raise PermissionError("see your system administrator") from None

            yield

            try:
                # If the administrator has *not* enabled PAM, then assume the
                # intention is for a paranoid safe process and drop all
                # privileges now ...
                logd("Dropping all process capabilities")
                pyprctl.CapState().set_current()

                # ... and stating that even if exec'ing might return some
                # privileges, "no."  In particular after this, SETUID executables
                # invoked from this process root will not get privileges
                logd("Allowing no new process privileges (no setuid executables!)")

                pyprctl.set_no_new_privs()
            except Exception:
                log.exception(f"({username}) Failed to import PRCTL library")
                raise PermissionError("see your system administrator") from None

            return

        sname = self._config.pam.service_name
        try:
            logd = _affix_logd(f"PAM ({sname}, {username}): ")
            logd("Importing module")
            pam = _import_pam()

            logd("Creating handle")
            with pam.PamHandle(sname, username=username) as pamh:
                logd("Invoking account stage")
                pamh.pam_acct_mgmt()
                logd("Creating credentials")
                pamh.credentials_establish()
                logd("Opening session")
                pamh.pam_open_session()

                yield

                # wiped by initgroups, so reinitialize
                logd("Recreating credentials")
                pamh.credentials_establish()
                logd("Closing session")
                pamh.pam_close_session()
                logd("Removing credentials")
                pamh.credentials_delete()

                logd("Closing handle")

        except pam.PamError as e:
            log.error(str(e))  # Share pamlib error with admin ...

            # ... but be opaque with user.
            raise PermissionError("see your system administrator") from None

        except Exception:
            log.exception(f"Unhandled error during PAM session for {username}")

            # Regardless, be opaque with user.
            raise PermissionError("see your system administrator") from None

    def cmd_start_endpoint(
        self,
        ident: MappedPosixIdentity,
        args: list[str] | None,
        kwargs: dict | None,
    ):
        if not args:
            args = []
        if not kwargs:
            kwargs = {}

        ep_name = kwargs.get("name", "")
        if not ep_name:
            raise InvalidCommandError("Missing endpoint name")

        for p, r in self._children.items():
            if r.ep_name == ep_name:
                log.info(
                    f"User endpoint {ep_name} is already running (pid: {p}); "
                    "caching arguments in case it's about to shut down"
                )
                self._cached_cmd_start_args[p] = (ident, args, kwargs)
                return

        user_record = ident.local_user_record
        udir, uid, gid = user_record.pw_dir, user_record.pw_uid, user_record.pw_gid
        uname = user_record.pw_name

        if self.identity_mapper and not self._allow_same_user:
            p_uname = self._mu_user.pw_name
            if uname == p_uname or uid == os.getuid():
                raise InvalidUserError(
                    "Requested UID is same as Manager Endpoint UID on a user-mapped"
                    " Manager Endpoint. To allow the same UID to run tasks, consider:"
                    "\n * using a non-root user,"
                    "\n * removing privileges from the UID, or"
                    "\n * removing the identity mapping configuration file"
                    f" ({self.identity_mapper.config_path})."
                    "\n\nDetails:"
                    f"\n  MU Process UID: {self._mu_user.pw_uid} ({p_uname})"
                    f"\n  Requested UID:  {uid} ({uname})"
                    f"\n  Via identity:   {ident.matched_identity}",
                )

        proc_args = [
            "globus-compute-endpoint",
            "start",
            ep_name,
            "--die-with-parent",
            *args,
        ]

        uep_amqp_creds: dict = kwargs["amqp_creds"]

        audit_r, audit_w = 0, 0
        if not self._audit_log_handler_stop:
            with self._audit_log_lock:
                audit_r, audit_w = os.pipe2(os.O_DIRECT)
                self._audit_pipes[audit_r] = {
                    "uid": uid,
                    "endpoint_id": uep_amqp_creds["endpoint_id"],
                }
                self._audit_selector.register(
                    audit_r, selectors.EVENT_READ, self._audit_log_write
                )

        try:
            pid = os.fork()
        except Exception as e:
            if audit_r:
                self._audit_log_close_reader(audit_r)
                os.close(audit_w)
            log.error(f"Unable to fork child process: ({e.__class__.__name__}) {e}")
            raise

        if pid > 0:
            proc_args_s = f"({uname}, {ep_name}) {' '.join(proc_args)}"
            self._children[pid] = UserEndpointRecord(
                ep_name=ep_name, local_user_info=user_record, arguments=proc_args_s
            )
            if audit_r:
                os.close(audit_w)
                self._audit_pipes[audit_r]["pid"] = pid
            log.info(f"Creating new user endpoint (pid: {pid}) [{proc_args_s}]")
            return

        # Reminder: from this point on, we are now the *child* process.
        pid = os.getpid()

        os.environ.clear()
        env = os.environ  # shorthand going forward

        exit_code = 70
        try:
            # only keep the stdio streams and audit.  We perform this operation
            # just before exec() as well, but no sense in keeping any parent fds
            # open for longer than necessary.
            close_all_fds(preserve_fds=[0, 1, 2, audit_w])

            os.chdir("/")  # always succeeds, so start from known place

            # in the child process; no need to load this in MUEP space
            import shutil
            from multiprocessing.process import current_process

            # hack to work with logging module; distinguish fork()ed process
            # beyond subtle pid: MainProcess-12345 --> UserEnd...(PreExec)-23456
            preexec_name = "UserEndpointProcess_Bootstrap(PreExec)"
            current_process().name = preexec_name

            from globus_compute_endpoint.logging_config import LOG_TS_FMT, setup_logging

            # We've closed all files (beyond std*), so log.* calls are not able to
            # access the parent's logs directly.  Now rely on stderr (not yet separated)
            # and fall back to the exit_code in the worst case.
            setup_logging(logfile=None, debug=log.getEffectiveLevel() <= logging.DEBUG)

            # load prior to dropping privileges
            template_str = load_user_config_template(self.user_config_template_path)
            user_config_schema = load_user_config_schema(self.user_config_schema_path)

            pybindir = pathlib.Path(sys.executable).parent
            default_path = ("/usr/local/bin", "/usr/bin", "/bin", pybindir)
            env.update({"PATH": ":".join(map(str, default_path))})
            env_path = self.conf_dir / "user_environment.yaml"
            try:
                log.debug("Load default environment variables from: %s", env_path)
                if env_path.exists():
                    env_text = env_path.read_text().strip()
                    env_data = yaml.safe_load(env_text)
                    if env_data:
                        os.environ.update({k: str(v) for k, v in env_data.items()})
                    else:
                        log.warning(
                            "Environment file read, but no variables parsed."
                            "  (Uncomment a line?  Editor buffer saved?)"
                            f"\n  Environment file: {env_path}"
                        )
                    del env_data, env_text
            except Exception as e:
                log.warning(
                    "Using only default environment variables as no variables read"
                    " from environment file.  Is it a valid YAML file?  (Hint: the"
                    " `yq` command line utility may be helpful.)"
                    f"\n Default environment: {dict(env)}"
                    f"\n    Environment file: {env_path}"
                    f"\n           Exception: [{type(e).__name__}] {e}"
                )
            user_home = {"HOME": udir, "USER": uname}

            if not os.path.isdir(udir):
                udir = "/"

            wd = env.get("PWD", udir)

            os.chdir("/")  # always succeeds, so start from known place
            exit_code += 1

            orig_uid, orig_gid = os.getuid(), os.getgid()
            if (orig_uid, orig_gid) != (uid, gid):
                # For template-only uses, there is no need to change the user.  But
                # for administrative installs, this is the expected path -- become the
                # identity-mapped user before doing anything else.

                with self.do_host_auth(uname):
                    log.debug("Setting process group for %s to %s", pid, gid)
                    os.setresgid(gid, gid, gid)  # raises (good!) on error
                    exit_code += 1

                    log.debug("Initializing groups for %s, %s", uname, gid)
                    os.initgroups(uname, gid)  # raises (good!) on error
                    exit_code += 1

                    log.debug("Setting process uid for %s to %s (%s)", pid, uid, uname)
                    os.setresuid(uid, uid, uid)  # raises (good!) on error
                    exit_code += 1

                try:
                    # Be paranoid by testing that we *can't* get back to orig_uid
                    os.setuid(orig_uid)
                except PermissionError:
                    pass  # good; the kernel has our back now
                else:
                    log.critical(
                        "Unexpectedly regained original privileges!  (Should not have"
                        f" been able to re-assume uid {orig_uid} from {uid}.)"
                    )

                    # This message is potentially (likely) sent back to the SDK; no
                    # sense in sharing the specifics (i.e., `msg`) beyond the
                    # administrator.
                    raise PermissionError("failed to start endpoint")
                del orig_uid, orig_gid

                exit_code += 1

            # Reminder: we are now the UID that will run the UEP.
            env.update(user_home)

            os.setsid()
            exit_code += 1

            umask = 0o077  # Let child process set less restrictive, if desired
            log.debug("Setting process umask for %s to 0o%04o (%s)", pid, umask, uname)
            os.umask(umask)
            exit_code += 1

            # some Q&D verification for admin debugging purposes
            if not shutil.which(proc_args[0]):
                log.warning(
                    "Unable to find executable."
                    f"\n  Executable (not found): {proc_args[0]}"
                    f'\n  Path: "{os.environ.get("PATH")}"'
                    f"\n\n  Will attempt exec anyway -- WARNING - it will likely fail."
                    f"\n  (pid: {pid}, user: {uname}, {ep_name})"
                    f"\n\nA common reason for this error is permissions; is the virtual"
                    " environment in a root-only accessible location, like /root/?"
                )

            log.debug("Changing directory to '%s'", wd)
            env["PWD"] = wd
            os.chdir(wd)
            exit_code += 1

            # in case "something gets stuck," let cmdline show it
            args_title = " ".join(proc_args)
            startup_proc_title = f"Endpoint starting up for {uname} [{args_title}]"
            setproctitle.setproctitle(startup_proc_title)

            gc_dir: pathlib.Path = ensure_compute_dir()
            ep_dir = gc_dir / ep_name
            ep_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
            ep_log = ep_dir / "endpoint.log"

            user_opts = kwargs.get("user_opts", {})
            user_runtime = kwargs.get("user_runtime", {})
            user_config = render_config_user_template(
                self._config,
                template_str,
                self.user_config_template_path,
                ident,
                user_config_schema,
                user_opts,
                user_runtime,
            )
            exit_code += 1
            _conf = yaml.safe_load(user_config)

            _ha_key = "high_assurance"
            if _ha_key in _conf:
                log.error(f"`{_ha_key}` may not be specified in template")
                raise ValueError("Error generating template; contact MEP administrator")

            if self._config.high_assurance:
                user_config = f"{_ha_key}: true\n{user_config}"
                _conf = yaml.safe_load(user_config)
            if bool(_conf.get(_ha_key)) ^ self._config.high_assurance:
                # final check that the configuration HAness aligns
                log.error(f"Unknown error generating correct template: `{_ha_key}`")
                raise ValueError("Error generating template; contact MEP administrator")

            ep_info: dict = {"posix_ppid": os.getppid()}
            if ident.matched_identity:
                ep_info.update(
                    globus_candidate_identities=ident.globus_identity_candidates,
                    globus_matched_identity=str(ident.matched_identity),
                )

            stdin_data_dict = {
                "amqp_creds": kwargs.get("amqp_creds"),
                "config": user_config,
                "ep_info": ep_info,
            }
            if self._config.allowed_functions is not None:
                stdin_data_dict["allowed_functions"] = self._config.allowed_functions
            if audit_w:
                stdin_data_dict["audit_fd"] = audit_w

            stdin_data = json.dumps(stdin_data_dict, separators=(",", ":"))
            exit_code += 1

            # Reminder: this is *os*.open, not *open*.  Descriptors will not be closed
            # unless we explicitly do so, so `null_fd =` in loop will work.
            null_fd = os.open(os.devnull, os.O_WRONLY, mode=0o200)
            while null_fd < 3:  # reminder 0/1/2 == std in/out/err, so ...
                # ... overkill, but "just in case": don't step on them
                null_fd = os.open(os.devnull, os.O_WRONLY, mode=0o200)
            exit_code += 1

            log.debug("Setting up process stdin")
            read_handle, write_handle = os.pipe()

            # fcntl.F_GETPIPE_SZ is not available in Python versions less than 3.10
            F_GETPIPE_SZ = 1032
            # 256 - Allow some headroom for multiple kernel-specific factors
            max_buf_size = fcntl.fcntl(write_handle, F_GETPIPE_SZ) - 256
            stdin_data_size = len(stdin_data)
            if stdin_data_size > max_buf_size:
                raise ValueError(
                    f"Unable to write {stdin_data_size} bytes of data to stdin; "
                    f"the maximum allowed is {max_buf_size} bytes"
                )

            exit_code += 1
            if os.dup2(read_handle, 0) != 0:  # close old stdin, use read_handle
                raise OSError("Unable to close stdin")
            os.close(read_handle)
            exit_code += 1

            log.debug("Convey credentials; redirect stdout, stderr (to '%s')", ep_log)
            log_fd_flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND | os.O_SYNC
            log_fd = os.open(ep_log, log_fd_flags, mode=0o600)
            with os.fdopen(log_fd, "w") as log_f:
                if os.dup2(log_f.fileno(), 1) != 1:
                    raise OSError(f"Unable to redirect stdout to {ep_log}")
                exit_code += 1
                if os.dup2(log_f.fileno(), 2) != 2:
                    raise OSError(f"Unable to redirect stderr to {ep_log}")

            # After the last os.dup2(), std* streams are sent to user's EP log file
            # and not the MEP's logs.  Use the exit_code as an avenue to share "what
            # went wrong where" to the parent process (the MEP).
            exit_code += 1

            if _conf.get("debug") is True:
                now = datetime.now().strftime(LOG_TS_FMT)
                num_lines = user_config.count("\n") + 1  # +1 ==> \n *splits* lines
                _rendered_config = user_config.replace("\n", "\n  | ")

                # Roughly approximate a `log.debug()` call, but don't leak some
                # details.  Minor, "but still."
                print(
                    f"{now} DEBUG {preexec_name} Endpoint Begin Compute endpoint"
                    f" configuration ({num_lines:,} lines):"
                    f"\n  | {_rendered_config}"
                    f"\nEnd Compute endpoint configuration"
                )

            with os.fdopen(write_handle, "w") as stdin_pipe:
                # intentional side effect: close handle
                stdin_pipe.write(stdin_data)

            # all parent process fds are long-since closed; for good measure,
            # ensure any interim fds are also closed.
            exit_code += 1
            close_all_fds(preserve_fds=[0, 1, 2, audit_w])

            exit_code += 1
            os.execvpe(proc_args[0], args=proc_args, env=env)

            # not executed, except perhaps in testing
            exit_code += 1  # type: ignore
        except Exception as e:
            msg = (
                f"Unable to start user endpoint process for {uname}"
                f" [exit code: {exit_code}; ({type(e).__name__}) {e}]"
            )
            log.error(msg)
            log.debug(f"Failed to exec for {uname}", exc_info=e)
            self.send_failure_notice(kwargs, msg=msg, fork=False)
        finally:
            # Only executed if execvpe fails (or isn't reached)
            sys.exit(exit_code)
