import argparse
import copy
import itertools
import json
import logging
import os
import subprocess
import sys
import re
import shutil
import tempfile
import importlib
import importlib.metadata
import platform
import urllib.request

# make sure that this run-time version check is in synch with the metadata for python requirement in the project.toml file.
if sys.version_info < (3,9):
    v = sys.version_info
    print(f"Python version 3.9 or above is required, the current one is {v.major}.{v.minor}.{v.micro} ({sys.executable}).") # e.g. str.removeprefix()
    sys.exit(1)

_logger = logging.getLogger(__file__)

COLOR_GRAY = "\033[38;20m"
COLOR_RED = "\033[1;31m"
COLOR_YELLOW = "\033[1;33m"
COLOR_CYAN = "\033[0;36m"
COLOR_GREEN = "\033[0;32m"
COLOR_RESET = "\033[0;0m"

coloring_enabled = sys.stdout.isatty() and sys.stderr.isatty()

def colored(color, x):
    global coloring_enabled
    return color + str(x) + COLOR_RESET if coloring_enabled else str(x)

def red(x): return colored(COLOR_RED, x)
def yellow(x): return colored(COLOR_YELLOW, x)
def cyan(x): return colored(COLOR_CYAN, x)
def green(x): return colored(COLOR_GREEN, x)
def gray(x): return colored(COLOR_GRAY, x)

def shell_esc(x): return x.replace("\u001b", "\\033")

SHELL_RED = shell_esc(COLOR_RED)
SHELL_YELLOW = shell_esc(COLOR_YELLOW)
SHELL_CYAN = shell_esc(COLOR_CYAN)
SHELL_GREEN = shell_esc(COLOR_GREEN)
SHELL_NOCOLOR = shell_esc(COLOR_RESET)

class ColoredLoggingFormatter(logging.Formatter):
    COLORS = {
        logging.DEBUG: COLOR_GREEN,
        logging.INFO: COLOR_GREEN,
        logging.WARNING: COLOR_YELLOW,
        logging.ERROR: COLOR_RED,
        logging.CRITICAL: COLOR_RED
    }

    def format(self, record):
        color = self.COLORS.get(record.levelno)
        format = colored(color, "%(levelname)s") + " %(message)s"
        formatter = logging.Formatter(format)
        return formatter.format(record)

def natural_sort_key(text):
    return [int(part) if part.isdigit() else part.lower() for part in re.split('([0-9]+)', text)]

def natural_less(a, b):
    return natural_sort_key(a) < natural_sort_key(b)

def natural_sorted(list):
    return sorted(list, key=natural_sort_key)

def project_natural_sort_key(project_description):
    return natural_sort_key(project_description.get_full_name())

def sorted_projects(project_description_list):
    return sorted(project_description_list, key=project_natural_sort_key)

def uniq(l):
    return list(dict.fromkeys(l))

def indent(txt, indent="    "):
    return indent + txt.replace("\n", "\n" + indent)

def join_lines(lines):
    assert type(lines) is list
    return "\n".join([li for li in lines if li])

def join_commands(commands):
    assert type(commands) is list
    return "\n".join([cmd for cmd in commands if cmd])  # using ";" as separator also works, " && " should too (script runs on 'set -e' anyway) but doesn't

def topological_sort(nodes, is_edge):
    visited = set()
    stack = []

    def dfs_rec(node):
        visited.add(node)
        for neighbor in nodes:
            if neighbor not in visited and is_edge(node, neighbor):
                dfs_rec(neighbor)
        stack.append(node)

    for node in nodes:
        if node not in visited:
            dfs_rec(node)

    stack.reverse()
    return stack

def sort_by_project_dependencies(project_descriptions):
    def depends_on(project_a, project_b):
        return project_b.name in project_a.required_projects
    sorted = topological_sort(project_descriptions, depends_on)
    return sorted

def is_semver(version):
    # supported formats: "3.2", "3.2.1", "3.2p1" or "3.2.1.231125"
    # note: this only VERY loosely based on https://semver.org/ (see BNF grammar there)
    pattern = r'^(\d+)\.(\d+)(?:[.p](\d+))?(?:\.(\d+))?$'
    return re.match(pattern, version) is not None

def parse_semver(version):
    # supported formats: "3.2", "3.2.1", "3.2p1"
    pattern = r'^(\d+)\.(\d+)(?:[.p](\d+))?(?:\.(\d+))?$'
    match = re.match(pattern, version)
    if match is None:
        raise ValueError('Invalid version string: ' + version)
    major, minor, micro, nano = match.groups()
    return int(major), int(minor), int(micro) if micro is not None else 0, int(nano) if nano is not None else 0

def version_matches(wildcard_version, version):
    if not re.match(r"^[^*?]+(\.\*)?$", wildcard_version):
        raise Exception(f"Unsupported version pattern '{wildcard_version}', only '.*' is allowed at the end")
    if wildcard_version.endswith(".*"):
        truncated = wildcard_version[0:-2]
        return version == truncated or version.startswith(truncated+".") or version.startswith(truncated+"p") # "3.3.*" should match "3.3p1" too
    else:
        return wildcard_version == version

def create_arg_parser():
    parser = argparse.ArgumentParser(prog="opp_env", description=
        "Provides automated installation of various versions of OMNeT++ and simulation frameworks -- enter 'opp_env list' for a list of supported projects. "
        "Uses Nix (nixos.org) to ensure a controlled software environment (compiler, libraries, tool) for building and running simulations. "
        "Projects are downloaded and built in dedicated directories called workspaces. "
        "Typical usage: mkdir workspace && cd workspace && opp_env init && opp_env install inet-latest && opp_env shell inet-latest",
        epilog="For command-specific help, type: opp_env COMMAND -h")
    parser.add_argument("--help-intro", default=False, action='store_true', help="Prints a short introduction to the opp_env tool")
    parser.add_argument("-d", "--debug", default=False, action='store_true', help="Equivalent to '--print-stacktrace --log-level DEBUG'")
    parser.add_argument("-l", "--log-level", choices=["ERROR", "WARN", "INFO", "DEBUG"], default="INFO", help="Log level of output")
    parser.add_argument("-p", "--print-stacktrace", default=False, action='store_true', help="Print stack trace on error")
    parser.add_argument("-v", "--version", action='version', version=get_version(), help="Print version information and exit")

    subparsers = parser.add_subparsers(help='', dest='subcommand', metavar='COMMAND')

    subparser = subparsers.add_parser("list", help="Lists all available projects", description="Lists all available projects")
    subparser.add_argument("project_name_patterns", nargs="*", metavar="project-name-or-pattern", help=
                           "Project names, project names with versions, or in general, regular expressions that match "
                           "the beginning of the project names with versions to be selected. Omit to list all projects.")
    group = subparser.add_mutually_exclusive_group()
    group.add_argument("--flat", dest="list_mode", action="store_const", const="flat", help="List projects with available versions, one per line")
    group.add_argument("--grouped", dest="list_mode", action="store_const", const="grouped", help="List the available versions for each project")
    group.add_argument("--names", dest="list_mode", action="store_const", const="names", help="List project names only (without version numbers)")
    group.add_argument("--descriptions", dest="list_mode", action="store_const", const="descriptions", help="List project names with descriptions only, one per line")
    group.add_argument("--aliases", dest="list_mode", action="store_const", const="aliases", help="List version aliases for projects")
    group.add_argument("--expand", dest="list_mode", action="store_const", const="expand", help="Expand dependency list of each matching project in the default version combination")
    group.add_argument("--expand-all", dest="list_mode", action="store_const", const="expand-all", help="Expand dependency list of each matching project in all supported version combinations")

    subparser = subparsers.add_parser("info", help="Describes the specified project", description="Describes the specified project")
    subparser.add_argument("projects", nargs="*", help="The list of projects to describe. You can specify exact versions like 'inet-4.0' or project names like 'inet'. The latter will print info on all versions of the project. An empty list prints info on all projects.")
    subparser.add_argument("--raw", action='store_true', default=False, help="Print the project descriptions in a raw form. The output is well-formed JSON, so you can use tools like 'jq' to further query it and extract the desired data.")
    subparser.add_argument("--options", action='append', metavar='name1,project:name2,...', help="Print the project description as if the given project options were selected")

    def add_argument(subparser, name):
        if name=="projects":     subparser.add_argument("projects", nargs="+", help=
            "List of projects with versions to work with, e.g. 'inet-4.2'. "
            "Abbreviated version numbers are understood as the latest minor/patchlevel version that "
            "matches the abbreviated version, e.g. 'inet-3' and 'inet-3.8' both refer to 'inet-3.8.3'. "
            "The pseudo-version 'latest' translates to the latest version of the project, e.g. 'inet-latest' stood for 'inet-4.5.0' at the time of writing. "
            "If the specified projects have dependencies, they will also be selected (unless the --no-deps option is present).")
        elif name=="projects-optional": subparser.add_argument("projects", nargs="*", help=
            "List of projects with versions to work with, e.g. 'inet-4.2'. "
            "Defaults to all projects in the workspace. "
            "Abbreviated version numbers are understood as the latest minor/patchlevel version that "
            "matches the abbreviated version, e.g. 'inet-3' and 'inet-3.8' both refer to 'inet-3.8.3'. "
            "The pseudo-version 'latest' translates to the latest version of the project, e.g. 'inet-latest' stood for 'inet-4.5.0' at the time of writing. "
            "If the specified projects have dependencies, they will also be selected (unless the --no-deps option is present).")
        elif name=="quiet":      subparser.add_argument("-q", "--quiet", dest="suppress_stdout", default=False, action='store_true', help="Suppress the standard output of executed commands")
        elif name=="force-init": subparser.add_argument("-f", "--force", default=False, action='store_true', help="Force turning a non-empty directory into a workspace")
        elif name=="init":       subparser.add_argument("--init", default=False, action='store_true', help=
            "Turn the current directory (or the directory specified via -w/--workspace) into a workspace if it is not already one")
        elif name=="workspace":  subparser.add_argument("-w", "--workspace", metavar='DIR', dest="workspace_directory", help=
            "Specifies the workspace directory. When this option is missing and opp_env needs to find the workspace, "
            "it will do so by searching up from the current directory.")
        elif name=="no-pause":   subparser.add_argument("-n", "--no-pause", dest="pause_after_warnings", default=True, action='store_false', help="Do not pause after printing warnings")
        elif name=="no-deps":    subparser.add_argument("--no-deps", "--no-dependency-resolution", dest="no_dependency_resolution", default=False, action='store_true', help=
            "Ignore dependencies among projects, only operate on the projects explicitly listed on the command line. "
            "This allows projects to be used together in previously untested or \"unofficial\" combinations.")
        elif name=="options":    subparser.add_argument("--options", action='append', metavar='[PROJECT:]NAME,...', help="Project options to use; use 'opp_env info' to see what options a selected project has. An option applies to all effective projects that support them, unless qualified with a project name.")
        elif name=="no-patch":   subparser.add_argument("--no-patch", dest="patch", default=True, action='store_false', help="Do not patch the project after download")
        elif name=="no-cleanup": subparser.add_argument("--no-cleanup", dest="cleanup", default=True, action='store_false', help="Do not delete a partially downloaded project if download or patching fails or is interrupted")
        elif name=="nixless-workspace":    subparser.add_argument("--nixless-workspace", default=False, action='store_true', help=
            "When creating a workspace, designate the new workspace as nixless. In a nixless workspace, projects are installed and run without Nix. "
            "A nixless workspace requires that all packages that the projects and opp_env itself require are already installed in the system. "
            "For opp_env itself, this translates to having 'curl' and/or 'git', and basic tools like 'tar' and 'gzip' available for downloading packages. "
            "The packages that OMNeT++ requires are documented in the Installation Guide of the particular version.")
        elif name=="keep":       subparser.add_argument("-k", "--keep", action='append', metavar='NAME,...', help=
            "Keep the specified environment variables, i.e. pass them into shells spawned by opp_env.")
        elif name=="local":      subparser.add_argument("--local", default=False, action='store_true', help=
            "Replaces internet access with file access. When specified, opp_env will use a local downloads directory and "
            "locally cloned Git repositories as installation sources instead of network access. "
            "It expects the file system locations to be passed in via environment variables. "
            "It is primarily useful for testing purposes.")
        elif name=="isolated":    subparser.add_argument("-i", "--isolated", action=argparse.BooleanOptionalAction, default=False, help=
            "Run in a Nix-based isolated environment from the host operating system. The default is to run non-isolated.")
        elif name=="no-isolated": subparser.add_argument("-i", "--isolated", action=argparse.BooleanOptionalAction, default=True, help=
            "Run in a Nix-based isolated environment from the host operating system. The default is to run in an isolated environment.")
        elif name=="mode":       subparser.add_argument("--mode", metavar='MODE,...', default="release,debug", help="Build mode(s), e.g. 'release' or 'debug', separated by commas.")
        elif name=="install":    subparser.add_argument("--install", dest='install', default=False, action='store_true', help="Download and build missing projects")
        elif name=="build":      subparser.add_argument("--build", dest='build', default=False, action='store_true', help="Build projects")
        elif name=="no-build":   subparser.add_argument("--no-build", dest='install_without_build', default=False, action='store_true', help="Do not build the projects after download")
        elif name=="smoke-test": subparser.add_argument("--smoke-test", dest='run_smoke_test', default=False, action='store_true', help="Run a short test to ensure that the project executables are working")
        elif name=="test":       subparser.add_argument("--test", dest='run_test', default=False, action='store_true', help="Run the project's test suite to ensure it is working correctly")
        elif name=="chdir":      subparser.add_argument("--chdir", action=argparse.BooleanOptionalAction, default="convenience", help=
            "Whether to change into the workspace directory (--chdir), or stay in the current working directory (--no-chdir). "
            "If neither is given, the default action to try doing what is likely the most convenient for the user, "
            "which is to change into the root of the (first) project if the current working directory is outside that project's directory tree, "
            "and stay in the current directory (inside the project) otherwise.")
        elif name=="command":    subparser.add_argument("-c", "--command", help="Specifies the command that is run in the environment")
        else: raise Exception(f"Internal error: unrecognized option name '{name}'")

    def add_arguments(subparser, names):
        for name in names:
            add_argument(subparser, name)

    subparser = subparsers.add_parser("init", help="Designates the current working directory to be an opp_env workspace", description="Designates the current working directory to be an opp_env workspace")
    add_arguments(subparser, [
        "workspace",
        "force-init",
        "nixless-workspace"
    ])

    subparser = subparsers.add_parser("install", help="Downloads and builds the specified projects in their environment",
        description="Downloads and builds the specified projects in their environment. Skips the download step if project is already downloaded.")
    add_arguments(subparser, [
        "projects",
        "quiet",
        "init",
        "nixless-workspace",
        "workspace",
        "options",
        "no-deps",
        "no-pause",
        "no-cleanup",
        "no-patch",
        "no-build",
        "mode",
        "no-isolated",
        "keep",
        "local"
    ])

    subparser = subparsers.add_parser("shell", help="Runs a shell in the environment of the specified projects",
        description="Runs a shell in the environment of the specified projects. It can also be asked to perform workspace initialization and project installation as well.")
    add_arguments(subparser, [
        "projects-optional",
        "init",
        "nixless-workspace",
        "workspace",
        "options",
        "chdir",
        "no-deps",
        "no-pause",
        "no-cleanup",
        "no-patch",
        "install",
        "no-build", # with --install
        "build",
        "mode",
        "quiet",
        "isolated",
        "keep",
        "local"
    ])

    subparser = subparsers.add_parser("run", help="Runs a command in the environment of the specified projects",
        description="Runs a command in the environment of the specified projects. It can also be asked to perform workspace initialization and project installation as well.")
    add_arguments(subparser, [
        "projects-optional",
        "init",
        "nixless-workspace",
        "workspace",
        "command",
        "options",
        "no-deps",
        "no-pause",
        "no-cleanup",
        "no-patch",
        "install",
        "no-build", # with --install
        "build",
        "smoke-test",
        "test",
        "mode",
        "quiet",
        "no-isolated",
        "keep",
        "local"
    ])

    subparser = subparsers.add_parser("maint", help="Maintenance functions", description="Maintenance functions")
    subparser.add_argument("-u", "--update-catalog", metavar="download-items-dir", dest="catalog_dir", help="Update the opp_env installation commands in the model catalog of omnetpp.org. The argument should point to the `download-items/` subdir of a checked-out copy of the https://github.com/omnetpp/omnetpp.org/ repository.")

    return parser

def process_arguments():
    parser = create_arg_parser()
    args = parser.parse_args(sys.argv[1:])
    if args.help_intro:
        print_intro()
    elif args.subcommand == None:
        parser.print_help()

    if args.debug:
        args.log_level = "DEBUG"
        args.print_stacktrace = True

    handler = logging.StreamHandler()
    handler.setFormatter(ColoredLoggingFormatter())
    _logger.setLevel(args.log_level)
    _logger.handlers = []
    _logger.addHandler(handler)
    kwargs = {k: v for k, v in vars(args).items() if v is not None}
    if "workspace_directory" in kwargs:
        kwargs["workspace_directory"] = os.path.abspath(kwargs["workspace_directory"])
    if "options" in kwargs:
        # split up and flatten list
        kwargs["requested_options"] = [name for arg in args.options for name in arg.split(",")]
        del kwargs["options"]
    if "keep" in kwargs:
        # split up and flatten list
        kwargs["vars_to_keep"] = [name for arg in args.keep for name in arg.split(",")]
        del kwargs["keep"]
    if "mode" in kwargs:
        kwargs["build_modes"] = args.mode.split(",")
        del kwargs["mode"]
    return kwargs


def print_intro():
    print("""To see all the supported simulation models, run:
  opp_env list          

To create a new workspace and install the latest version of OMNeT++, run:
  mkdir workspace && cd workspace && opp_env init && opp_env install omnetpp-latest && opp_env shell omnetpp-latest

To install and use the latest version of the INET Framework with a single command, run:
  opp_env shell --install inet-latest
          
To run a simulation model directly with the latest version of OMNeT++, run:
  opp_env run --install omnetpp-latest -c 'cd $OMNETPP_ROOT/samples/aloha;./aloha'""")

def get_version():
    return importlib.metadata.version("opp_env")

def detect_nix():
    minimum_nix_version = "2.9"
    # check nix is installed
    try:
        _logger.debug(f"Running nix --version")
        result = subprocess.run(['nix', '--version'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
        output = result.stdout.decode('utf-8')
    except Exception as ex:
        _logger.debug(f"Error: {ex}")
        raise Exception(f"Nix does not seem to be installed (running `nix --version` failed). You can install it from https://nixos.org/download.html or using your system's package manager (important: at least version {minimum_nix_version} is required). See also the --nixless-workspace option in the help.")

    # check it is recent enough
    nix_version = output.strip().split()[-1]
    if not re.match("^[0-9.]+$", nix_version):
        raise Exception("Cannot parse Nix version number: Output of 'nix --version' diverges from expected format")
    if natural_less(nix_version, minimum_nix_version):
        raise Exception(f"Your Nix installation of version {nix_version} is too old, at least version {minimum_nix_version} is required. The newest version is available from https://nixos.org/download.html. See also the --nixless-workspace option in the help.")

def detect_tools():
    tools = [ "bash", "git", "curl", "grep", "find", "xargs", "shasum", "tar", "gzip", "sed", "touch" ]

    is_macos = platform.system().lower() == "darwin"
    if not is_macos:
        tools.append("nproc")

    errors = []
    for tool in tools:
        try:
            result = subprocess.run([tool, '--version'], capture_output=True)
        except Exception as ex:
            errors.append(tool)
    if errors:
        raise Exception(f"The following programs were not found: {', '.join(errors)}.")

class ProjectDescription:
    def __init__(self, name, version, description=None, details=None, warnings=[],
                 nixos=None, stdenv=None, folder_name=None,
                 required_projects={}, nix_packages=[], vars_to_keep=[],
                 download_url=None, git_url=None, git_branch=None, download_commands=[],
                 patch_commands=[], patch_url=None,
                 shell_hook_commands=[], setenv_commands=[],
                 build_commands=[], clean_commands=[], smoke_test_commands=[], test_commands=[],
                 potential_build_inputs=None, potential_build_outputs=None,
                 options=None, metadata=None):
        def remove_empty(list):
            return [x for x in list if x] if list else []
        self.name = name
        self.version = version
        self.description = description
        self.details = details
        self.warnings = remove_empty(warnings)
        self.nixos = nixos
        self.stdenv = stdenv
        self.folder_name = folder_name or name
        self.required_projects = required_projects
        self.nix_packages = remove_empty(nix_packages)
        self.vars_to_keep = remove_empty(vars_to_keep)
        self.download_url = download_url
        self.git_url = git_url
        self.git_branch = git_branch
        self.download_commands = remove_empty(download_commands)
        self.patch_commands = remove_empty(patch_commands)
        self.patch_url = patch_url
        self.shell_hook_commands = remove_empty(shell_hook_commands)
        self.setenv_commands = remove_empty(setenv_commands)
        self.build_commands = remove_empty(build_commands)
        self.clean_commands = remove_empty(clean_commands)
        self.smoke_test_commands = remove_empty(smoke_test_commands)
        self.test_commands = remove_empty(test_commands)
        self.potential_build_inputs = potential_build_inputs or [ "src/*", "*.cc", "*.cxx", "*.c", "*.h", "*.hpp", "*.hh", "*.msg", "Makefile", "*/Makefile", "makefrag", "*/makefrag" ]
        self.potential_build_outputs = potential_build_outputs or [ "out/*", "*.o", "*.a", "*.a.*", "*.so", "*.so.*", "*.dylib", "*.dylib.*", "*.dll", "*.exe", ":noext" ]
        self.options = options or {}
        self.metadata = metadata or {}  # examples: catalog_url, release_year, original_version

        # remove null elements from lists inside options, too
        for option_name, option_entries in self.options.items():
            for field_name, field_value in option_entries.items():
                if type(field_value) is list:
                    field_value[:] = remove_empty(field_value)

        if bool(download_url) + bool(git_url) + bool(download_commands) > 1:
            raise Exception(f"project {name}-{version}: download_url, git_url, and download_commands are mutually exclusive")

        if self.description and "\n" in self.description:
            raise Exception(f"project {name}-{version}: description may not contain newlines -- use the details field to store additional information")
        if self.description and len(self.description) > 180:
            raise Exception(f"project {name}-{version}: description may not be longer than 180 characters (currently {len(self.description)}) -- use the details field to store additional information")

    def __repr__(self):
        return self.get_full_name()

    def __str__(self):
        return self.get_full_name()

    def get_full_name(self, colored=False):
        full_name = self.name + "-" + self.version
        return cyan(full_name) if colored else full_name

    def get_full_folder_name(self):
        return f"{self.folder_name}-{self.version}"

    def get_supported_options(self):
        return list(self.options.keys())

    def get_default_options(self):
        return [option_name for option_name, option_entries in self.options.items() if option_entries.get("option_is_default")]

    def activate_project_options(self, requested_options, activate_default_options=True, quiet=False):
        def get_conflicting_options(the_option_name, option_names):
            return [o for o in option_names if the_option_name != o and self.options[o].get("option_category") == self.options[the_option_name].get("option_category")]

        # activate requested options, and those of the default options that don't conflict with the requested ones
        effective_options = []
        for option in requested_options or []:
            prefix,_,option = option.rpartition(":") # allows for project-specific options, e.g. "inet:from-git-repo"
            if option in self.options and (not prefix or prefix == self.name):
                conflicting_options = get_conflicting_options(option, effective_options)
                if conflicting_options:
                    raise Exception(f"Option '{option}' conflicts with option '{conflicting_options[0]}' due to both belonging in the category '{self.options[option].get('category')}' (Note that options in the same category are exclusive)")
                effective_options.append(option)
        if activate_default_options:
            for option in self.get_default_options():
                if not get_conflicting_options(option, effective_options):
                    effective_options.append(option)

        new_project_description = copy.deepcopy(self)

        if effective_options:
            if not quiet:
                _logger.debug(f"Selecting options {cyan(requested_options)} for project {cyan(self)}")
            for option in effective_options:
                if option in self.options:
                    for field_name, field_value in self.options[option].items():
                        if not re.match("option_", field_name): # not option metadata
                            setattr(new_project_description, field_name, field_value)
                else:
                    _logger.warning(f"Project {cyan(self)} does not support option {cyan(option)}")
        return new_project_description

class ProjectReference:
    def __init__(self, name, version):
        self.name = name
        self.version = version

    def __repr__(self):
        return self.get_full_name()

    def __str__(self):
        return self.get_full_name()

    @staticmethod
    def parse(string):
        # split to name and version
        return ProjectReference(*string.rsplit("-", 1)) if "-" in string else ProjectReference(string, "")

    def get_full_name(self):
        return self.name + "-" + self.version if self.version else self.name

class ProjectRegistry:
    def __init__(self):
        self.all_project_descriptions = self.collect_project_descriptions()
        self.index = self.build_index(self.all_project_descriptions)

    def collect_project_descriptions(self):
        python_files = [
            "omnetpp",
            "inet",
            "veins",
            "simulte",
            "simu5g",
            "external"
        ]

        json_files = [
            "external.json"
        ]

        all_project_descriptions = []
        for fname in python_files:
            module = importlib.import_module("opp_env.database." + fname)
            raw_project_descriptions = module.get_project_descriptions()
            project_descriptions = [ProjectDescription(**e) for e in raw_project_descriptions]
            all_project_descriptions += project_descriptions

        for fname in json_files:
            with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "database", fname)) as f:
                all_project_descriptions += [ProjectDescription(**e) for e in json.load(f)]

        # expand to wildcard versions such as "4.2.*" to list of matching versions
        return [self.expand_wildcards_in_project_dependencies(p, all_project_descriptions) for p in all_project_descriptions]

    def get_all_project_descriptions(self):
        return self.all_project_descriptions

    def get_project_names(self, project_descriptions=None):
        return uniq([p.name for p in project_descriptions or self.get_all_project_descriptions()])

    def get_project_versions(self, project_name, project_descriptions=None):
        # Note: this does not include "pseudo" versions like "latest", or "omnetpp-4" that means "omnetpp-4.6.1"
        return [p.version for p in project_descriptions or self.get_all_project_descriptions() if p.name == project_name]

    def build_index(self, project_descriptions):
        # index structure: { name: {version: description}}
        index = {}
        versions = {}
        for project in project_descriptions:
            project_name = project.name
            project_version = project.version
            if project_name not in index:
                index[project_name] = {}
                versions[project_name] = []
            index[project_name][project_version] = project
            versions[project_name].append(project_version)

        # add meta entries: "latest"; "3", "3.8" -> 3.8.2 (latest minor/patch version)
        # ordering is determined by order of items in the project_descriptions array.
        for project_name, project_versions in versions.items():
            semver_versions = [v for v in project_versions if is_semver(v)] # exclude weird versions, and ".x" versions like "6.0.x" (6.0 or "latest" should NOT resolve to 6.0.x)
            index[project_name]["latest"] = index[project_name][semver_versions[0]] if semver_versions else index[project_name][project_versions[0]]
            for version in semver_versions:
                truncated_version = version
                while "." in truncated_version:
                    truncated_version = truncated_version.rsplit(".",1)[0] # chop off part after last dot
                    if truncated_version not in index[project_name]:
                        index[project_name][truncated_version] = index[project_name][version]
        return index

    def get_project_version_aliases(self, project_reference):
        # collect version aliases for a given project; e.g. if project_reference is "omnetpp-6.0.2", then it may return ["6", "6.0", "latest"]
        version_to_project_dict = self.index[project_reference.name]
        return [v for v,p in version_to_project_dict.items() if p.version != v and p.version == project_reference.version]

    def get_project_description(self, project_reference):
        if type(project_reference) is str:
            project_reference = ProjectReference.parse(project_reference)
        if project_reference.name not in self.index:
            raise Exception(f"Cannot resolve '{project_reference}': " + self.get_unknown_project_message(project_reference.name))
        if not project_reference.version:
            raise Exception(f"Which version of '{project_reference.name}' do you mean? (Use '{project_reference.name}-latest' for latest version)")
        project_description = self.index[project_reference.name].get(project_reference.version)
        if not project_description:
            raise Exception(f"Project '{project_reference.name}' has no version '{project_reference.version}'")
        if project_reference.version != project_description.version:
            _logger.debug(f"Resolved {cyan(project_reference)} as {cyan(project_description)}")
        return project_description

    def expand_wildcards_in_project_dependencies(self, project_description, all_project_descriptions):
        def expand(project_name, version, all_project_descriptions):
            if not '*' in version:
                return [ version ]
            candidates = self.get_project_versions(project_name, all_project_descriptions)
            return [ candidate for candidate in candidates if version_matches(version, candidate) ]

        def expand_all(project_name, versions, all_project_descriptions):
            result = []
            for version in versions:
                result += expand(project_name, version, all_project_descriptions)
            return result

        project_description.required_projects = { project_name: expand_all(project_name, versions, all_project_descriptions)
            for project_name, versions in project_description.required_projects.items() }

        return project_description

    def compute_effective_project_descriptions(self, specified_project_descriptions, requested_options=None):
        selected_project_descriptions = self.expand_dependencies(specified_project_descriptions)
        if not selected_project_descriptions:
            raise Exception("The specified set of project versions cannot be satisfied")
        return activate_project_options(selected_project_descriptions, requested_options)

    def expand_dependencies(self, specified_project_descriptions, return_all=False):
        _logger.debug(f"Computing list of effective projects for {specified_project_descriptions}")
        # 1. collect all required projects ignoring the project versions
        required_project_names = []
        todo_list = specified_project_descriptions.copy()
        while todo_list:
            project_description = todo_list.pop(0)
            required_project_names.append(project_description.name)
            for project_name, project_versions in project_description.required_projects.items():
                # maintains the proper ordering of required projects
                if project_name in required_project_names:
                    required_project_names = [e for e in required_project_names if e != project_name]
                else:
                    todo_list.append(self.get_project_description(ProjectReference.parse(project_name + "-" + project_versions[0])))
                required_project_names.append(project_name)
        required_project_names.reverse()
        # _logger.debug(f"{required_project_names=}")

        # 2. collect all available project versions for all required projects separately
        available_project_versions = { name: self.get_project_versions(name) for name in required_project_names }
        # _logger.debug(f"{available_project_versions=}")

        # 3. iterate over all combinations of the available project versions for the different required projects
        result = []
        sets = available_project_versions.values()
        keys = list(available_project_versions.keys())
        for combination in itertools.product(*sets):
            # _logger.debug(f"checking combination: {combination=}")
            accept_combination = True
            selected_project_descriptions = []
            # 4. turn the combination of version numbers (a tuple of strings) into a tuple of project descriptions
            for i in range(len(combination)):
                selected_project_name = f"{keys[i]}-{combination[i]}"
                selected_project_description = self.get_project_description(ProjectReference.parse(selected_project_name))
                selected_project_descriptions.append(selected_project_description)
            # _logger.debug(f"checking combination: {selected_project_descriptions=}")
            # 5. check if the specified project versions are included in the project version combination
            for specified_project_description in specified_project_descriptions:
                if not specified_project_description in selected_project_descriptions:
                    # _logger.debug(f"  rejecting because {specified_project_description} is not in {selected_project_descriptions}")
                    accept_combination = False
                    break
            # 6. check if one of the required project versions are included in the project version combination for all project versions
            for selected_project_description in selected_project_descriptions:
                # _logger.debug(f"  checking {selected_project_description}")
                for required_project_name, required_project_versions in selected_project_description.required_projects.items():
                    accept_selected_project_description = False
                    for required_project_version in required_project_versions:
                        required_project_description = self.get_project_description(ProjectReference.parse(required_project_name + "-" + required_project_version))
                        if required_project_description in selected_project_descriptions:
                            accept_selected_project_description = True
                    if not accept_selected_project_description:
                        # _logger.debug(f"  rejecting {required_project_name} {required_project_versions}")
                        accept_combination = False
                        break
            if accept_combination:
                selected_project_descriptions = list(reversed(selected_project_descriptions))  # most derived project first, omnetpp last (usually)
                if return_all:
                    result.append(selected_project_descriptions)
                else:
                    return selected_project_descriptions
        return result

    def get_unknown_project_message(self, project_name):
        def jaccard_similarity(word1, word2):
            set1 = set(word1)
            set2 = set(word2)
            intersection = len(set1.intersection(set2))
            union = len(set1) + len(set2) - intersection
            similarity = intersection / union if union != 0 else 0
            return similarity
        names = self.get_project_names()
        most_similar =  max(names, key=lambda p: jaccard_similarity(project_name.lower(), p.lower()))
        more_candidates = [p for p in names if p!=most_similar and (p.lower() in project_name.lower() or project_name.lower() in p.lower())]
        if more_candidates:
            return f"Unknown project '{project_name}'. Did you mean one of {[most_similar] + more_candidates}?"
        else:
            return f"Unknown project '{project_name}'. Did you mean '{most_similar}'?"

def activate_project_options(project_descriptions, requested_options):
    # check requested options exist at all
    all_supported_options = []
    for desc in project_descriptions:
        all_supported_options += desc.get_supported_options()
    all_supported_options = uniq(all_supported_options)
    for option in requested_options or []:
        prefix,_,option = option.rpartition(":") # allows for project-specific options, e.g. "inet:from-git-repo"
        if prefix:
            # the project selected with the prefix (e.g. "inet:") must have such option
            targeted_projects = [p for p in project_descriptions if p.name == prefix]
            if not targeted_projects:
                raise Exception(f"Invalid option '{prefix}:{option}': There is no project named '{prefix}' among the selected projects")
            targeted_project = targeted_projects[0]  # reasonably, there can be only one matching
            if not option in targeted_project.get_supported_options():
                raise Exception(f"Invalid option '{prefix}:{option}': Project '{targeted_project}' has no option named '{option}'")
        else:
            if option not in all_supported_options:
                raise Exception(f"None of the selected projects supports option '{option}'")
    # create and return updated project descriptions
    return [desc.activate_project_options(requested_options) for desc in project_descriptions]

class Workspace:
    # project states
    ABSENT = "ABSENT"
    INCOMPLETE = "INCOMPLETE"
    DOWNLOADED = "DOWNLOADED"

    WORKSPACE_ADMIN_DIR = ".opp_env_workspace"
    PROJECT_ADMIN_DIR = ".opp_env"

    def __init__(self, root_directory, default_nixos=None, default_stdenv=None):
        assert(os.path.isabs(root_directory))
        self.root_directory = root_directory
        self.default_nixos = default_nixos or "22.11"
        self.default_stdenv = default_stdenv or "llvmPackages.stdenv"

        opp_env_directory = os.path.join(self.root_directory, self.WORKSPACE_ADMIN_DIR)
        if not os.path.exists(opp_env_directory):
            raise Exception(f"'{root_directory}' is not an opp_env workspace, run 'opp_env init' to turn it into one")
        self.nixless = os.path.exists(os.path.join(self.get_workspace_admin_directory(), ".nixless"))  #TODO do it properly!!!

        if self.nixless:
            detect_tools()
        else:
            detect_nix()

        _logger.debug(f"Workspace {root_directory=}, {self.nixless=}")

    @staticmethod
    def is_workspace(dir):
        return os.path.isdir(os.path.join(dir, Workspace.WORKSPACE_ADMIN_DIR))

    @staticmethod
    def find_workspace(from_dir=None):
        dir = os.path.abspath(from_dir) if from_dir else os.getcwd()
        while True:
            if os.path.isdir(os.path.join(dir, Workspace.WORKSPACE_ADMIN_DIR)):
                return dir
            parent_dir = os.path.dirname(dir)
            if parent_dir == dir:
                break
            dir = parent_dir
        raise Exception(f"No opp_env workspace found in '{from_dir}' or its parent directories, run 'opp_env init' to create one")
        #return None

    @staticmethod
    def init_workspace(dir=None, allow_existing=False, nixless=False):
        if not dir:
            dir = os.getcwd()
        if not os.path.isdir(dir):
            raise Exception(f"Directory does not exist: {dir}")
        opp_env_dir = os.path.join(dir, Workspace.WORKSPACE_ADMIN_DIR)
        if os.path.isdir(opp_env_dir):
            if allow_existing:
                _logger.debug(f"Using existing workspace in folder {cyan(dir)}")
                return
            else:
                raise Exception(f"'{dir}' is already an opp_env workspace")
        if re.search("\\s", dir):
            raise Exception(f"Whitespace characters are not allowed in the name and path of the workspace directory")
        shutil.copytree(os.path.join(os.path.dirname(__file__), "templates", "workspace"), opp_env_dir)
        if nixless:
            # write an empty file called .nixless to indicate that this is a nixless workspace
            open(os.path.join(opp_env_dir, ".nixless"), "w").close()
        _logger.info(f"Workspace created in folder {cyan(dir)}")

    def get_workspace_admin_directory(self):
        return os.path.join(self.root_directory, self.WORKSPACE_ADMIN_DIR)

    def get_installed_projects(self):
        def is_project(folder_name):
            return os.path.isdir(os.path.join(self.root_directory, folder_name, self.PROJECT_ADMIN_DIR))
        def get_project_description(folder_name):
            try:
                global project_registry
                return project_registry.get_project_description(folder_name)
            except Exception as e:
                _logger.warning(f"Failed to load project description for '{folder_name}': {e}")
                return None
        result = [get_project_description(folder) for folder in os.listdir(self.root_directory) if is_project(folder)]
        return [p for p in result if p]

    def get_project_root_directory(self, project_description):
        return os.path.join(self.root_directory, project_description.get_full_folder_name())

    def get_project_admin_directory(self, project_description, create=False):
        dir = os.path.join(self.get_project_root_directory(project_description), self.PROJECT_ADMIN_DIR)
        if create and not os.path.isdir(dir):
            os.mkdir(dir)
        return dir

    def get_project_admin_file(self, project_description, filename, create_dir=False):
        return os.path.join(self.get_project_admin_directory(project_description, create=create_dir), filename)

    def is_project_modified(self, project_description):
        postdownload_shasums = self.read_project_shasums(project_description, "postdownload")
        last_shasums = self.read_project_shasums(project_description, "last")
        new_files, disappeared_files, changed_files = self.compare_shasums(postdownload_shasums, last_shasums)
        self.print_shasums_comparison_result(new_files, disappeared_files, changed_files, label=f"File changes in {project_description} since download", root_dir=self.get_project_root_directory(project_description))
        return disappeared_files or changed_files # new files do not count  (TODO or: should count, except for build outputs?)

    def read_project_state_file(self, project_description):
        state_file_name = self.get_project_admin_file(project_description, "state")
        if not os.path.isfile(state_file_name):
            return {}
        with open(state_file_name) as f:
            return json.load(f)

    def write_project_state_file(self, project_description, data):
        state_file_name = self.get_project_admin_file(project_description, "state", create_dir=True)
        with open(state_file_name, "w") as f:
            json.dump(data, f)

    def get_project_status(self, project_description):
        project_directory = self.get_project_root_directory(project_description)
        data = self.read_project_state_file(project_description)
        return self.ABSENT if not os.path.isdir(project_directory) else \
               self.INCOMPLETE if not data else \
               self.DOWNLOADED

    def update_project_state(self, project_description, **kwargs):
        data = self.read_project_state_file(project_description)
        data.update(kwargs)
        self.write_project_state_file(project_description, data)

    def download_project(self, project_description, effective_project_descriptions, patch=True, cleanup=True, local=False, **kwargs):
        def get_env(varname, what):
            value = os.environ.get(varname)
            _logger.debug(f"Checking {cyan('$'+varname)} for {what}: {cyan(value)}")
            if not value:
                raise Exception(f"Environment variable {varname} not set, it should point to {what}")
            return value

        _logger.info(f"Downloading project {cyan(project_description.get_full_name())} in workspace {cyan(self.root_directory)}")
        project_dir = self.get_project_root_directory(project_description)
        if os.path.exists(project_dir):
            raise Exception(f"{project_dir} already exists")
        try:
            if project_description.download_commands:
                commands = [ f"export LOCAL_OPERATION={'1' if local else ''}", *project_description.download_commands ]
                self.nix_develop(effective_project_descriptions, self.root_directory, commands, run_setenv=False, **kwargs)
            elif project_description.download_url:
                if not local:
                    self.download_and_unpack_tarball(project_description.download_url, project_dir)
                else:
                    downloads_dir = get_env("DOWNLOADS_DIR", "the downloads directory on the local disk")
                    fname = os.path.basename(project_description.download_url)
                    if project_description.name.lower() not in fname.lower():  # e.g. just "v1.2.0.tar.gz"
                        fname = project_description.name() + "-" + fname
                    tarball = os.path.join(downloads_dir, fname)
                    self.unpack_tarball(tarball, project_dir)
            elif project_description.git_url:
                if not local:
                    git_url = project_description.git_url
                else:
                    git_url = get_env(project_description.name.upper() + "_REPO", f"the location of the '{project_description.name}' git repository on the local disk")
                branch_option = "-b " + project_description.git_branch if project_description.git_branch else ""
                self.run_command(f"git clone --config advice.detachedHead=false {branch_option} {git_url} {project_dir}") #TODO maybe optionally use --single-branch
            else:
                raise Exception(f"{project_description}: No download_url or download_commands in project description -- check project options for alternative download means (enter 'opp_env info {project_description}')")
            if not os.path.exists(project_dir):
                raise Exception(f"{project_description}: Download process did not create {project_dir}")

            if project_description.patch_commands or project_description.patch_url:
                if patch:
                    _logger.info(f"Patching project {cyan(project_description.get_full_name())}")
                    if project_description.patch_url:
                        self.download_and_apply_patch(project_description.patch_url, project_dir)
                    if project_description.patch_commands:
                        commands = [ f"export LOCAL_OPERATION={'1' if local else ''}", *project_description.patch_commands ]
                        self.nix_develop(effective_project_descriptions, project_dir, commands, run_setenv=False, **kwargs)
                else:
                    _logger.info(f"Skipping patching step of project {cyan(project_description.get_full_name())}")

            self.update_project_state(project_description, name=project_description.get_full_name())
            self.record_project_shasums(project_description, "postdownload")
        except KeyboardInterrupt as e:
            if cleanup:
                _logger.info("Download interrupted by user, cleaning up")
                if os.path.isdir(project_dir):
                    shutil.rmtree(project_dir)
            raise e
        except Exception as e:
            if cleanup:
                _logger.info("Error during download, cleaning up")
                if os.path.isdir(project_dir):
                    shutil.rmtree(project_dir)
            raise e

    def record_project_shasums(self, project_description, snapshot_name):
        # exclude the Simulation IDE's directory from the shasum, because ./configure and eclipse itself modifies stuff in it
        project_root = self.get_project_root_directory(project_description)
        shasum_file = self.get_project_admin_file(project_description, snapshot_name+".sha", create_dir=True)
        # note: we chdir into project_root so that the shasum file will contain project-relative paths not absolute ones
        self.run_command(f"cd {project_root} && find . \\( -path ./{self.PROJECT_ADMIN_DIR} -o -path ./ide \\) -prune -o -type f -print0 | xargs -0 shasum > {shasum_file}")

    def read_project_shasums(self, project_description, snapshot_name, allow_missing=False):
        shasum_file = self.get_project_admin_file(project_description, snapshot_name+".sha")
        if allow_missing and not os.path.isfile(shasum_file):
            return None
        with open(shasum_file, 'r') as f:
            output = f.read()
        results = {}
        lines = output.strip().split('\n')
        for line in lines:
            shasum, filepath = line.strip().split(maxsplit=1)
            results[filepath] = shasum
        return results

    def compare_shasums(self, shasums1, shasums2, label=None, root_dir=None):
        new_files = []
        disappeared_files = []
        changed_files = []
        for filepath, shasum in shasums1.items():
            if filepath in shasums2:
                if shasums2[filepath] != shasum:
                    changed_files.append(filepath)
            else:
                disappeared_files.append(filepath)
        for filepath in shasums2.keys():
            if filepath not in shasums1:
                new_files.append(filepath)

        return new_files, disappeared_files, changed_files

    def print_shasums_comparison_result(self, new_files, disappeared_files, changed_files, label=None, root_dir=None, max_num=10):
        if _logger.isEnabledFor(logging.DEBUG):
            if root_dir and root_dir[-1] != "/":
                root_dir += "/"
            def log_list(label, list):
                if list:
                    note = f" ... and {len(list)-max_num} more" if len(list) > max_num else ""
                    _logger.debug(label + ": " + ' '.join([(f.removeprefix(root_dir or "")) for f in list[:max_num]]) + note)
            _logger.debug(f"{label or 'Files'}: {len(new_files)} new, {len(disappeared_files)} disappeared, {len(changed_files)} changed")
            log_list('New files', new_files)
            log_list('Disappeared files', disappeared_files)
            log_list('Changed files', changed_files)

    def show_warnings_before_download(self, project_descriptions, pause_after_warnings=True):
        # the ones that have warnings and are not yet downloaded
        projects_to_warn = [p for p in project_descriptions if self.get_project_status(p) ==  Workspace.ABSENT and p.warnings ]
        if projects_to_warn:
            for p in projects_to_warn:
                for warning in p.warnings:
                    _logger.warning(f"Project {cyan(p)}: {warning}")
            if pause_after_warnings and sys.stdout.isatty() and sys.stdin.isatty():
                input("Press Enter to continue, or Ctrl+C to abort ")

    @staticmethod
    def _get_unique_project_attribute(project_descriptions, attr_name, default_value):
        values = set([getattr(p, attr_name) for p in project_descriptions if getattr(p, attr_name)])
        if not values:
            _logger.debug(f"None of the projects specify the '{attr_name}' attribute, using '{default_value}' as default")
            return default_value
        elif len(values) > 1:
            raise Exception(f"The projects disagree on the choice of '{attr_name}': {values}")
        else:
            return list(values)[0]

    def download_project_if_needed(self, project_description, effective_project_descriptions, patch=True, cleanup=True, **kwargs):
        project_state = self.get_project_status(project_description)
        if project_state == Workspace.ABSENT:
            self.download_project(project_description, effective_project_descriptions, patch, cleanup, **kwargs)
        elif project_state == Workspace.INCOMPLETE:
            raise Exception(f"Cannot download '{project_description}': Directory already exists")
        elif project_state == Workspace.DOWNLOADED:
            self.record_project_shasums(project_description, "last")
            if self.is_project_modified(project_description):
                _logger.warning(f"Project {project_description.get_full_name(colored=True)} has been {yellow('MODIFIED')} since download, use the check_{project_description.name} command to see what changed")
            else:
                _logger.info(f"Project {project_description.get_full_name(colored=True)} is {green('unmodified')} since download")
        else:
            assert False, f"Unknown project state '{project_state}'"

        assert self.get_project_status(project_description) == Workspace.DOWNLOADED, f"Wrong project status {self.get_project_status(project_description)} after download"

    def _read_file_if_exists(self, fname):
        try:
            with open(fname) as f:
                return f.read()
        except:
            return ""

    def download_and_unpack_tarball(self, download_url, target_folder):
        os.makedirs(target_folder)
        tar_log_file = os.path.join(target_folder, "tar.log")
        try:
            print(f"{download_url}")
            self.run_command(f"cd {target_folder} && curl -L --fail --progress-bar {download_url} | tar --strip-components=1 -xzf - 2>{tar_log_file}")
            os.remove(tar_log_file)
        except Exception as e:
            print(self._read_file_if_exists(tar_log_file).strip())
            raise e

    def unpack_tarball(self, tarball_fname, target_folder):
        os.makedirs(target_folder)
        self.run_command(f"cd {target_folder} && tar --strip-components=1 -xzf {tarball_fname}")

    def download_and_apply_patch(self, patch_url, target_folder):
        curl_log_file = os.path.join(target_folder, "curl.log")
        patching_log_file = os.path.join(target_folder, "patch.log")
        try:
            self.run_command(f"cd {target_folder} && curl -L --fail --stderr {curl_log_file} {patch_url} | git apply --whitespace=nowarn - 2>{patching_log_file}")
            os.remove(curl_log_file)
            os.remove(patching_log_file)
        except Exception as e:
            print(self._read_file_if_exists(curl_log_file).strip())
            print(self._read_file_if_exists(patching_log_file).strip())
            raise e

    @staticmethod
    def _get_dependencies(project_description, effective_project_descriptions):
        # find dependencies of the project among effective_projects
        todo = [project_description]
        processed = set()
        deps = []
        while todo:
            project = todo.pop()
            if project not in processed:
                new_deps = [p for p in effective_project_descriptions if p.name in project.required_projects and p not in processed]
                deps += new_deps
                processed.add(project)
                todo.extend(new_deps)
        return deps

    def _define_shell_functions(self, effective_project_descriptions):
        def make_build_function(function_name, directory, build_commands):
            return f"""
                function {function_name} ()
                {{
                    # Collect build modes from args and $BUILD_MODE; if there is one, build it;
                    # if there are several, recursively call this this function for each.
                    local modes="$*"
                    if [ -z "$modes" ]; then modes="$BUILD_MODE"; fi
                    if [ -z "$modes" ]; then modes="release debug"; fi
                    local mode_count=$(echo "$modes" | wc -w)
                    if [ "$mode_count" -gt 1 ]; then
                        echo -e "{SHELL_GREEN}Invoking {function_name} with modes: $modes{SHELL_NOCOLOR}";
                        for i in $modes; do
                            {function_name} $i || return 1
                        done
                        return 0
                    fi
                    local mode=$modes
                    echo -e "{SHELL_GREEN}Invoking {function_name} $mode:{SHELL_NOCOLOR}"
                    if (
                        set -eo pipefail
                        cd '{directory}'
                        BUILD_MODE=$mode
                        {build_commands}
                    ); then
                        echo -e "{SHELL_GREEN}Done {function_name} $mode{SHELL_NOCOLOR}"
                    else
                        echo -e "{SHELL_RED}ERROR in {function_name} $mode{SHELL_NOCOLOR}";
                        return 1;
                    fi
                }}
                export -f {function_name}
            """

        def make_check_function(function_name, project_name, directory):
            return f"""
                function {function_name} ()
                {{
                    (
                    echo -e "{SHELL_GREEN}Invoking {function_name}:{SHELL_NOCOLOR}"
                    echo 'Checking whether files have changed since download...'
                    cd '{directory}'
                    tmp=.opp_env/postdownload.out
                    if shasum --check --quiet .opp_env/postdownload.sha > $tmp 2>/dev/null; then
                        echo OK
                    else
                        cat $tmp | sed 's/FAILED open or read/MISSING/; s/FAILED$/MODIFIED/'
                        echo -e "{SHELL_YELLOW}WARNING:{SHELL_NOCOLOR} {project_name}: $(cat $tmp | wc -l) file(s) changed since download"
                    fi
                    rm $tmp
                    )
                }}
                export -f {function_name}
            """

        def make_function(function_name, commands):
            return f"""function {function_name}() {{
                {join_commands(commands)}
            }}
            export -f {function_name}"""

        project_build_function_commands = [
            make_build_function("build_" + p.name, self.get_project_root_directory(p), join_commands(p.build_commands))
            for p in effective_project_descriptions
        ]

        project_clean_function_commands = [
            make_build_function("clean_" + p.name, self.get_project_root_directory(p), join_commands(p.clean_commands))
            for p in effective_project_descriptions
        ]

        project_smoke_test_function_commands = [
            make_build_function("smoke_test_" + p.name, self.get_project_root_directory(p), join_commands(p.smoke_test_commands if p.smoke_test_commands else [ f"echo -e '{SHELL_YELLOW}SKIPPING:{SHELL_NOCOLOR} No smoke test commands were specified'"]))
            for p in effective_project_descriptions
        ]

        project_test_function_commands = [
            make_build_function("test_" + p.name, self.get_project_root_directory(p), join_commands(p.test_commands if p.test_commands else [ f"echo -e '{SHELL_YELLOW}SKIPPING:{SHELL_NOCOLOR} No test commands were specified'"]))
            for p in effective_project_descriptions
        ]

        project_check_function_commands = [
            make_check_function("check_" + p.name, p.get_full_name(), self.get_project_root_directory(p))
            for p in effective_project_descriptions
        ]

        function_definitions = [
            *project_build_function_commands,
            *project_clean_function_commands,
            *project_smoke_test_function_commands,
            *project_test_function_commands,
            *project_check_function_commands,
            make_function("build_all", [f"build_{p.name} || return 1" for p in reversed(effective_project_descriptions)]),
            make_function("clean_all", [f"clean_{p.name} || return 1" for p in effective_project_descriptions]),
            make_function("smoke_test_all", [f"smoke_test_{p.name}" for p in reversed(effective_project_descriptions)]),
            make_function("test_all", [f"test_{p.name}" for p in reversed(effective_project_descriptions)]),
            make_function("check_all", [f"check_{p.name}" for p in effective_project_descriptions]),
            make_function("opp_env", [ "printf 'error: Cannot run opp_env commands in an opp_env shell -- exit the shell to run it.\n' && return 1" ]),
        ]
        return function_definitions

    def nix_develop(self, effective_project_descriptions, working_directory=None, commands=[], vars_to_keep=None, run_setenv=True, interactive=False, isolated=True, check_exitcode=True, suppress_stdout=False, build_modes=None, tracing=False, **kwargs):

        nixful = not self.nixless

        if nixful:
            nixos = Workspace._get_unique_project_attribute(effective_project_descriptions, "nixos", self.default_nixos)
            stdenv = Workspace._get_unique_project_attribute(effective_project_descriptions, "stdenv", self.default_stdenv)

        session_name = '+'.join([str(d) for d in reversed(effective_project_descriptions)])
        project_shell_hook_commands = sum([p.shell_hook_commands for p in effective_project_descriptions if p.shell_hook_commands], [])
        project_nix_packages = sum([p.nix_packages for p in effective_project_descriptions], [])
        project_vars_to_keep = sum([p.vars_to_keep for p in effective_project_descriptions], [])
        project_setenv_commands = sum([[f"cd '{self.get_project_root_directory(p)}'", *p.setenv_commands] for p in reversed(effective_project_descriptions)], [])
        project_root_environment_variable_assignments = [f"export {p.name.upper()}_ROOT={self.get_project_root_directory(p)}" for p in effective_project_descriptions]
        project_version_environment_variable_assignments = [f"export {p.name.upper()}_VERSION=\"{p.version}\"" for p in effective_project_descriptions]

        # a custom prompt spec to help users distinguish an opp_env shell from a normal terminal session
        prompt = f"\\[\\e[01;33m\\]{session_name}\\[\\e[00m\\]:\[\\e[01;34m\\]\\w\[\\e[00m\\]\\$ "

        is_macos = platform.system().lower() == "darwin"
        nproc_command = "nproc" if not is_macos else "sysctl -n hw.ncpu"

        shell_hook_lines = [
            'function error() { echo "$*" 1>&2; return 1; }; export -f error',
            'function ll() { ls -l $*; }; export -f ll',
            f"export BUILD_MODE=\"{' '.join(build_modes) if build_modes else ''}\"",
            f"export OPP_ENV_VERSION=\"{get_version()}\"",
            *project_root_environment_variable_assignments,
            *project_version_environment_variable_assignments,
            *(project_shell_hook_commands if nixful else []),
            f"export NIX_BUILD_CORES=$({nproc_command})" if self.nixless else None, # otherwise Nix defines it
            f"export PS1='{prompt}'" if interactive and nixful else None,
            *(["pushd . > /dev/null", *project_setenv_commands, "popd > /dev/null"] if run_setenv else []),
            f"cd '{working_directory}'" if working_directory else None,
            *self._define_shell_functions(effective_project_descriptions),
            *commands
        ]

        vars_to_keep = (vars_to_keep or []) + project_vars_to_keep
        script = join_lines(shell_hook_lines)

        if nixful:
            return self._do_nix_develop(nixos=nixos, stdenv=stdenv, nix_packages=project_nix_packages,
                        session_name=session_name, script=script, vars_to_keep=vars_to_keep, interactive=interactive,
                        isolated=isolated, check_exitcode=check_exitcode, suppress_stdout=suppress_stdout, tracing=tracing)
        else:
            if interactive:
                # launch an interactive bash session; setting PROMPT_COMMAND ensures the custom prompt
                # takes effect despite PS1 normally being overwritten by the user's profile and rc files
                script += f"\nPROMPT_COMMAND=\"PS1='{prompt}'\" bash -i"
            return self._do_run_command(script, suppress_stdout=suppress_stdout, check_exitcode=check_exitcode, tracing=tracing)

    def _do_nix_develop(self, nixos, stdenv, nix_packages=[], session_name="", script="", vars_to_keep=None, interactive=False, isolated=True, check_exitcode=True, suppress_stdout=False, tracing=False):
        if not nixos or not stdenv:
            raise Exception(f"The nixos or stdenv field is not defined in any of the effective projects! {nixos=} {stdenv=}")

        nix_develop_flake = """{
        inputs = {
            nixpkgs.url = "nixpkgs/@NIXOS@";
            flake-utils.url = "github:numtide/flake-utils";
        };
        outputs = { self, nixpkgs, flake-utils }:
        flake-utils.lib.eachDefaultSystem(system:
        let
            overlay = final: prev: {
                # allow python2 to be installed despite being EOL and having known vulnerabilities
                python2 = prev.python2.overrideAttrs (oldAttrs: {
                    meta = oldAttrs.meta // { knownVulnerabilities = []; };
                });
            };
            pkgs = import nixpkgs { inherit system; overlays = [ overlay ]; };
            in rec {
                devShells = rec {
                    default = pkgs.@STDENV@.mkDerivation {
                        name = "@SESSION_NAME@";
                        hardeningDisable = [ "all" ];
                        buildInputs = with pkgs; [ @PACKAGES@ bashInteractive ];
                        shellHook = ''
                            set @SHELL_OPTIONS@
                            @SCRIPT@
                        '';
                    };
                };
            });
        }"""

        tools_nix_packages = ["bashInteractive", "gitFull", "openssh", "curl", "gzip", "which", "gnused", "gnutar", "perl", "findutils", "coreutils"]
        nix_packages = uniq(nix_packages + tools_nix_packages)

        shell_options = "-exo pipefail" if tracing else "-eo pipefail"
        flake_dir = os.path.join(self.get_workspace_admin_directory(), nixos)
        os.makedirs(flake_dir, exist_ok=True)
        flake_file_name = os.path.join(flake_dir, "flake.nix")
        with open(flake_file_name, "w") as f:
            nix_develop_flake = (nix_develop_flake
                .replace("@NIXOS@", nixos)
                .replace("@STDENV@", stdenv)
                .replace("@SESSION_NAME@", session_name)
                .replace("@PACKAGES@", " ".join(nix_packages))
                .replace("@SHELL_OPTIONS@", shell_options)
                .replace("@SCRIPT@", script)
            )
            f.write(nix_develop_flake)

        _logger.debug(f"Using nixpkgs {cyan(nixos)} with {cyan(stdenv)}, packages: {cyan(' '.join(sorted(nix_packages)))}")
        _logger.debug(f"Nix flake shellHook script:\n{indent(script)}")
        #_logger.debug(f"Nix flake file {cyan(flake_file_name)}:\n{yellow(nix_develop_flake)}")
        vars_to_keep = (vars_to_keep or []) + ['HOME', 'TERM', 'COLORTERM', 'DISPLAY', 'XAUTHORITY', 'XDG_RUNTIME_DIR', 'XDG_DATA_DIRS', 'XDG_CACHE_HOME', 'QT_AUTO_SCREEN_SCALE_FACTOR']
        isolation_options = ('-i ' + ' '.join(['-k ' + varname for varname in vars_to_keep])) if isolated else ''
        command = '-c bash --norc' if interactive else '-c true'
        nix_develop_command = f"nix --extra-experimental-features nix-command --extra-experimental-features flakes develop {isolation_options} {flake_dir} {command}"

        env = dict(os.environ)

        if isolated:
            # some programs prefer the home directory to exist and be writable
            temp_home = tempfile.mkdtemp()
            env["HOME"] = temp_home

        # This is a workaround for an error message that is printed multiple times when the "shell" command starts e.g. with Ubuntu 22.04 Unity desktop:
        # ERROR: ld.so: object 'libgtk3-nocsd.so.0' from LD_PRELOAD cannot be preloaded (cannot open shared object file): ignored.
        # The reason is that under NIX, the lib's directory is not in the default linker path. Workaround: use full path for lib.
        gtk3_lib_fname = "libgtk3-nocsd.so.0"
        gtk3_lib_dir = "/usr/lib/x86_64-linux-gnu/"
        if "LD_PRELOAD" in env and gtk3_lib_fname in env["LD_PRELOAD"].split(":") and os.path.isfile(gtk3_lib_dir + gtk3_lib_fname):
            def replace_in_list(list, old_value, new_value):
                return [new_value if x == old_value else x for x in list]
            env["LD_PRELOAD"] = ":".join(replace_in_list(env["LD_PRELOAD"].split(":"), gtk3_lib_fname, gtk3_lib_dir + gtk3_lib_fname))

        # This is a workaround for the following warning printed by Perl:
        # perl: warning: Setting locale failed. / Please check that your locale settings: / LANGUAGE = (unset), / LC_ALL = (unset), ... / Falling back to the standard locale ("C").
        env["LC_ALL"] = "C"

        result = self._do_run_command(nix_develop_command, env=env, suppress_stdout=not interactive and suppress_stdout, check_exitcode=check_exitcode)

        # cleanup: remove temporary home dir, as we don't want it to interfere with subsequent sessions
        if isolated:
            shutil.rmtree(temp_home)
        return result

    def run_command(self, command, suppress_stdout=False, check_exitcode=True, tracing=False):
        if not self.nixless:
            return self._do_nix_develop(nixos=self.default_nixos, stdenv=self.default_stdenv, session_name="run_command", script=command,
                        interactive=False, isolated=True, suppress_stdout=suppress_stdout, check_exitcode=check_exitcode, tracing=tracing)
        else:
            return self._do_run_command(command, suppress_stdout=suppress_stdout, check_exitcode=check_exitcode, tracing=tracing)

    def _do_run_command(self, command, env=None, suppress_stdout=False, check_exitcode=True, tracing=False):
        if "\n" not in command:
            _logger.debug(f"Running command: {command}")
        else:
            _logger.debug(f"Running command:\n{indent(command)}")

        # make the script exit on first error, and also on errors in piped commands
        options = "-exo pipefail" if tracing else "-eo pipefail"
        command = f"set {options}; {command}"

        result = subprocess.run(["bash", "-c", command],
                                env=env,
                                stdout=subprocess.DEVNULL if suppress_stdout else sys.stdout,
                                stderr=subprocess.STDOUT if suppress_stdout else sys.stderr)
        _logger.debug(f"Exit code: {result.returncode}")
        if check_exitcode and result.returncode != 0:
            raise Exception(f"Child process exit code {result.returncode}")
        return result

def resolve_projects(project_full_names, remove_trailing_slash=True):
    global project_registry
    project_descriptions = [project_registry.get_project_description(ProjectReference.parse(p.rstrip('/') if remove_trailing_slash else p)) for p in project_full_names]
    return project_descriptions

def init_workspace(workspace_directory, force=False, allow_existing=False, nixless=False):
    workspace_directory = workspace_directory or os.getcwd()
    if os.path.isdir(workspace_directory):
        if not Workspace.is_workspace(workspace_directory):
            if os.listdir(workspace_directory) and not force: # dir not empty
                raise Exception(f"Refusing to turn non-empty directory '{workspace_directory}' into an opp_env workspace -- use 'opp_env init --force' if it was intentional")
    else:
        parent_dir = os.path.dirname(os.path.abspath(workspace_directory))
        if not os.path.isdir(parent_dir):
            raise Exception(f"Cannot create workspace at '{workspace_directory}': refusing to create more than one level of directories")
        os.mkdir(workspace_directory)
    Workspace.init_workspace(workspace_directory, allow_existing=allow_existing, nixless=nixless)
    return workspace_directory

def resolve_workspace(workspace_directory, init, nixless_workspace):
    if init:
        workspace_directory = init_workspace(workspace_directory, allow_existing=True, nixless=nixless_workspace)
        workspace = Workspace(workspace_directory, nixless_workspace)
    else:
        if nixless_workspace:
            raise Exception("--nixless-workspace is only supported with --init")
        workspace_directory = os.path.abspath(workspace_directory) if workspace_directory else Workspace.find_workspace(os.getcwd())
        workspace = Workspace(workspace_directory)
    return workspace

def check_project_dependencies(effective_project_descriptions, workspace, pause_after_warnings=True):
    for project_description in effective_project_descriptions:
        data = workspace.read_project_state_file(project_description)
        last_started_with = data.get("last_started_with", None)
        starting_with = [ p.get_full_name() for p in Workspace._get_dependencies(project_description, effective_project_descriptions) ]
        if last_started_with is not None and starting_with != last_started_with:
            def q(l): return "[" + ", ".join(l) + "]"
            _logger.warning(f"Project {cyan(project_description)} is now being used with a different set of dependencies "
                            f"({cyan(q(starting_with))}) than last time ({cyan(q(last_started_with))}), rebuild recommended. "
                            f"Hint: type " + green(f"clean_{project_description.name} && build_{project_description.name}") + " in the opp_env shell.")
            if pause_after_warnings and sys.stdout.isatty() and sys.stdin.isatty():
                input("Press Enter to continue, or Ctrl+C to abort ")

def update_saved_project_dependencies(effective_project_descriptions, workspace):
    for project_description in effective_project_descriptions:
        starting_with = [ p.get_full_name() for p in Workspace._get_dependencies(project_description, effective_project_descriptions) ]
        workspace.update_project_state(project_description, last_started_with=starting_with)

def list_subcommand_main(project_name_patterns=None, list_mode="grouped", **kwargs):
    global project_registry
    projects = project_registry.get_all_project_descriptions()
    if project_name_patterns:
        tmp = []
        for project_name_pattern in project_name_patterns:
            matching_projects = [p for p in projects if re.match(project_name_pattern, p.get_full_name())] # note: prefix match!
            if not matching_projects:
                raise Exception(f"Name/pattern '{project_name_pattern}' does not match any project")
            tmp += matching_projects
        projects = tmp # NOTE: No sorting! Order of project versions is STRICTLY determined by the order they are in ProjectRegistry.

    names = uniq([p.name for p in projects])

    def move_to_front(list, name):
        try:
            list.remove(name)
            list.insert(0, name)
        except ValueError:
            pass

    names.sort()
    move_to_front(names, "inet")
    move_to_front(names, "omnetpp")

    name_width = len(max(names, key=len))
    if list_mode == "flat":
        for p in projects:
            print(p.get_full_name())
    elif list_mode == "grouped":
        for name in names:
            versions = [p.version for p in projects if p.name == name]
            print(f"{name.ljust(name_width)} {cyan('  '.join(versions))}")
    elif list_mode == "names":
        for name in names:
            print(name)
    elif list_mode == "aliases":
        for project in projects:
            alias_versions = project_registry.get_project_version_aliases(ProjectReference(project.name, project.version))
            for alias_version in natural_sorted(alias_versions):
                print(f"{project.name}-{alias_version} -> {project.get_full_name()}")
    elif list_mode == "descriptions":
        for name in names:
            descriptions = uniq([p.description for p in projects if p.name == name if p.description])
            description = descriptions[0] if descriptions else "(no description)"
            print(f"{name.ljust(name_width)} {cyan(description)}")
    elif list_mode == "expand":
        for project in projects:
            expanded = project_registry.expand_dependencies([project])
            print(' '.join([p.get_full_name() for p in expanded]))
    elif list_mode == "expand-all":
        for project in projects:
            combinations_list = project_registry.expand_dependencies([project], return_all=True)
            for combination in combinations_list:
                print(' '.join([p.get_full_name() for p in combination]))
    else:
        raise Exception(f"invalid list mode '{list_mode}'")

def info_subcommand_main(projects, raw=False, requested_options=None, **kwargs):
    # resolve project list
    global project_registry
    if not projects:
        project_descriptions = project_registry.get_all_project_descriptions()
    else:
        project_descriptions = []
        for project in projects:
            if '-' in project:
                project_descriptions += [project_registry.get_project_description(ProjectReference.parse(project))]
            elif project in project_registry.get_project_names():
                project_descriptions += [project_registry.get_project_description(ProjectReference(project, version)) for version in project_registry.get_project_versions(project)]
            else:
                raise Exception(project_registry.get_unknown_project_message(project))

    if raw:
        serializable = [vars(p.activate_project_options(requested_options)) for p in project_descriptions]
        print(json.dumps(serializable, indent=4))
        return

    # print info for each
    for project_description in project_descriptions:
        project_description = project_description.activate_project_options(requested_options)
        print()
        print(project_description.get_full_name() + (": " + project_description.description if project_description.description else ""))
        if project_description.details:
            print("\n" + project_description.details.rstrip())
        if project_description.warnings:
            for warning in project_description.warnings:
                print(yellow("\nWARNING: ") + warning)
        if (project_description.required_projects):
            print(f"\nRequires:")
            for name, versions in project_description.required_projects.items():
                print(f"- {cyan(name)}: {' / '.join(versions)}")
        if project_description.nixos or project_description.stdenv or project_description.nix_packages:
            print("\nNix:")
            if project_description.nixos:
                print(f"- version:  {cyan(project_description.nixos)}")
            if project_description.stdenv:
                print(f"- stdenv:   {cyan(project_description.stdenv)}")
            if project_description.nix_packages:
                print(f"- packages: {cyan(' '.join(project_description.nix_packages))}")
        if project_description.options:
            print("\nAvailable options:")
            for option_name, option in project_description.options.items():
                option_description = option.get('option_description')
                default_mark = "*" if option.get('is_default') else "";
                if option_description:
                    print(f"- {cyan(option_name)}{default_mark}: {option_description}")
                else:
                    print(f"- {cyan(option_name)}{default_mark}")
        if len(project_descriptions) > 1:
            print("\n--------")
    print()
    print("Note: Specify `--raw` to `opp_env info` for more details.")
    print("Note: Options can be selected by adding `--options <optionname>` to the opp_env command line, see help. Options active by default are marked with '*'.")

def init_subcommand_main(workspace_directory=None, force=False, nixless_workspace=False, **kwargs):
    init_workspace(workspace_directory, force=force, nixless=nixless_workspace)

def install_subcommand_main(projects, workspace_directory=None, install_without_build=False, requested_options=None, no_dependency_resolution=False, nixless_workspace=False, init=False, pause_after_warnings=True, **kwargs):
    global project_registry

    workspace = resolve_workspace(workspace_directory, init, nixless_workspace)

    specified_project_descriptions = resolve_projects(projects)
    if no_dependency_resolution:
        effective_project_descriptions = sort_by_project_dependencies(activate_project_options(specified_project_descriptions, requested_options))
    else:
        effective_project_descriptions = sort_by_project_dependencies(project_registry.compute_effective_project_descriptions(specified_project_descriptions, requested_options))
    _logger.info(f"Using specified projects {cyan(str(specified_project_descriptions))} with effective projects {cyan(str(effective_project_descriptions))} in workspace {cyan(workspace.root_directory)}")

    check_project_dependencies(effective_project_descriptions, workspace, pause_after_warnings)

    workspace.show_warnings_before_download(effective_project_descriptions, pause_after_warnings)

    for project_description in effective_project_descriptions:
        workspace.download_project_if_needed(project_description, effective_project_descriptions, **kwargs)

    update_saved_project_dependencies(effective_project_descriptions, workspace)

    if not install_without_build:
        workspace.nix_develop(effective_project_descriptions, commands=["build_all"])

def is_subdirectory(child_dir, parent_dir):
    # Check if a directory is a subdirectory of another directory.
    return os.path.commonpath([child_dir, parent_dir]) == parent_dir

def check_multiple_versions(project_descriptions):
    name_versions_dict = {}
    for project_description in project_descriptions:
        name, version = project_description.name, project_description.version
        if name in name_versions_dict:
            name_versions_dict[name].append(version)
        else:
            name_versions_dict[name] = [version]
    for name, versions in name_versions_dict.items():
        if len(versions) > 1:
            def q(l): return "[" + ", ".join(l) + "]"
            raise Exception(f"Multiple versions specified for project {cyan(name)}: {cyan(q(versions))} -- only one version of a project may be active at a time")

def shell_subcommand_main(projects, workspace_directory=[], chdir=False, requested_options=None, no_dependency_resolution=False, init=False, install=False, install_without_build=False, build=False, nixless_workspace=False, isolated=True, pause_after_warnings=True, **kwargs):
    global project_registry

    workspace = resolve_workspace(workspace_directory, init, nixless_workspace)

    specified_project_descriptions = resolve_projects(projects) if projects else workspace.get_installed_projects()
    check_multiple_versions(specified_project_descriptions)
    if no_dependency_resolution:
        effective_project_descriptions = sort_by_project_dependencies(activate_project_options(specified_project_descriptions, requested_options))
    else:
        effective_project_descriptions = sort_by_project_dependencies(project_registry.compute_effective_project_descriptions(specified_project_descriptions, requested_options))
    _logger.info(f"Using specified projects {cyan(str(specified_project_descriptions))} with effective projects {cyan(str(effective_project_descriptions))} in workspace {cyan(workspace.root_directory)}")

    if not install:
        for project_description in effective_project_descriptions:
            if workspace.get_project_status(project_description) != Workspace.DOWNLOADED:
                raise Exception(f"Project {cyan(project_description.get_full_name())} is not downloaded, please run {cyan('opp_env install')} first, or use {cyan('opp_env shell --install')}")

    check_project_dependencies(effective_project_descriptions, workspace, pause_after_warnings)

    workspace.show_warnings_before_download(effective_project_descriptions, pause_after_warnings)

    if install:
        for project_description in effective_project_descriptions:
            workspace.download_project_if_needed(project_description, effective_project_descriptions, **kwargs)

    update_saved_project_dependencies(effective_project_descriptions, workspace)

    hint_command = f"echo -e '{SHELL_GREEN}HINT{SHELL_NOCOLOR} To build, clean, test or check a project, use the `build_*`, `clean_*`, `test_*`, `smoke_test_*` and `check_*` commands.'"
    commands = ["build_all", hint_command] if build or (install and not install_without_build) else [hint_command]

    kind = "nixless" if workspace.nixless else "isolated" if isolated else "non-isolated"
    _logger.info(f"Starting {cyan(kind)} shell for projects {cyan(str(effective_project_descriptions))} in workspace {cyan(workspace.root_directory)}")

    if chdir and effective_project_descriptions:
        first_project_description = effective_project_descriptions[0]
        first_project_dir = workspace.get_project_root_directory(first_project_description)
        if chdir == "convenience":
            chdir = not is_subdirectory(os.getcwd(), first_project_dir)  # "is outside the project dir"
        if chdir:
            _logger.debug(f"Changing into the first project's directory {cyan(first_project_dir)}")
            os.chdir(first_project_dir)
        else:
            _logger.debug(f"No need to change directory, wd={cyan(os.getcwd())} is already under the first project's directory {cyan(first_project_dir)}")

    workspace.nix_develop(effective_project_descriptions, commands=commands, interactive=True, isolated=isolated, check_exitcode=False, **kwargs)

def run_subcommand_main(projects, command=None, workspace_directory=None, requested_options=None, no_dependency_resolution=False, init=False, install=False, install_without_build=False, build=False, nixless_workspace=False,  isolated=True, pause_after_warnings=True, run_test=False, run_smoke_test=False, **kwargs):
    global project_registry

    workspace = resolve_workspace(workspace_directory, init, nixless_workspace)

    specified_project_descriptions = resolve_projects(projects) if projects else workspace.get_installed_projects()
    check_multiple_versions(specified_project_descriptions)
    if no_dependency_resolution:
        effective_project_descriptions = sort_by_project_dependencies(activate_project_options(specified_project_descriptions, requested_options))
    else:
        effective_project_descriptions = sort_by_project_dependencies(project_registry.compute_effective_project_descriptions(specified_project_descriptions, requested_options))
    _logger.info(f"Using specified projects {cyan(str(specified_project_descriptions))} with effective projects {cyan(str(effective_project_descriptions))} in workspace {cyan(workspace.root_directory)}")

    if not install:
        for project_description in effective_project_descriptions:
            if workspace.get_project_status(project_description) != Workspace.DOWNLOADED:
                raise Exception(f"Project {cyan(project_description.get_full_name())} is not downloaded, please run {cyan('opp_env install')} first, or use {cyan('opp_env run --install')}")

    check_project_dependencies(effective_project_descriptions, workspace, pause_after_warnings)

    workspace.show_warnings_before_download(effective_project_descriptions, pause_after_warnings)

    if install:
        for project_description in effective_project_descriptions:
            workspace.download_project_if_needed(project_description, effective_project_descriptions, **kwargs)

    update_saved_project_dependencies(effective_project_descriptions, workspace)

    if run_test:
        command = "test_all"

    if run_smoke_test:
        command = "smoke_test_all"

    commands = ["build_all", command] if build or (install and not install_without_build) else [command]

    kind = "nixless" if workspace.nixless else "isolated" if isolated else "non-isolated"
    _logger.info(f"Running {'test ' if run_test else 'smoke_test ' if run_smoke_test else ''}command for projects {cyan(str(effective_project_descriptions))} in workspace {cyan(workspace.root_directory)} in {cyan(kind)} mode")
    workspace.nix_develop(effective_project_descriptions, workspace_directory, commands=commands, **dict(kwargs, suppress_stdout=False))

def maint_subcommand_main(catalog_dir, **kwargs):
    update_catalog(catalog_dir)

def update_catalog(catalog_dir):
    # collect catalog URLs by project name
    _logger.info(f"Collecting catalog_url entries from projects")
    global project_registry
    project_catalog_urls = {}
    for project_description in project_registry.get_all_project_descriptions():
        catalog_url = project_description.metadata.get("catalog_url")
        if catalog_url:
            if project_description.name in project_catalog_urls:
                if project_catalog_urls[project_description.name] != catalog_url:
                    raise Exception(f"The 'metadata/catalog_url' field differs across versions for project '{project_description.name}'")
            else:
                project_catalog_urls[project_description.name] = catalog_url
    _logger.debug(f"Result: {project_catalog_urls=}")

    # insert them into the pages
    _logger.info(f"Updating pages in {cyan(catalog_dir)}")
    for project_name, catalog_url in project_catalog_urls.items():
        # read file
        filename = os.path.join(catalog_dir, os.path.basename(catalog_url).replace(".html", ".md"))
        with open(filename, "r") as f:
            content = f.read()

        # update content
        line = f"opp-env-command: opp_env install {project_name}-latest"
        if "opp-env-command:" in content:
            updated_content = re.sub("^opp-env-command:.*$", line, content, flags=re.MULTILINE)
        else:
            updated_content = re.sub("\n---", f"\n{line}\n---", content)
        assert f"\n{line}\n" in updated_content

        # write back change
        if content != updated_content:
            _logger.debug(f"Writing {filename}")
            with open(filename, "w") as f:
                f.write(updated_content)

def main():
    kwargs = process_arguments()
    subcommand = kwargs.get('subcommand')

    try:
        _logger.debug(f"Starting {cyan(subcommand)} operation")
        if subcommand == None:
            pass # parser.print_help() already called
        elif subcommand == "list":
            list_subcommand_main(**kwargs)
        elif subcommand == "info":
            info_subcommand_main(**kwargs)
        elif subcommand == "init":
            init_subcommand_main(**kwargs)
        elif subcommand == "install":
            install_subcommand_main(**kwargs)
        elif subcommand == "shell":
            shell_subcommand_main(**kwargs)
        elif subcommand == "run":
            run_subcommand_main(**kwargs)
        elif subcommand == "maint":
            maint_subcommand_main(**kwargs)
        else:
            raise Exception(f"Unknown subcommand '{subcommand}'")
        _logger.debug(f"The {cyan(subcommand)} operation completed successfully")
        return 0
    except Exception as e:
        if not kwargs["print_stacktrace"]:
            _logger.error(f"The {cyan(subcommand)} operation stopped with error: {str(e)}")
            return 1
        else:
            raise e
    except KeyboardInterrupt:
        _logger.error(f"The {cyan(subcommand)} operation was interrupted by the user")
        return 130 # = 128 + SIGINT

project_registry = ProjectRegistry()


if __name__ == '__main__':
    sys.exit(main())
