#!/usr/bin/env python3
# SPDX-License-Identifier: LicenseRef-SideFX-EULA
import argparse
import base64
import binascii
import collections
import configparser
import contextlib
import ctypes
import datetime
import functools
import glob
import hashlib
import itertools
import io
import json
import math
import os
import platform
import re
import shutil
import stat
import subprocess
import sys
import textwrap
import time
import types
import urllib
import webbrowser
import zipfile
from email.utils import parsedate_to_datetime
from pathlib import Path

from . import __version__


if sys.version_info < (3, 9):
    raise RuntimeError("Error: Python 3.9+ is required.")

try:
    import requests
except ImportError:
    sys.exit("This program requires Python's requests module. Please run:\n"
        "    pip install requests")

# Note that requests will use simplejson if it's installed, so we need to
# handle errors originating from it.
REQUESTS_JSON_DECODE_ERRORS = (json.JSONDecodeError,)
try:
    import simplejson
    REQUESTS_JSON_DECODE_ERRORS += (simplejson.errors.JSONDecodeError,)
except ImportError:
    pass


CHUNK_SIZE = 65536
MANIFEST_FILE = ".manifest.json"


class Error(Exception):
    def __init__(self, message, word_wrap=True, exit_code=1):
        super().__init__(message)
        self.word_wrap = word_wrap
        self.exit_code = exit_code


DEFAULT_HOOKS = {}
def register_hook(function):
    """This decorator marks a function as something that can be replaced by
    the user of this module.
    """
    DEFAULT_HOOKS[function.__name__] = function
    return function


class App:
    def __init__(self):
        self.verbosity = 2
        self.server = os.environ.get(
            "HOUDINI_PACKAGE_SERVER", "https://www.sidefx.com")
        self.timeout = 30
        self.use_unicode = True

        self.authentication = None
        self.is_in_toplevel_call = False
        self.toplevel_call_succeeded = False
        self.is_running_in_background = False
        self.hooks = types.SimpleNamespace()
        self.hooks.__dict__.update(DEFAULT_HOOKS)

    @contextlib.contextmanager
    def override_hooks(self):
        """Users of the module can write "with app.override_hooks() as hooks
        to temporarily override hooks.
        """
        saved_hooks = self.hooks
        self.hooks = types.SimpleNamespace(**self.hooks.__dict__)
        try:
            yield self.hooks
        finally:
            self.hooks = saved_hooks

    def find_authentication(self, force=False, find_all=False):
        if force:
            self.authentication = None
        self.authentication = _find_authentication(
            self, self.authentication, find_all=find_all)
        return self.authentication

    def require_authentication(self):
        self.find_authentication()
        if self.authentication is None:
            raise Error("You are not logged in. Please run:\n"
                "hpackage auth login")
        return self.authentication


def toplevel_catches_and_prints_errors(function):
    """This decorator marks a function as implementing a command and it catches
    and prints Error exceptions.
    """
    @functools.wraps(function)
    def wrapper(app, *args, **kwargs):
        # If a command calls a function that implements another command, we
        # want it to raise an error and the outer command will catch it.
        if app.is_in_toplevel_call:
            return function(app, *args, **kwargs)

        app.is_in_toplevel_call = True
        app.toplevel_call_succeeded = True
        try:
            return function(app, *args, **kwargs)
        except Error as e:
            app.toplevel_call_succeeded = False
            print_error(app, e.args[0], word_wrap=e.word_wrap)
            return None
        finally:
            app.is_in_toplevel_call = False

    # The original function that always raises Error is still accessible.
    wrapper.with_raise_errors = function
    return wrapper


def yield_means_run_in_background(function):
    """Mark a function as a possible generator that may use a yield statement
    to indicate that the rest of the function should run in the background.
    """
    @functools.wraps(function)
    def wrapper(app, *args, **kwargs):
        generator = function(app, *args, **kwargs)

        # If we're already running in a background thread then don't use the
        # hook so that we run everything in this thread.
        if app.is_running_in_background:
            return run_generator_in_background(generator)

        # Note that it is up to hooks's implementation to set and clear
        # app.is_running_in_background appropriately.
        return app.hooks.run_generator_in_background(generator)
    return wrapper


@register_hook
def run_generator_in_background(generator):
    """Users of the module can override this hook to actually run things in
    the background.
    """
    # If it was a regular function we'll receive its return value instead of a
    # generator.  It will have run entirely in the current thread.
    if not isinstance(generator, types.GeneratorType):
        return generator

    # The command-line implementation runs everything in the current thread.
    while True:
        try:
            next(generator)
        except StopIteration as e:
            return e.value


def create_parser():
    parser = argparse.ArgumentParser(prog="hpackage")
    parser.add_argument(
        "-v", "--verbosity", default="2",
        help="set the verbosity level (2 by default)")
    parser.add_argument(
        "--timeout", help="set the connection timeout in seconds")
    parser.add_argument("--server", help=argparse.SUPPRESS)

    subparsers = parser.add_subparsers(dest="command", required=True)

    # version command
    version_parser = subparsers.add_parser(
        "version", help="print the version of this program")
    version_parser.add_argument(
        "-l", "--show-latest", action="store_true",
        help="show the latest version available")

    # auth subcommand:
    auth_parser = subparsers.add_parser("auth", help="authentication commands")
    auth_subparsers = auth_parser.add_subparsers(dest="auth_cmd", required=True)
    auth_subparsers.add_parser("login", help="log in via a browser")
    auth_subparsers.add_parser("logout", help="log out")
    auth_subparsers.add_parser("status", help="show authentication status")
    auth_subparsers.add_parser("installkey",
        help="install an API key, generating it if necessary")
    uninstallkey_parser = auth_subparsers.add_parser("uninstallkey",
        help="delete the local API key, keeping it on sidefx.com")
    uninstallkey_parser.add_argument(
        "client_id", metavar="CLIENT-ID",
        help="client id of the key to uninstall")
    deletekey_parser = auth_subparsers.add_parser("deletekey",
        help="delete the local API key and delete it from sidefx.com")
    deletekey_parser.add_argument(
        "client_id", metavar="CLIENT-ID",
        help="client id of the key to delete")

    # install subcommand:
    install_parser = subparsers.add_parser("install", help="install packages")
    install_parser.add_argument(
        "-U", "--upgrade",
        help="upgrade the specified package to the latest version")
    install_parser.add_argument(
        "-r", "--requirements", metavar="FILE",
        help="install all packages listed in the given requirements file")
    install_parser.add_argument(
        "--force-reinstall", action="store_true",
        help="force a reinstall of the specified package")
    install_parser.add_argument(
        "package_spec", nargs="?",
        help="package specification to install (e.g. pkg or pkg==1.2.3)")

    # uninstall subcommand:
    uninstall_parser = subparsers.add_parser(
        "uninstall", help="uninstall a package")
    uninstall_parser.add_argument(
        "package",
        help="name of the package to uninstall")
    uninstall_parser.add_argument(
        "-y", "--yes", action="store_true",
        help="answer yes when asked to break dependencies")

    # list subcommand:
    list_parser = subparsers.add_parser("list", help="list installed packages")
    list_parser.add_argument(
        "--show-location", action="store_true",
        help="show the location of each package")
    list_parser.add_argument(
        "--show-dependencies", action="store_true",
        help="show the dependencies of each package")
    list_parser.add_argument(
        "package_name", nargs="*",
        help="optional package names")

    # author subcommand:
    author_parser = subparsers.add_parser(
        "author", help="author a package")
    author_subparsers = author_parser.add_subparsers(
        dest="author_cmd", required=True)
    reserve_parser = author_subparsers.add_parser(
        "reserve", help="reserve a package name and create zip file")
    reserve_parser.add_argument(
        "package", help="a package name")
    reserve_parser.add_argument(
        "-i", "--interactive", action="store_true",
        help="run in interactive mode, prompting for input")
    reserve_parser.add_argument(
        "-I", "--implicit", action="store_true",
        help="use implicit search paths instead of a houdini/ subdirectory")
    unreserve_parser = author_subparsers.add_parser(
        "unreserve", help="undo reserving a package name")
    unreserve_parser.add_argument(
        "package", help="a package name")
    verifyjson_parser = author_subparsers.add_parser(
        "verifyjson", help="verify a package json file")
    verifyjson_parser.add_argument(
        "package", help="a package name or path to a json file")
    upload_parser = author_subparsers.add_parser(
        "upload", help="upload a package")
    upload_parser.add_argument(
        "package",
        help="a package name or path")
    upload_parser.add_argument(
        "--requires-grant", action="store_true",
        help="require access to be granted to download the package")
    upload_parser.add_argument(
        "--auto-publish", action="store_true",
        help="automatically publish after review completes")
    status_parser = author_subparsers.add_parser(
        "status", help="show the review status and local modifications")
    status_parser.add_argument(
        "packages", nargs="*",  help="package names")
    reviewstatus_parser = author_subparsers.add_parser(
        "reviewstate", help="show the review state")
    reviewstatus_parser.add_argument(
        "packages", nargs="*", help="package names")
    grant_parser = author_subparsers.add_parser(
        "grant", help="grant access to a package")
    grant_parser.add_argument(
        "package", help="name of the package for which to grant access")
    grant_parser.add_argument(
        "emails", nargs="+",
        help="one or more email addresses to grant access to")
    publish_parser = author_subparsers.add_parser(
        "publish", help="publish a package to make it accessible")
    publish_parser.add_argument(
        "package", help="a package name or path")
    unpublish_parser = author_subparsers.add_parser(
        "unpublish", help="unpublish a package")
    unpublish_parser.add_argument(
        "package_and_version", help="In the form <name>==<version>")
    delete_parser = author_subparsers.add_parser(
        "delete", help="delete a package")
    delete_parser.add_argument(
        "package_and_version", help="In the form <name>==<version>")
    return parser


def handle_command_line(app):
    parser = create_parser()
    args = parser.parse_args()

    try:
        app.verbosity = int(args.verbosity)
    except ValueError:
        print_error(app, "Invalid verbosity level")
        return 1
    if os.environ.get("HOUDINI_PACKAGE_VERBOSE", "0") != "0":
        app.verbosity = max(app.verbosity, 3)

    if args.server:
        app.server = args.server
        while app.server.endswith("/"):
            app.server = app.server[:-1]
    if args.timeout:
        try:
            app.timeout = int(args.timeout)
        except ValueError:
            print_error(app, "The timeout is not an integer")
            return 1

    # Dispatch to the appropriate handler
    if args.command == "version":
        print_version(app, args.show_latest)
    elif args.command == "auth":
        if args.auth_cmd == "login":
            log_in(app)
        elif args.auth_cmd == "logout":
            logout(app)
        elif args.auth_cmd == "status":
            print_auth_status(app)
        elif args.auth_cmd == "installkey":
            create_and_install_oauth2_keys(app)
        elif args.auth_cmd == "uninstallkey":
            uninstall_oauth2_keys(app, args.client_id)
        elif args.auth_cmd == "deletekey":
            uninstall_and_delete_oauth2_keys(app, args.client_id)
    elif args.command == "install":
        if args.upgrade:
            upgrade(app, args.upgrade)
        elif args.requirements:
            install_from_requirements(app, args.requirements)
        elif args.package_spec:
            install(
                app, args.package_spec, force_reinstall=args.force_reinstall)
        else:
            parser.error("install requires a package specification")
            return 1
    elif args.command == "uninstall":
        uninstall(app, args.package, skip_prompt=args.yes)
    elif args.command == "list":
        list_packages(
            app, args.package_name, show_location=args.show_location,
            show_dependencies=args.show_dependencies)
    elif args.command == "author":
        if args.author_cmd == "reserve":
            reserve_upload(
                app, args.package, not args.implicit, args.interactive)
        elif args.author_cmd == "unreserve":
            unreserve_upload(app, args.package)
        elif args.author_cmd == "verifyjson":
            verify_package_json(app, args.package)
        elif args.author_cmd == "upload":
            upload(app, args.package, requires_grant=args.requires_grant,
                auto_publish=args.auto_publish)
        elif args.author_cmd == "status":
            show_statuses(app, args.packages)
        elif args.author_cmd == "reviewstate":
            show_review_states(app, args.packages)
        elif args.author_cmd == "grant":
            grant(app, args.package, args.emails)
        elif args.author_cmd == "publish":
            publish(app, args.package)
        elif args.author_cmd == "unpublish":
            unpublish(app, args.package_and_version)
        elif args.author_cmd == "delete":
            delete(app, args.package_and_version)

    return (0 if app.toplevel_call_succeeded else 1)


@toplevel_catches_and_prints_errors
def print_version(app, show_latest=False):
    version = (get_latest_hpackage_version(app)
        if show_latest else __version__)
    app.hooks.print_info(f"hpackage {version}")


def get_latest_hpackage_version(app):
    try:
        response = requests.get(
            "https://pypi.org/simple/hpackage/",
            headers={"Accept": "application/vnd.pypi.simple.v1+json"},
            timeout=app.timeout)
    except requests.exceptions.ConnectionError:
        raise Error("Could not connect to pypi.org")
    except requests.exceptions.Timeout:
        raise Error("Timeout connecting to pypi.org")
    return max(response.json()["versions"], key=Version)


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def log_in(app, print_status=True):
    authentication = app.find_authentication()
    if authentication is not None and authentication.session_id:
        print_auth_status(app)
        raise Error("You are already logged in")

    webbrowser.open(f"{app.server}/login/launcher/", new=2)
    waiting_message = "Waiting for you to log in via a browser"
    if app.hooks.print_info is print_info:
        waiting_message += " (press Ctrl+C to quit)"
    app.hooks.print_info(waiting_message)

    yield
    time.sleep(1)
    while True:
        session_id, _csrf_token = _get_session_id_from_hserver(
            app, require_hserver_running=True)
        if session_id:
            break

        if was_interrupted_while_waiting(app, 5):
            return False

    if print_status:
        # Clear the authentication so we re-find it, otherwise we might use
        # OAuth2 keys instead of the session id.
        app.find_authentication(force=True, find_all=True)
        print_auth_status(app)
    return True


@toplevel_catches_and_prints_errors
def logout(app):
    authentication = app.find_authentication(find_all=True)
    if authentication is None:
        print_warning(app, "You are already logged out.")
        return

    if authentication.session_id:
        _call_hserver(app, "cmd_del_session", method="PUT", require_json=False,
            domain=_get_cookie_domain(app))

    if authentication.client_id:
        print_warning(app, "Note that you are still logged in via"
            f" {authentication.method_name()}")
    else:
        app.authentication = None


@toplevel_catches_and_prints_errors
def print_auth_status(app):
    authentication = app.find_authentication(find_all=True)
    if not authentication:
        app.hooks.print_info("You are not logged in")
        return
    for name, label in (
            ("username", "username"),
            ("email", "email"),
            ("company_name", "company")):
        app.hooks.print_info(
            f"{label:<14}: {authentication.validation_output.get(name)}")
    app.hooks.print_info(
        f"{'Login method':<14}: {authentication.method_name()}")
    if authentication.client_id:
        app.hooks.print_info(
            f"{'API client id':<14}: {authentication.client_id}")


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def create_and_install_oauth2_keys(app):
    authentication = app.require_authentication()
    if authentication.client_id:
        raise Error("You are already using an API key")

    application = _post_to_sidefx(
        app, "/oauth2/find-or-create-application/", authentication,
        {"name": "hpackage keys"}).json()

    for path in _find_hserver_ini_paths(os.W_OK):
        with path.open("a") as open_file:
            open_file.write(
                f"ClientID = {application['client_id']}\n"
                f"ClientSecret = {application['client_secret']}\n")
        break
    else:
        raise Error("You do not have permission to write any hserver.ini files")


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def uninstall_oauth2_keys(app, client_id):
    path = None
    client_secret_line_index = None
    for cur_path in _find_hserver_ini_paths(os.W_OK):
        lines = cur_path.read_text().splitlines()
        for i, line in enumerate(lines):
            stripped_line = line.strip()
            if not stripped_line.startswith("ClientID"):
                continue
            parts = [part.strip() for part in stripped_line.split("=")]
            if len(parts) == 2 and parts[-1] == client_id:
                path = cur_path
                client_id_line_index = i
                client_secret_line_index = None
                for j in range(i + 1, len(lines)):
                    stripped_line = lines[j].strip()
                    if (stripped_line.startswith("[") or
                            stripped_line.startswith("ClientID")):
                        break
                    if stripped_line.startswith("ClientSecret"):
                        client_secret_line_index = j
                        break
                break

        if path is not None:
            break

    if path is None:
        raise Error("This client id is not installed")

    with path.open("w") as open_file:
        for i, line in enumerate(lines):
            if i in (client_id_line_index, client_secret_line_index):
                continue
            open_file.write(line + "\n")


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def uninstall_and_delete_oauth2_keys(app, client_id):
    authentication = app.require_authentication()
    _post_to_sidefx(
        app, "/oauth2/delete-application/", authentication,
        {"client_id": client_id})
    return uninstall_oauth2_keys(app, client_id)


def _parse_package_and_version(package_name_and_optional_version):
    parts = package_name_and_optional_version.split("==", 1)
    # We don't return an if-else expression here because of a bug in pylint.
    if len(parts) == 1:
        return parts[0], None
    return parts[0], parts[1]


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def install(
        app, package_name, force_reinstall=False, do_upgrade=False,
        installation_dir=None):
    package_name, version = _parse_package_and_version(package_name)

    # If they're not upgrading and the package is already installed, stick
    # with the current version.
    installed_packages = get_installed_packages(app)
    installed_info = installed_packages.get(package_name)
    if not do_upgrade and installed_info and version is None:
        version = installed_info["version"]

    authentication = app.find_authentication()
    package_name, version, info = _get_package_info_from_name(
        app, package_name, authentication, default_version=version)

    if installation_dir is None and do_upgrade and installed_info:
        installation_dir = Path(installed_info["location"]).parent

    # If the desired version is already installed, do nothing.
    if (installed_info and installed_info["version"] == version and
            not force_reinstall):
        app.hooks.print_info(
            f"{package_name}=={version} is already installed.")
        return

    houdini_major_minor = _houdini_major_minor()
    if Version(houdini_major_minor) < Version(info["min_houdini_major_minor"]):
        raise Error(f"{package_name}=={version} requires at least Houdini"
            f" {info['min_houdini_major_minor']} and is not compatible with"
            f" Houdini {houdini_major_minor}")

    # Check dependencies.
    dependencies_to_install = []
    for dependency in info.get("depends_on", []):
        dep_name = dependency["name"]
        dep_min_version = dependency.get("min_version", None)
        dep_max_version = dependency.get("max_version", None)
        dep_installed_info = installed_packages.get(dep_name)
        if dep_installed_info is None:
            # TODO: What if dep_max_version doesn't exist?  If not, find the
            #       version closest to it.
            if dep_max_version:
                dependencies_to_install.append(
                    f"{dep_name}=={dep_max_version}")
            else:
                dependencies_to_install.append(dep_name)
        else:
            dep_installed_version = dep_installed_info["version"]
            if (dep_max_version and
                    Version(dep_max_version) < Version(dep_installed_version)):
                raise Error(
                    f"{package_name}=={version} requires"
                    f" {dep_name}<={dep_max_version}"
                    f" but {dep_installed_version} is installed")
            if (dep_min_version and
                    Version(dep_installed_version) < Version(dep_min_version)):
                raise Error(
                    f"{package_name}=={version} requires"
                    f" {dep_name}>={dep_min_version}"
                    f" but {dep_installed_version} is installed")

    # See if this package requires a grant.  If it does and the user does not
    # have access we'll exit with an error.
    if not info["requires_grant"]:
        download_url = (
            f"{app.server}/hpackage/download/{package_name}/{version}.zip")
    else:
        if authentication is None:
            authentication = app.require_authentication()
        download_url = _call_sidefx(app, "hpack.get_download_url_with_grant",
            authentication, authentication.email(), package_name,
            version=version)

    # Now that we've done all the checks, we can start taking actions.  Install
    # dependencies as necessary.
    for dependency_to_install in dependencies_to_install:
        if app.verbosity > 0:
            app.hooks.print_info(f"Installing {dependency_to_install}")
        install(app, dependency_to_install, installation_dir=installation_dir)

    # Uninstall if we're upgrading/forcing reinstallation.
    if installed_info is not None:
        _uninstall_ignoring_dependencies(app, package_name)

    if installation_dir is None:
        installation_dir = Path(houdini_user_pref_dir()) / "packages"
    installation_dir.mkdir(parents=True, exist_ok=True)

    zip_path = installation_dir / f"{package_name}-{version}.zip"
    yield
    _download_file(app, authentication, download_url, zip_path,
        f"{package_name}=={version}", info["file_size"],
        info["md5_checksum"], print_progress=app.verbosity > 0)
    _extract_zip(zip_path, installation_dir)
    zip_path.unlink()

    # Write a manifest file to detect local modifications to the package.
    _write_manifest(installation_dir / f"{package_name}.json")


def _get_package_info_from_name(
        app, package_name, authentication, default_version=None):
    # Note that this call will validate that the package version exists.
    return _call_sidefx(
        app, "hpack.get_info_to_install_package",
        authentication, package_name, default_version)


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def install_from_requirements(app, requirements_path):
    path = Path(requirements_path)
    if not path.is_file():
        raise Error(f"{requirements_path} is not a valid file")

    package_names = [line.strip() for line in path.read_text().splitlines()
        if line.strip() and not line.strip().startswith("#")]
    yield
    for package_name in package_names:
        install(app, package_name)


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def upgrade(app, package_name):
    specified_version = "==" in package_name

    # Check if the package is installed and it's not already at the
    # specified/latest version.
    authentication = app.find_authentication()
    package_name, version, _info = _get_package_info_from_name(
        app, package_name, authentication)
    installed_info = get_installed_packages(app).get(package_name)
    if installed_info and installed_info.get("version") == version:
        message = f"{package_name}=={version} is already installed"
        if not specified_version:
            message += " and is the latest version"
        app.hooks.print_info(message)
        return

    # If it's already installed, install the new version to the same location.
    installation_dir = None
    if installed_info:
        installation_dir = Path(installed_info["location"]).parent

    return install(app, f"{package_name}=={version}", do_upgrade=True,
        installation_dir=installation_dir)


@toplevel_catches_and_prints_errors
def uninstall(app, package_name, skip_prompt=False):
    # Note that we auto-add dependencies when installing but do not
    # remove them when uninstalling.  This matches pip's behaviour and
    # is intentional.
    package_name, version = _parse_package_and_version(package_name)

    installed_packages = get_installed_packages(app)
    installed_info = installed_packages.get(package_name)
    if installed_info is None:
        raise Error(f"{package_name} is not installed")

    if version is not None and installed_info["version"] != version:
        raise Error(
            f"{package_name}=={installed_info['version']} is installed,"
            f" not version {version}")

    dependencies = _calculate_dependencies(app, installed_packages)
    needed_by = dependencies[package_name]["needed_by"]
    if needed_by:
        print_warning(app, f"{package_name} is needed by:" + "".join(
            "\n    " + other_package_name
            for other_package_name in sorted(needed_by)))
        if not skip_prompt:
            if not app.hooks.prompt_yes_or_no(
                    "Are you sure you want to continue", default_to_yes=False):
                return

    _uninstall_ignoring_dependencies(app, package_name, installed_info)


def _calculate_dependencies(app, installed_packages=None):
    if installed_packages is None:
        installed_packages = get_installed_packages(app)

    # Calculate direct dependencies.  If they have local/edited package
    # requirements on things that don't exist, skip them.
    direct_dependencies = {
        package_name: set() for package_name in installed_packages}
    for package_name, installed_info in installed_packages.items():
        for requirement in installed_info["requires"]:
            dependency_name = requirement["name"]
            if dependency_name in installed_packages:
                direct_dependencies[package_name].add(dependency_name)
            else:
                print_warning(app,
                    f"{package_name} depends on {dependency_name} but it is"
                    " not installed")

    # Now calculate indirect dependencies.  First use Kahn's algorithm to sort
    # topologically so that we process parents (what has the dependency)
    # before children (the thing it depends on).
    indegrees = {package_name: 0 for package_name in installed_packages}
    for dependencies in direct_dependencies.values():
        for dependency_name in dependencies:
            indegrees[dependency_name] += 1

    # Walk down from toplevel dependencies to build the topography.
    queue = collections.deque([
        package_name for package_name, degree in indegrees.items()
        if degree == 0])
    topo = []
    while queue:
        package_name = queue.popleft()
        topo.append(package_name)
        for dependency_name in direct_dependencies[package_name]:
            indegrees[dependency_name] -= 1
            if indegrees[dependency_name] == 0:
                queue.append(dependency_name)

    if len(topo) != len(installed_packages):
        raise Error("Cycle detected in package dependencies.  Please check"
            " 'requires' in the packages you are authoring and check if you've"
            " made local modifications to other packages.")

    # Compute the transitive closures in reverse topological order.
    depends_on = {package_name: set() for package_name in installed_packages}
    for package_name in reversed(topo):
        closure = depends_on[package_name]
        for dependency_name in direct_dependencies[package_name]:
            closure.add(dependency_name)
            closure |= depends_on[dependency_name]

    # Now calculate reverse dependencies, including both direct and indirect.
    needed_by = {package_name: set() for package_name in installed_packages}
    for package_name, dependencies in depends_on.items():
        for dependency_name in dependencies:
            needed_by[dependency_name].add(package_name)

    return {
        package_name: {
            "directly_depends_on": direct_dependencies[package_name],
            "depends_on": depends_on[package_name],
            "needed_by": needed_by[package_name],
        } for package_name in installed_packages}


def _uninstall_ignoring_dependencies(app, package_name, installed_info=None):
    if installed_info is None:
        installed_info = get_installed_packages(app).get(package_name)

    installed_info["location"].with_suffix(".json").unlink()
    shutil.rmtree(installed_info["location"])


@toplevel_catches_and_prints_errors
def list_packages(app, package_names, show_location, show_dependencies):
    package_names = (set(package_names) if package_names else None)

    installed_packages = get_installed_packages(app)
    if show_dependencies:
        dependencies = _calculate_dependencies(app, installed_packages)

    max_len = max(
        (len(package_name) + 2 + len(info["version"])
        for package_name, info in installed_packages.items()), default=0)
    for package_name, info in sorted(installed_packages.items()):
        if package_names and package_name not in package_names:
            continue
        line = f"{package_name}=={info['version']}"
        if show_location:
            line += " " * (max_len + 2 - len(line))
            line += str(info["location"])
        app.hooks.print_info(line)
        if show_dependencies:
            for dependency_name in dependencies[
                    package_name]["directly_depends_on"]:
                app.hooks.print_info(f"    -{dependency_name}")


@toplevel_catches_and_prints_errors
def reserve_upload(
        app, package_name, use_explicit, interactive=False, print_output=True):
    authentication = app.require_authentication()

    user_pref_dir = houdini_user_pref_dir()
    if user_pref_dir is None:
        raise Error("No installed Houdini version could be detected, is it"
            " installed?")

    packages_dir = Path(user_pref_dir) / "packages"
    packages_dir.mkdir(parents=True, exist_ok=True)
    package_dir = packages_dir / package_name
    package_json = package_dir.with_suffix(".json")

    if package_json.exists() or package_dir.exists():
        print_warning(app,
            "You already have a package with this name so no example"
            " package will be created.  Move aside the existing package to"
            " create an example.")
        _validate_package_json(app, package_json)
        return None

    if interactive:
        readme_markdown = input(
            "Enter a short one-line readme description:\n") + "\n"

    zip_contents = _call_sidefx(app, "hpack.reserve_package_name",
        authentication, package_name, _houdini_major_minor(), use_explicit)
    with zipfile.ZipFile(io.BytesIO(zip_contents), "r") as zip_file:
        zip_file.extractall(packages_dir)
    if print_output:
        app.hooks.print_info(f"Created {package_dir}")

    getting_started_path = package_dir / "getting-started.txt"
    if interactive:
        (package_dir / "README.md").write_text(readme_markdown)
        app.hooks.print_info(getting_started_path.read_text())
        if app.hooks.prompt_yes_or_no(
                "Delete getting-started.txt?", default_to_yes=True):
            getting_started_path.unlink()
    else:
        if print_output:
            app.hooks.print_info(
                f"\nSee {getting_started_path} for instructions.")
    return package_dir


@toplevel_catches_and_prints_errors
def unreserve_upload(app, package_name):
    authentication = app.require_authentication()
    _call_sidefx(
        app, "hpack.unreserve_package_name", authentication, package_name)


@toplevel_catches_and_prints_errors
def verify_package_json(app, package_name_or_path):
    json_path = _find_package_json_path(app, package_name_or_path)
    if app.verbosity > 0:
        app.hooks.print_info(f"{json_path} is valid")


@toplevel_catches_and_prints_errors
@yield_means_run_in_background
def upload(app, package_name_or_path, requires_grant, auto_publish):
    authentication = app.require_authentication()

    json_path = _find_package_json_path(app, package_name_or_path)
    package_dir = json_path.with_suffix("")
    package_name = package_dir.stem

    _, package_info = _read_package_info_from_json(app, json_path)
    version = package_info["version"]

    getting_started_path = package_dir / "getting-started.txt"
    if os.path.exists(getting_started_path):
        raise Error(
            f"Please perform the actions in {getting_started_path} and then"
            " delete this file.")

    readme_md_path = _find_readme_md_path(json_path)
    if "<enter a one-line description" in readme_md_path.read_text():
        raise Error(f"Please edit {readme_md_path} and add a description.")

    yield

    # Note that the server side will verify there are no symlinks.
    zip_path = _create_zipfile(json_path)
    try:
        file_size = zip_path.stat().st_size
        upload_info = _call_sidefx(app, "hpack.start_upload", authentication,
            json.loads(json_path.read_text()), readme_md_path.read_text(),
            file_size, _compute_md5_checksum(zip_path),
            requires_grant=requires_grant, auto_publish=auto_publish)

        # Resume a partially complete upload if possible.
        uploaded = upload_info["partial_upload_size"]
        if (uploaded and _compute_md5_checksum(zip_path, uploaded) ==
                upload_info["partial_upload_md5_checksum"]):
            index = uploaded
        else:
            index = 0

        with zip_path.open("rb") as open_file:
            open_file.seek(index)
            while True:
                data = open_file.read(CHUNK_SIZE * 10)
                if not data:
                    break
                # TODO: Handle rate limiting here.
                result = _call_sidefx(app, "hpack.upload_chunk",
                    authentication, upload_info["token"], index,
                    data=bytearray(data))
                index += len(data)

                assert result["uploaded"] == index
                assert result["expecting"] == file_size

                if app.verbosity > 0:
                    app.hooks.set_progress_fraction(
                        app, index / file_size,
                        suffix=f"uploading {package_name}")
        if app.verbosity > 0:
            app.hooks.erase_progress_bar(app)
            app.hooks.print_info(
                f"Successfully uploaded {package_name}=={version}")

        # Create/update the manifest file after a successful upload.
        _write_manifest(json_path)
    finally:
        zip_path.unlink()

    app.hooks.print_info(get_package_page_url(app, package_name))


def get_package_page_url(app, package_name):
    return f"{app.server}/hpackage/package/{package_name}"


def _compute_md5_checksum(file_path, only_first_bytes=None):
    infinity = 1 << 63    # effectively infinite
    hasher = hashlib.md5()
    with open(file_path, "rb") as open_file:
        remain = only_first_bytes if only_first_bytes is not None else infinity
        while remain > 0:
            b = open_file.read(min(CHUNK_SIZE, remain))
            if not b:
                break
            hasher.update(b)
            remain -= len(b)
    return hasher.hexdigest()


def _find_readme_md_path(json_path):
    readme_md_path = json_path.with_suffix("") / "README.md"
    if not readme_md_path.is_file():
        raise Error(f"You need to create {readme_md_path} and put a package"
            " description there.")
    return readme_md_path


def _create_zipfile(json_path):
    package_dir = json_path.with_suffix("")
    parent_dir = json_path.parent

    package_name = json_path.stem
    zip_path = parent_dir / (package_name + ".zip")

    with zipfile.ZipFile(zip_path, "w") as zip_file:
        zip_file.write(json_path, arcname=json_path.relative_to(parent_dir))

        for subpath in dict.fromkeys(itertools.chain(
                package_dir.rglob("*"), package_dir.rglob(".*"))):
            # Determine the path in the archive file, and skip the manifest.
            archive_path = Path(subpath).relative_to(parent_dir)
            if str(archive_path) == f"{package_name}/{MANIFEST_FILE}":
                continue

            if subpath.is_dir():
                _mkdir_in_zipfile(zip_file, archive_path)
            elif subpath.is_file():
                zip_file.write(subpath, arcname=archive_path)

    return zip_path


def _mkdir_in_zipfile(zip_file, dir_path):
    # Note that zipfile.mkdir was not added until Python 3.11.
    dir_path = str(dir_path)
    if not dir_path.endswith("/"):
        dir_path += "/"
    dir_info = zipfile.ZipInfo(dir_path)
    dir_info.date_time = time.localtime()[:6]
    dir_info.external_attr = (stat.S_IFDIR | 0o755) << 16
    zip_file.writestr(dir_info, "")


def _find_package_json_path(app, package_name_or_path):
    # TODO: Can we ever get a path here on Windows containing backslashes?
    #       Convert any backslashes to forward slashes to be safe.
    if package_name_or_path.endswith(".json"):
        json_path = package_name_or_path
    elif "/" in package_name_or_path or os.path.exists(
            package_name_or_path + ".json"):
        json_path = package_name_or_path + ".json"
    else:
        # Treat the path as a package name.
        json_path = None
        user_pref_dir = houdini_user_pref_dir()
        if user_pref_dir:
            json_path = (Path(user_pref_dir) / "packages" /
                (package_name_or_path + ".json"))
            if not json_path.is_file():
                json_path = None

        if json_path is None:
            raise Error(
                f"{package_name_or_path} does not have a package json file")

    if not _validate_package_json(app, json_path):
        raise Error("The package json is not valid")

    return Path(json_path)


def _validate_package_json(app, json_path):
    json_path = Path(json_path)
    if not json_path.exists():
        print_error(app, f"{json_path} does not exist")
        return False
    if not json_path.is_file():
        print_error(app, f"{json_path} is not a file")
        return False

    try:
        contents = json_path.read_text()
    except (PermissionError, OSError):
        print_error(app, f"{json_path} could not be read")
        return False

    try:
        j = json.loads(contents)
    except json.JSONDecodeError as e:
        print_error(app, f"{json_path} does not contain valid JSON data\n{e}")
        return False

    if not isinstance(j, dict):
        print_error(app, f"{json_path} does not contain a JSON object")
        return False

    if "env" not in j:
        print_error(app, f'{json_path} does not contain an "env" entry')
        return False

    if not isinstance(j["env"], (list, dict)):
        print_error(
            app, f'In {json_path}, "env" should be an array or an object')
        return False

    if "hpackage" not in j:
        print_error(app, f'{json_path} does not contain an "hpackage" entry')
        return False

    if not isinstance(j["hpackage"], dict):
        print_error(app, f'In {json_path}, "hpackage" should be a JSON object')
        return False

    if "version" not in j["hpackage"]:
        print_error(app,
            f'In {json_path}, "hpackage" does not contain a "version" entry')
        return False

    version = j["hpackage"]["version"]
    if not isinstance(version, str):
        print_error(app, f'In {json_path}, "version" should be a string')
        return False

    try:
        Version(version)
    except ValueError:
        print_error(
            app, f'In {json_path}, "{version}" is not a valid version number')
        return False

    orig_version = version
    version = str(Version(version))
    if orig_version != version:
        print_error(
            app, f'In {json_path}, shorten "{orig_version}" to "{version}"')
        return False

    hpath = j.get("hpath")
    if not isinstance(hpath, str):
        print_error(
            app, f'{json_path} does not contain an "hpath" single string entry')
        return False

    if hpath.endswith("/"):
        print_error(app, f'In {json_path}, "hpath" cannot end with a "/"')
        return False

    package_name = Path(json_path).stem
    base_hpath = f"$HOUDINI_PACKAGE_PATH/{package_name}"
    if hpath not in (base_hpath, base_hpath + "/houdini"):
        print_error(app,
            f'In {json_path}, "hpath" must be either {base_hpath} or'
            f' {base_hpath}/houdini')
        return False

    if isinstance(j["env"], dict):
        env_dict = j["env"]
    else:
        env_dict = {}
        for sub_dict in j["env"]:
            env_dict.update(sub_dict)

    is_valid = True
    for d, name, expected in (
                (j, "name", package_name),
                (env_dict, f"PKG_{package_name.upper()}",
                    f"$HOUDINI_PACKAGE_PATH/{package_name}"),
                (j, "load_package_once", True),
                (j, "show", True),
            ):
        actual = d.get(name)
        if actual != expected:
            print_error(app,
                f'In {json_path}, "{name}" should be "{expected}",'
                f' not {actual}')
            is_valid = False

    # Make sure they specify a minimum Houdini version.
    minimum_houdini_major_minor = (
        _extract_houdini_major_minor_from_enable_value(
            app, json_path, j.get("enable", {})))
    if minimum_houdini_major_minor is None:
        print_error(app,
            f'In {json_path}, there is not an "enable" entry defining a'
            ' strictly minimum Houdini version such as:\n'
            '"houdini_version >= \'21.0\'"')
        is_valid = False

    requires = j.get("requires") or []
    if requires:
        if not isinstance(requires, list):
            print_error(app, f'In {json_path}, "requires" is not an array')
            is_valid = False
        else:
            for requires_name in requires:
                if not isinstance(requires_name, str):
                    print_error(app,
                        f'In {json_path}, invalid element {requires_name!r} in'
                        ' "requires"')
                    is_valid = False

    for key in ("min_versions", "max_versions"):
        versions = j["hpackage"].get(key)
        if versions is None:
            continue
        if not isinstance(versions, dict):
            print_error(app, f'In {json_path}, "{key}" is not a JSON object')
            is_valid = False
            continue
        for name, version_str in j["hpackage"][key].items():
            if not isinstance(name, str) or not isinstance(version_str, str):
                print_error(
                    app, f'In {json_path}, "{key}" entries must be strings')
                is_valid = False
            elif name not in requires:
                print_error(app,
                    f'In {json_path}, "{name}" is not a requirement but it is'
                    f' listed in "{key}"')
                is_valid = False
            elif not Version.is_valid(version_str):
                print_error(app,
                    f'In {json_path}, "{version_str}" is not a valid version')
                is_valid = False

    package_dir = json_path.with_suffix("")
    if not package_dir.is_dir():
        print_error(app, f"{package_dir} is not a directory")
        is_valid = False

    return is_valid


def _extract_houdini_major_minor_from_enable_value(
        app, json_path, enable_value):
    # They must provide a strict minimum version, which can be specified in
    # one of the following ways:
    # - "houdini_version >= '20.0'"
    # - {"houdini_version >= '20.0'": true}
    # - {"houdini_version <= '20.0'": false}
    assert enable_value is not None
    enable_dict = ({enable_value: True}
        if isinstance(enable_value, str) else enable_value)

    if not isinstance(enable_dict, dict):
        print_warning(app,
            f'In {json_path}, invalid type for "enable"')
        return None

    for condition, enable in enable_dict.items():
        if enable not in (True, False):
            print_warning(app,
                f'In {json_path}, "enable" items must be either true or false')
            continue

        match = re.match(
            r"^\s*houdini_version\s*(<=|>=)\s*'(.+?)'\s*$", condition)
        if not match:
            continue

        comparison, version = match.groups()
        version_str = version.strip()

        try:
            version = Version(version_str)
        except ValueError:
            is_valid_version = False
        else:
            is_valid_version = len(version.parts) == 2

        if not is_valid_version:
            print_warning(app, f'In {json_path}, the Houdini version'
                f' {version_str} is not valid.')
            continue

        if (comparison == ">=") == enable:
            return version_str

    return None


@toplevel_catches_and_prints_errors
def show_statuses(app, package_names, print_output=True):
    installed_packages = get_installed_packages(app)
    if package_names:
        for package_name in package_names:
            if "==" in package_name:
                raise Error(
                    "You cannot specify a specific package version, use\n"
                    f"hpackage author reviewstate {package_name}")
    else:
        package_names = list(sorted(installed_packages))

    app.require_authentication()
    package_name_to_state = get_package_name_to_state(
        app, installed_packages, package_names)

    # If no manifest file exists then this package hasn't been uploaded.
    # A package with that name may or may not exist on the server.
    result = []
    for package_name, state in package_name_to_state.items():
        if print_output:
            app.hooks.print_info(f"{state:14} {package_name}")

        installed_info = installed_packages.get(package_name)
        if installed_info is None:
            continue

        manifest_path = installed_info["location"] / MANIFEST_FILE

        # Show any local modifications.
        if manifest_path.is_file():
            package_dir = manifest_path.parent.parent
            saved_manifest = json.loads(manifest_path.read_text())
            actual_manifest = _compute_manifest(
                package_dir / f"{package_name}.json", skip_checksums=True)
            for (subpath, status), is_last in _iter_with_is_last(
                    _diff_manifests(
                        saved_manifest, actual_manifest, package_dir)):
                if print_output:
                    title = {
                        "modified": "modified:",
                        "deleted": " deleted:",
                        "new": "     new:",
                    }[status]
                    if app.use_unicode:
                        prefix = ("  └─" if is_last else "  ├─")
                    else:
                        prefix = ("  +-" if is_last else "  |-")
                    app.hooks.print_info(f"{prefix} {title} {subpath}")
                result.append([status, subpath])

    return result


def get_package_name_to_state(app, installed_packages, package_names=None):
    if package_names is None:
        package_names = list(sorted(installed_packages))

    package_name_to_state = {
        package_name: None for package_name in package_names}

    if app.authentication:
        package_names_with_versions = [
            f"{package_name}=={installed_packages[package_name]['version']}"
            for package_name, state in package_name_to_state.items()
            if state is None and package_name in installed_packages]
        if package_names_with_versions:
            package_name_to_state.update(_call_sidefx(
                app, "hpack.get_review_states", app.authentication,
                package_names_with_versions))

    for package_name, state in list(package_name_to_state.items()):
        if state is not None:
            continue
        installed_info = installed_packages.get(package_name)
        if installed_info is None:
            package_name_to_state[package_name] = "not installed"
            continue

        manifest_path = installed_info["location"] / MANIFEST_FILE
        package_name_to_state[package_name] = (
            "not uploaded" if not manifest_path.is_file() else "installed")

    return package_name_to_state


@toplevel_catches_and_prints_errors
def show_review_states(app, package_names):
    authentication = app.require_authentication()
    installed_packages = get_installed_packages(app)

    package_name_to_version = {}
    for package_name in package_names:
        package_name, version = _parse_package_and_version(package_name)
        if version is None:
            installed_info = installed_packages.get(package_name)
            if installed_info is None:
                raise Error(
                    f"You did not specify a version for {package_name} and it"
                    " is not installed.\nPlease specify a version.")
            version = installed_info["version"]
        package_name_to_version[package_name] = version


    package_name_to_state = _call_sidefx(
        app, "hpack.get_review_states", authentication,
        [f"{package_name}=={version}"
            for package_name, version in package_name_to_version.items()])
    for package_name, state in package_name_to_state.items():
        version = package_name_to_version[package_name]
        app.hooks.print_info(f"{state:14} {package_name}=={version}")


@toplevel_catches_and_prints_errors
def grant(app, package_name, emails):
    authentication = app.require_authentication()
    _call_sidefx(app, "hpack.grant_access",
        authentication, package_name, emails)


@toplevel_catches_and_prints_errors
def publish(app, package_and_version):
    authentication = app.require_authentication()
    package_name, version, _info = _get_package_info_from_name(
        app, package_and_version, authentication)

    _call_sidefx(app, "hpack.publish_package_version",
        authentication, package_name, version)


@toplevel_catches_and_prints_errors
def unpublish(app, package_and_version):
    _unpublish_or_delete(app, package_and_version, "unpublish")


@toplevel_catches_and_prints_errors
def delete(app, package_and_version):
    _unpublish_or_delete(app, package_and_version, "delete")


def _unpublish_or_delete(app, package_and_version, operation):
    authentication = app.require_authentication()
    package_name, version = _parse_package_and_version(package_and_version)
    if version is None:
        raise Error("You must pass <package_name>==<version>")

    _call_sidefx(app, f"hpack.{operation}_package_version",
        authentication, package_name, version)


# Code related to package json files:
# ----------------------------------------------------------------------------

def get_installed_packages(app):
    result = {}
    for package_dir in _package_path_dirs():
        package_dir = Path(package_dir)
        if not package_dir.is_dir():
            continue

        for json_path in package_dir.glob("*.json"):
            package_name, package_info = _read_package_info_from_json(
                app, json_path)
            if package_name:
                assert package_info
                result[package_name] = package_info

    return result


def _read_package_info_from_json(app, json_path):
    try:
        with json_path.open() as open_file:
            try:
                package_json = json.load(open_file)
            except json.JSONDecodeError:
                print_warning(app,
                    f"{json_path} does not contain valid JSON")
                return None, None
    except (OSError, PermissionError):
        print_warning(app,
            f"{json_path} is not readable")
        return None, None

    package_name = package_json.get("name")
    if not package_name:
        return None, None

    hpackage_dict = package_json.get("hpackage", {})
    if not isinstance(hpackage_dict, dict):
        return None, None
    version = hpackage_dict.get("version")
    if not version:
        return None, None

    # Build a dict mapping from required package name to a version range dict.
    requires_dict = {
        requires_name: {"min_version": "", "max_version": ""}
        for requires_name in package_json.get("requires", [])}
    for list_key, key, operation in (
            ("min_versions", "min_version", "minimum"),
            ("max_versions", "max_version", "maximum")):
        for requires_name, version in package_json["hpackage"].get(
                list_key, {}).items():
            if requires_name in requires_dict:
                requires_dict[requires_name][key] = version
            else:
                print_warning(app,
                    f"'{requires_name}' is not listed as a requirement but has"
                    f" a {operation} version")

    # Now build a list of dicts containing each requirement's name, min
    # version, and max version.
    requires_list = [{
        "name": requires_name,
        "min_version": version_dict["min_version"],
        "max_version": version_dict["max_version"],
    } for requires_name, version_dict in requires_dict.items()]

    # Trim unneeded trailing zeros from the version.
    return package_name, {
        "version": str(Version(version)),
        "location": json_path.with_suffix(""),
        "requires": requires_list,
    }


# Code related to manifest files.
# ----------------------------------------------------------------------------

def _write_manifest(json_path):
    """Writes the manifest file into the package directory."""
    package_dir = json_path.with_suffix("")
    (package_dir / MANIFEST_FILE).write_text(
        _manifest_to_str(_compute_manifest(json_path)))


def _walk_with_scandir(dir_path):
    """Produces a sequence of (os.DirEntry, is_empty_dir) for files and
    empty directories, recursively.  It is similar to recursive results
    of os.scandir.

    Directories with contents inside them are not returned in the result.
    """
    # This approach is faster than stat'ing each file separately, since it
    # does one stat per directory.
    for dir_entry in os.scandir(dir_path):
        if not dir_entry.is_dir():
            yield dir_entry, False
        else:
            found_something = False
            for sub_result in _walk_with_scandir(dir_entry.path):
                found_something = True
                yield sub_result

            if not found_something:
                yield dir_entry, True


def _compute_manifest(json_path, skip_checksums=False):
    """Computes the contents of the manifest and returns it as a dict."""
    # Add the json file itself to the manifest.
    stat_info = json_path.stat()
    files = {json_path.name: [
        int(stat_info.st_mtime), stat_info.st_size,
        _crc32_checksum_file(json_path)]}
    empty_dirs = []

    package_dir = json_path.with_suffix("")
    package_name = package_dir.stem
    for dir_entry, is_empty_dir in _walk_with_scandir(package_dir):
        sub_path = str(Path(dir_entry).relative_to(package_dir.parent))
        if sub_path == f"{package_name}/{MANIFEST_FILE}":
            continue
        if is_empty_dir:
            empty_dirs.append(sub_path)
        else:
            stat_info = dir_entry.stat()
            crc32_checksum = (_crc32_checksum_file(dir_entry.path)
                if not skip_checksums else None)
            files[sub_path] = [
                int(stat_info.st_mtime), stat_info.st_size, crc32_checksum]

    return {"files": files, "empty_dirs": empty_dirs}


def _manifest_to_str(manifest):
    # We use custom json whitespace format to keep it human readable while
    # still being compact.
    parts = ["{\n"]
    for (toplevel_key, toplevel_value), is_last_toplevel in _iter_with_is_last(
            manifest.items()):
        parts.append(json.dumps(toplevel_key))
        parts.append(": ")
        if isinstance(toplevel_value, dict):
            parts.append("{\n")
            for (key, value), is_last in _iter_with_is_last(
                    toplevel_value.items()):
                parts.append("    ")
                parts.append(json.dumps(key))
                parts.append(": ")
                parts.append(json.dumps(value))
                if not is_last:
                    parts.append(",")
                parts.append("\n")
            parts.append("}")
        elif isinstance(toplevel_value, list):
            parts.append("[\n")
            for value, is_last in _iter_with_is_last(toplevel_value):
                parts.append("    ")
                parts.append(json.dumps(value))
                if not is_last:
                    parts.append(",")
                parts.append("\n")
            parts.append("]")
        else:
            parts.append(json.dumps(toplevel_value))

        if not is_last_toplevel:
            parts.append(",")
        parts.append("\n")
    parts.append("}\n")

    return "".join(parts)


def _iter_with_is_last(sequence):
    """Generates (elem, is_last) for each item in the sequence."""
    # Handle the case where the sequence is empty.
    iterator = iter(sequence)
    try:
        current_item = next(iterator)
    except StopIteration:
        return

    for next_item in iterator:
        yield current_item, False
        current_item = next_item
    yield current_item, True


def _diff_manifests(
        old_manifest, new_manifest, new_package_dir, compare_checksums=True):
    old_files = set(old_manifest["files"])
    new_files = set(new_manifest["files"])
    missing_files = old_files - new_files
    created_files = new_files - old_files

    result = []
    for existing_file in old_files.intersection(new_files):
        old_mod_time, old_size, old_crc32 = old_manifest["files"][existing_file]
        new_mod_time, new_size, new_crc32 = new_manifest["files"][existing_file]

        # Manifests use CRC32 checksums in case we ever need to create a
        # manifest from the zip file, since it stores these checksums.  Zip
        # file mode times are sometimes off by one second when creating them,
        # and it's not due to rounding.  We work around this ziplib problem by
        # allowing the timestamps to differ slightly.
        changed = abs(old_mod_time - new_mod_time) > 1 or old_size != new_size
        if changed and compare_checksums:
            assert old_crc32 is not None
            if new_crc32 is None:
                new_crc32 = _crc32_checksum_file(
                    Path(new_package_dir) / existing_file)
            if old_crc32 == new_crc32:
                changed = False
        if changed:
            result.append((existing_file, "modified"))

    missing_empty_dirs = (
        set(old_manifest["empty_dirs"]) - set(new_manifest["empty_dirs"]))
    created_empty_dirs = (
        set(new_manifest["empty_dirs"]) - set(old_manifest["empty_dirs"]))

    _remove_dirs_from_set_having_children(
        missing_empty_dirs, created_files | created_empty_dirs)
    _remove_dirs_from_set_having_children(
        created_empty_dirs, missing_files | missing_empty_dirs)

    for created_file in created_files:
        result.append((created_file, "new"))
    for created_empty_dir in created_empty_dirs:
        result.append((created_empty_dir, "new"))
    for missing_empty_dir in missing_empty_dirs:
        result.append((missing_empty_dir, "deleted"))
    for missing_file in missing_files:
        result.append((missing_file, "deleted"))

    result.sort()
    return result


def _remove_dirs_from_set_having_children(dir_paths, child_paths):
    for dir_path in dir_paths.copy():
        for child_path in child_paths:
            if child_path.startswith(dir_path + "/"):
                dir_paths.remove(dir_path)
                break


# Code related to Houdini paths:
# ----------------------------------------------------------------------------

@functools.cache
def _package_path_dirs():
    return [os.path.join(path, "packages")
        for path in _houdini_path_user_dirs()]


@functools.cache
def _houdini_path_user_dirs():
    default_dirs = _default_houdini_path_user_dirs()
    houdini_path = os.environ.get("HOUDINI_PATH")
    if houdini_path:
        houdini_path = houdini_path.split(os.pathsep)
    else:
        houdini_path = ["&"]

    result = []
    for path in houdini_path:
        if path != "&":
            result.append(path)
        else:
            result.extend(default_dirs)

    # Make sure the result does not contain duplicates.
    return list(dict.fromkeys(result))


@functools.cache
def _default_houdini_path_user_dirs():
    """Return [$HOUDINI_USER_PREF_DIR, $HSITE/houdiniX.Y].
    These are the directories where the user can install packages to.
    """
    result = [houdini_user_pref_dir()]
    hsite_houdini_dir = _hsite_houdini_dir()
    if hsite_houdini_dir:
        result.append(hsite_houdini_dir)
    return result


@functools.cache
def _hsite_houdini_dir():
    hsite = os.environ.get("HSITE")
    if not hsite:
        return None

    major_minor = _houdini_major_minor()
    hsite_houdini = os.path.join(hsite, f"houdini{major_minor}")
    if not os.path.isdir(hsite_houdini):
        return None
    return hsite_houdini


@functools.cache
def houdini_user_pref_dir():
    result = os.environ.get("HOUDINI_USER_PREF_DIR")
    if result:
        return result

    major_minor = _houdini_major_minor()
    system = _get_platform_system()

    # Note that we can't use %USERPROFILE% on Windows and instead need to look
    # in the user's Documents directory. This location can be configured by the
    # user and is typically in %USERPROFILE%/Documents or
    # %USERPROFILE%/OneDrive/Documents.
    if system in ("Windows", "Cygwin"):
        return f"{_get_windows_documents_path()}/houdini{major_minor}"
    return os.path.expandvars({
        "Linux": f"$HOME/houdini{major_minor}",
        "Darwin": f"$HOME/Library/Preferences/houdini/{major_minor}",
    }[system])


def _get_platform_system():
    system = platform.system()
    return ("Cygwin" if system.lower().startswith("cygwin") else system)


def _get_windows_documents_path():
    from ctypes import wintypes

    SHGetKnownFolderPath = ctypes.windll.shell32.SHGetKnownFolderPath
    SHGetKnownFolderPath.argtypes = [
        ctypes.POINTER(wintypes.GUID), # rfid
        wintypes.DWORD, # dwFlags
        wintypes.HANDLE, # hToken
        ctypes.POINTER(ctypes.c_wchar_p), # ppszPath
    ]
    SHGetKnownFolderPath.restype = wintypes.HRESULT

    FOLDERID_Documents = wintypes.GUID(
        "{FDD39AD0-238F-46AF-ADB4-6C85480369C7}")

    path_ptr = ctypes.c_wchar_p()
    hresult = SHGetKnownFolderPath(
        ctypes.byref(FOLDERID_Documents),
        0, # flags
        None, # user token
        ctypes.byref(path_ptr))

    if hresult != 0:
        raise OSError(
            "An error occurred while determining the Documents directory:"
            "SHGetKnownFolderPath failed for Documents, "
            f"HRESULT=0x{hresult:08X}")

    result = path_ptr.value
    ctypes.windll.ole32.CoTaskMemFree(path_ptr)
    return result


@functools.cache
def _houdini_major_minor():
    major = os.environ.get("HOUDINI_MAJOR_RELEASE")
    minor = os.environ.get("HOUDINI_MINOR_RELEASE")
    if major and minor:
        return f"{major}.{minor}"

    hfs = _find_hfs()
    if hfs:
        return _read_major_minor_from_hfs(hfs)

    raise Error("The current Houdini version could not be determined")


def _read_major_minor_from_hfs(hfs):
    houdini_setup = Path(hfs) / "houdini_setup_bash"
    if not houdini_setup.is_file():
        return None

    minor, major = None, None
    for line in houdini_setup.read_text().split("\n"):
        match = re.search(r"export HOUDINI_MAJOR_RELEASE=(\d+)", line)
        if match:
            major = match.group(1)
        match = re.search(r"export HOUDINI_MINOR_RELEASE=(\d+)", line)
        if match:
            minor = match.group(1)
        if major and minor:
            return f"{major}.{minor}"

    return None


@functools.cache
def _find_hfs():
    hfs_dirs = []
    hfs = os.environ.get("HFS")
    if hfs:
        hfs_dirs.append(hfs)

    platform_system = _get_platform_system()
    if platform_system == "Linux":
        hfs_dirs.extend(sorted(glob.glob("/opt/hfs*.*.*"), reverse=True))
    elif platform_system == "Darwin":
        hfs_dirs.extend(sorted(
            glob.glob("/Applications/Houdini/Houdini*.*.*"), reverse=True))
    elif platform_system in ("Windows", "Cygwin"):
        hfs_dirs.extend(get_hfs_dirs_from_registry())

    for hfs in hfs_dirs:
        path = Path(hfs)
        if path.is_dir() and (path / "houdini_setup_bash").is_file():
            return str(hfs)
    return None


def get_hfs_dirs_from_registry():
    import winreg

    try:
        reg_key = winreg.OpenKey(
            winreg.HKEY_LOCAL_MACHINE,
            r"SOFTWARE\Side Effects Software\Houdini", 0,
            winreg.KEY_WOW64_64KEY | winreg.KEY_READ)
    except FileNotFoundError:
        return []

    try:
        # Note that we skip the "(Default)" value.
        version_to_hfs_dir = {}
        for index in itertools.count(0):
            try:
                name, val, _vtype = winreg.EnumValue(reg_key, index)
            except OSError:
                break
            if name and name[0].isdigit():
                version_to_hfs_dir[name] = val
    finally:
        winreg.CloseKey(reg_key)

    return [version_to_hfs_dir[version]
        for version in sorted(version_to_hfs_dir, reverse=True)
        if os.path.isdir(version_to_hfs_dir[version])]


# Code related to authentication and OAuth2 keys for sidefx.com:
# ----------------------------------------------------------------------------

class Authentication:
    def __init__(
            self, app, session_id=None, csrf_token=None,
            client_id=None, client_secret=None):
        self.app = app
        self.session_id = session_id
        self.csrf_token = csrf_token
        self.client_id = client_id
        self.client_secret = client_secret
        self.access_token = None
        self.access_token_expiry_time = None
        self.validation_output = None

        # Only a session id or an OAuth2 client can be given, but not both.
        assert (self.session_id is None) != (self.client_id is None)

        # If a session id is given a csrf token must also be given.
        assert (self.session_id is None) == (self.csrf_token is None)

        # If a client id is given a client secret must also be given.
        assert (self.client_id is None) == (self.client_secret is None)

    def method_name(self):
        if self.session_id and self.client_id:
            return "Session and API Key"
        return ("Session" if self.session_id else "API Key")

    def validate(self):
        if not self.session_id and not self.access_token:
            assert self.client_id
            if not self.redeem_access_token():
                return False

        response = _post_to_sidefx(
            self.app, "/validate/", self, {}, allowed_statuses=(200, 403, 429))
        if response.status_code == 403:
            return False

        output = response.json()
        if output.get("error"):
            print_warning(self.app,
                f"Error calling /validate/: {output.get('error')}")
            return False

        self.validation_output = output
        return True

    def redeem_access_token(self):
        assert self.client_id

        try:
            response = requests.post(
                f"{self.app.server}/oauth2/application_token",
                headers={"Authorization": "Basic " + _b64encode_text(
                    f"{self.client_id}:{self.client_secret}")},
                timeout=self.app.timeout)
        except requests.exceptions.ConnectionError:
            raise Error(f"Could not connect to {self.app.server}")
        except requests.exceptions.Timeout:
            raise Error(f"Timeout connecting to {self.app.server}")

        if response.status_code == 403:
            return False
        if response.status_code != 200:
            _raise_error_from_web_response(response)

        response_json = response.json()
        self.access_token = response_json["access_token"]
        self.access_token_expiry_time = time.time() - 2 + response_json[
            "expires_in"]
        return True

    def email(self):
        assert self.validation_output
        return self.validation_output["email"]

    def merge_in_missing_credentials(self, authentication):
        if not self.session_id and authentication.session_id:
            self.session_id = authentication.session_id
            self.csrf_token = authentication.csrf_token

        if not self.client_id and authentication.client_id:
            self.client_id = authentication.client_id
            self.client_secret = authentication.client_secret
            self.access_token = authentication.access_token
            self.access_token_expiry_time = (
                authentication.access_token_expiry_time)


def _find_authentication(app, authentication=None, find_all=False):
    if authentication is None:
        authentication = _find_authentication_by_session(app)
        if authentication is None:
            authentication = _find_authentication_by_keys(app)

    if not find_all or authentication is None:
        return authentication

    if not authentication.session_id:
        authentication_by_session = _find_authentication_by_session(app)
        if authentication_by_session:
            authentication.merge_in_missing_credentials(
                authentication_by_session)
            return authentication

    if not authentication.client_id:
        authentication_by_keys = _find_authentication_by_keys(app)
        if authentication_by_keys:
            authentication.merge_in_missing_credentials(authentication_by_keys)
            return authentication

    return authentication


def _find_authentication_by_session(app):
    session_id, csrf_token = _get_session_id_from_hserver(app)
    if not session_id:
        return None

    authentication = Authentication(
        app, session_id=session_id, csrf_token=csrf_token)
    if not authentication.validate():
        print_warning(app,
            f"Invalid session id found {session_id}", verbosity=2)
        return None

    return authentication


def _find_authentication_by_keys(app):
    for client_id, client_secret in _find_oauth_keys(app):
        authentication = Authentication(
            app, client_id=client_id, client_secret=client_secret)
        if authentication.validate():
            return authentication

        print_warning(app,
            f"Invalid API key found with client id of {client_id}",
            verbosity=2)

    return None


def _find_oauth_keys(app):
    keys = []

    api_key_file = os.environ.get("HOUDINI_API_KEY_FILE")
    if api_key_file:
        api_key_file = Path(api_key_file)
        if api_key_file.is_file():
            keys.extend(_read_api_keys_from_api_key_file(api_key_file))
        else:
            print_warning(app,
                f"HOUDINI_API_KEY_FILE refers to {str(api_key_file)}"
                " but it does not exist.")

    for path in _find_hserver_ini_paths(os.R_OK):
        keys.extend(_read_api_keys_from_hserver_ini(path))
    return keys


def _find_hserver_ini_paths(mode):
    platform_system = _get_platform_system()
    if platform_system == "Windows":
        paths = ["%SystemDrive%/ProgramData/SideFX",
            "%USERPROFILE%/AppData/Roaming/SideFX"]
    elif platform_system == "Cygwin":
        paths = ["$SystemDrive/ProgramData/SideFX",
            "$USERPROFILE/AppData/Roaming/SideFX"]
    elif platform_system == "Darwin":
        paths = ["/Library/Preferences/sesi/hserver",
            "$HOME/Library/Application Support/sidefx"]
    elif platform_system == "Linux":
        paths = ["/usr/lib/sesi/hserver",
            "$HOME/.local/share/sidefx"]
    else:
        raise Error(f"Unsupported platform: {platform_system}")

    for path in paths:
        path = Path(os.path.expandvars(path)) / "hserver.ini"
        if _can_access_file(path, mode):
            yield path


def _read_api_keys_from_hserver_ini(path):
    # Note that Houdini supports multiple ini file keys, particularly APIKey
    # and APIKeyFile.
    ini = MultiValueConfigParser()
    ini.read(path)

    keys = []
    for section in ini.values():
        client_ids = _get_section_values(section, "ClientID")
        client_secrets = _get_section_values(section, "ClientSecret")
        keys.extend(zip(client_ids, client_secrets))

        for api_key in _get_section_values(section, "APIKey"):
            keys.extend(_read_keys_from_api_key_line(api_key))

        for api_key_file in _get_section_values(section, "APIKeyFile"):
            keys.extend(_read_api_keys_from_api_key_file(api_key_file.strip()))

    return keys


def _get_section_values(section, key):
    """Given a MultiValueConfigParser section, return a key's values."""
    # If there is only one value it won't be a list, so convert it to a list.
    values = section.get(key, [])
    if isinstance(values, str):
        values = [values]
    return values


def _read_api_keys_from_api_key_file(api_key_file):
    keys = []
    if _can_read_file(api_key_file):
        for line in Path(api_key_file).read_text().splitlines():
            keys.extend(_read_keys_from_api_key_line(line))
    return keys


def _read_keys_from_api_key_line(line):
    # line can be one of
    #     <client_id> <client_secret>
    #     <server> <client_id> <client_secret>
    #     <server1> <client_id1> <client_secret1>; <server2> <id2> <secret2>
    keys = []
    for chunk in line.strip().split(";"):
        key_parts = chunk.strip().split()
        if len(key_parts) == 3:
            server, client_id, client_secret = key_parts
            if not server.endswith(".sidefx.com"):
                continue
        elif len(key_parts) == 2:
            client_id, client_secret = key_parts
        else:
            continue
        keys.append((client_id, client_secret))
    return keys


# Code related to ini file parsing:
# ----------------------------------------------------------------------------

class MultiValueConfigParser(configparser.RawConfigParser):
    def __init__(self):
        super().__init__(strict=False, dict_type=MultiValueDict)

    def _join_multiline_values(self):
        # No-op: prevents collapsing lists into "\n".join(...) strings, to
        # support multiple values per name.
        pass

    def _read(self, fp, fpname):
        # Support .ini files that do not have any section headers.
        try:
            return super()._read(fp, fpname)
        except configparser.MissingSectionHeaderError:
            fp.seek(0)
            string_file = io.StringIO("[DEFAULT]\n" + fp.read())
            return super()._read(string_file, fpname)


class MultiValueDict(dict):
    """Accumulate repeated keys into lists of values, for use with the config
    parser.
    """
    def __setitem__(self, key, value):
        if key in self:
            existing = super().__getitem__(key)

            # Both existing and value from _read are lists of strings
            if isinstance(existing, list) and isinstance(value, list):
                existing.extend(value)
                return
            if isinstance(existing, list):
                existing.append(value)
                return
            super().__setitem__(key, [existing] + (
                value if isinstance(value, list) else [value]))
        else:
            super().__setitem__(key, value)


# Code related to calling hserver and sidefx.com:
# ----------------------------------------------------------------------------

def _get_session_id_from_hserver(app, require_hserver_running=False):
    # Note that hserver only supports pull_cookies in version 20.5.220 and up.
    try:
        session_id_result = _call_hserver(app, "pull_cookies", "sessionid",
            require_hserver_running=require_hserver_running)
    except RuntimeError as e:
        if "unknown api request" not in str(e).lower():
            raise
        raise Error("hserver version 20.5.220 or higher is required")

    # If hserver isn't running then the result will be None.
    if session_id_result is None:
        return None, None

    csrf_token_result = _call_hserver(app, "pull_cookies", "csrftoken",
        require_hserver_running=require_hserver_running)

    cookies = (session_id_result.get("cookies", []) +
        csrf_token_result.get("cookies", []))

    session_id, csrf_token = _get_cookie_values(
        app, cookies, ["sessionid", "csrftoken"])
    if session_id and not csrf_token:
        session_id = None
        print_warning(app,
            "A session id was found without a CSRF token so it"
            " cannot be used.")
    return session_id, csrf_token


def _get_cookie_values(app, cookie_strs, cookie_names):
    cookie_domain = _get_cookie_domain(app)
    num_cookies_left = len(cookie_names)
    cookie_values = [None] * num_cookies_left
    cookie_name_to_index = {cookie_name: i
        for i, cookie_name in enumerate(cookie_names)}

    for cookie_str in cookie_strs:
        cookie_str_parts = cookie_str.split(";")
        if "=" not in cookie_str_parts[0]:
            continue
        cookie_name, cookie_value = cookie_str_parts[0].strip().split("=", 1)
        cookie_index = cookie_name_to_index.get(cookie_name)
        if cookie_index is None:
            continue

        cookie_parts_dict = {}
        for part_name_and_value in cookie_str_parts[1:]:
            if "=" not in part_name_and_value:
                continue
            part_name, value = part_name_and_value.strip().split("=", 1)
            cookie_parts_dict[part_name] = value

        if "domain" not in cookie_parts_dict:
            print_warning(app, f"Cookie is missing a domain:\n{cookie_str}")
            continue

        if cookie_parts_dict["domain"] == cookie_domain:
            is_overwriting = cookie_values[cookie_index] is not None
            cookie_values[cookie_index] = cookie_value
            if not is_overwriting:
                num_cookies_left -= 1
                if num_cookies_left == 0:
                    break

    return cookie_values


def _get_cookie_domain(app):
    # Note that cookies don't include the port number in the domain.
    return urllib.parse.urlparse(app.server).hostname


def _call_hserver(
        app, function, *args, method="POST", require_json=True,
        require_hserver_running=True, **kwargs):
    started_hserver = False
    while True:
        try:
            response = requests.request(method, "http://127.0.0.1:1714/api",
                data={"json": json.dumps([function, args, kwargs])},
                headers={"Accept": "application/json"},
                timeout=app.timeout)
            break
        except requests.exceptions.ConnectionError:
            if not require_hserver_running:
                print_warning(app, "hserver is not running")
                return None

            if started_hserver:
                raise Error("Attempting to start hserver failed")

            if not _start_hserver():
                raise Error("hserver is not running and could not be started")

            app.hooks.print_info("hserver was started")
            started_hserver = True
        except requests.exceptions.Timeout:
            message = "Timeout connecting to hserver"
            if require_hserver_running:
                raise Error(message)
            print_error(app, "Timeout connecting to hserver")
            return None

    if response.status_code == 429:
        raise Error("Rate limit exceeded connecting to hserver")

    if response.status_code != 200:
        raise RuntimeError(response.content.decode())

    try:
        return response.json()
    except REQUESTS_JSON_DECODE_ERRORS:
        if require_json:
            raise RuntimeError(response.content.decode())
        return response.content.decode()


def _start_hserver():
    hfs = _find_hfs()
    if not hfs:
        return False

    extension = (".exe"if _get_platform_system() in ("Windows", "Cygwin")
        else "")
    result = subprocess.run([f"{hfs}/bin/hserver{extension}"], check=False)
    if result.returncode:
        return False

    time.sleep(1)
    return True


def _post_to_sidefx(
        app, path, authentication, post_data, *, file_data=None,
        allowed_statuses=(200, 429)):
    assert path.startswith("/")
    assert isinstance(authentication, (Authentication, type(None)))
    if file_data is None:
        file_data = {}
    headers = {"Referer": app.server}
    cookies = {}

    if authentication:
        if authentication.session_id:
            post_data = post_data.copy()
            post_data["sessionid"] = authentication.session_id
            post_data["csrftoken"] = authentication.csrf_token
            cookies["sessionid"] = authentication.session_id
            cookies["csrftoken"] = authentication.csrf_token
            headers["X-CSRFToken"] = authentication.csrf_token
        elif authentication.access_token:
            headers["Authorization"] = "Bearer " + authentication.access_token

    try:
        response = requests.post(f"{app.server}{path}",
            headers=headers, cookies=cookies, data=post_data, files=file_data,
            timeout=app.timeout)
    except requests.exceptions.ConnectionError:
        raise Error(f"Could not connect to {app.server}")
    except requests.exceptions.Timeout:
        raise Error(f"Timeout connecting to {app.server}")

    if response.status_code == 429:
        _handle_rate_limit_response(response)
    if response.status_code not in allowed_statuses:
        _raise_error_from_web_response(response)
    return response


def _raise_error_from_web_response(response):
    content = response.content.decode()
    if response.status_code != 200:
        if content.strip().startswith("<!DOCTYPE html"):
            content = _extract_text_from_html(content)
        content = f"{response.status_code}\n{content}"
    raise RuntimeError(content)


def _call_sidefx(app, function, authentication, *args, **kwargs):
    file_data = {}
    for arg_name in list(kwargs):
        arg_value = kwargs[arg_name]
        if isinstance(arg_value, (bytes, bytearray)):
            file_data[arg_name] = (
                "unnamed.bin", io.BytesIO(arg_value),
                "application/octet-stream")
            kwargs.pop(arg_name)

    kwargs["client_version"] = __version__

    response = _post_to_sidefx(
        app, "/api/", authentication,
        {"json": json.dumps([function, args, kwargs])},
        file_data=file_data, allowed_statuses=(200, 422, 429))
    if response.status_code == 429:
        _handle_rate_limit_response(response)
    if response.status_code == 422:
        raise Error(response.content.decode())
    if response.headers.get("Content-Type") == "application/octet-stream":
        return response.content
    return response.json()


def _handle_rate_limit_response(_response):
    raise Error(
        "Rate limit exceeded, please wait a couple minutes and try again.")


def _extract_text_from_html(html):
    try:
        from bs4 import BeautifulSoup
    except ImportError:
        return html

    soup = BeautifulSoup(html, "html.parser")
    for tag in soup(["script", "style"]):
        tag.extract()

    text = soup.get_text(separator="\n")
    lines = [line.strip() for line in text.splitlines()]
    chunks = [phrase for line in lines
        for phrase in line.split("  ") if phrase]
    return "\n".join(chunks)


# Code related to version parsing.
# ----------------------------------------------------------------------------

@functools.total_ordering
class Version:
    """Support comparison between N(.N)* version strings like 1, 1.2, 1.2.3."""
    MAX_DIGITS_IN_PARTS = [4, 4, 4, 4, 3]

    def __init__(self, version_str):
        # Note that packaging.version.Version is close to what we want but it's
        # not part of the standard library so we can't easily use it on the
        # Houdini side.
        match = re.match(r"^(\d+(?:\.\d+)*)$", version_str)
        if not match:
            raise ValueError(f"Invalid version: {version_str!r}")

        # Note that we leave trailing ".0"'s.
        parts = []
        for segment in match.group(1).split("."):
            if len(segment) > 1 and segment.startswith("0"):
                raise ValueError(
                    "Version segments cannot have leading zeros:"
                    f" {version_str!r}")
            parts.append(int(segment))

        # Permit versions like 1.0 but trim any later zeros, such as in 1.2.0.
        while len(parts) > 2 and parts[-1] == 0:
            parts.pop()

        if len(parts) > len(self.MAX_DIGITS_IN_PARTS):
            raise ValueError(
                f"Too many segments in version: {version_str!r}")

        for part, max_digits in zip(parts, self.MAX_DIGITS_IN_PARTS):
            if part >= 10 ** max_digits:
                raise ValueError(
                    f"In version: {version_str!r}, the segment {part} is too"
                    " large")

        self.parts = tuple(parts)

    @classmethod
    def is_valid(cls, version_str):
        try:
            Version(version_str)
        except ValueError:
            return False
        return True

    def as_int(self):
        parts = self.parts + (0,) * (
            len(self.MAX_DIGITS_IN_PARTS) - len(self.parts))
        result_strs = []
        for part, max_digits in zip(parts, self.MAX_DIGITS_IN_PARTS):
            result_strs.append(f"{part:0>{max_digits}}")
        return int("".join(result_strs))

    def trimmed_parts(self):
        parts = list(self.parts)
        while len(parts) > 1 and parts[-1] == 0:
            parts.pop()
        return tuple(parts)

    def __eq__(self, other):
        assert isinstance(other, Version)
        return self.trimmed_parts() == other.trimmed_parts()

    def __lt__(self, other):
        assert isinstance(other, Version)
        return self.trimmed_parts() < other.trimmed_parts()

    def __str__(self) -> str:
        return ".".join(str(n) for n in self.parts)

    def __repr__(self) -> str:
        return f"Version({str(self)!r})"


# Miscellaneous helper functions:
# ----------------------------------------------------------------------------

@register_hook
def check_if_was_cancelled():
    return False


def _download_file(
        app, authentication, url, path, package_name_and_version, file_size,
        md5_checksum, print_progress=True):
    assert file_size > 0

    # If a partial file exists from an interrupted download, resume from where
    # we left off.
    headers = {"Referer": app.server}
    cookies = {}
    if path.exists():
        size_downloaded = path.stat().st_size
        headers["Range"] = f"bytes={size_downloaded}-"
    else:
        size_downloaded = 0

    if authentication:
        if authentication.session_id:
            cookies["sessionid"] = authentication.session_id
        elif authentication.access_token:
            headers["Authorization"] = "Bearer " + authentication.access_token

    # Open a request for the file, retrying if we get rate limited.
    max_retries = 5
    write_mode = "ab"
    session = requests.Session()
    for retries in range(0, max_retries + 1):
        try:
            response = session.get(
                url, headers=headers, cookies=cookies, stream=True,
                timeout=app.timeout)
        except requests.exceptions.ConnectionError:
            raise Error(f"Could not connect to {app.server}")
        except requests.exceptions.Timeout:
            raise Error(f"Timeout connecting to {app.server}")
        if app.hooks.check_if_was_cancelled():
            raise Error("Download was cancelled")

        # The server will return http 206 for partial downloads.
        if response.status_code in (200, 206):
            if response.status_code == 200 and size_downloaded > 0:
                write_mode = "wb"
                size_downloaded = 0
            break

        if response.status_code != 429 or retries >= max_retries:
            raise Error(f"Error downloading {url}\n"
                f"{response.status_code} {response.text}", word_wrap=False)

        # Handle Retry-After headers with either number of seconds or a
        # specific date.
        retry_after = response.headers.get("Retry-After")
        if retry_after is None:
            retry_after = "30"
        try:
            wait_in_s = int(retry_after)
        except (TypeError, ValueError):
            dt = parsedate_to_datetime(retry_after)
            now = datetime.datetime.now(datetime.timezone.utc)
            wait_in_s = int(math.ceil((dt - now).total_seconds()))

        wait_in_s = max(wait_in_s, 1)
        wait_in_s = min(wait_in_s, 120)
        if print_progress:
            app.hooks.set_progress_fraction(
                app, 0.0, suffix=f"(rate limited) {package_name_and_version}")

        if was_interrupted_while_waiting(app, wait_in_s):
            raise Error("Download was cancelled")

    # Download the file in chunks, updating the progress bar.
    with path.open(write_mode) as open_file:
        for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
            if app.hooks.check_if_was_cancelled():
                raise Error("Download was cancelled")

            if not chunk:
                continue
            open_file.write(chunk)
            size_downloaded += len(chunk)
            if print_progress:
                app.hooks.set_progress_fraction(
                    app, size_downloaded / file_size,
                    suffix=package_name_and_version)
    parent_with_slashes = str(path.parent).replace("\\", "/")
    app.hooks.erase_progress_bar(app)
    app.hooks.print_info(
        f"Successfully installed {package_name_and_version} to"
        f" {parent_with_slashes}")

    # Verify that we got the right final file size and checksum.
    actual_file_size = path.stat().st_size
    actual_md5_checksum = _md5_checksum_file(path)
    if actual_file_size != file_size or actual_md5_checksum != md5_checksum:
        path.unlink()
        raise Error(
            "The data that was downloaded does not match what was expected.\n"
            f"File size: expected {file_size}, got {actual_file_size}\n"
            f"Checksum: expected {md5_checksum}, got {actual_md5_checksum}")


def was_interrupted_while_waiting(app, num_seconds):
    for i in range(num_seconds):
        time.sleep(1)
        if app.hooks.check_if_was_cancelled():
            return True
    return False


@register_hook
def set_progress_fraction(app, fraction, prefix="", suffix=""):
    bar_width = max(
        10, _num_terminal_cols() - len(prefix) - max(len(suffix), 60))
    percent = f"{(100 * fraction):1.1f}"
    filled = int(bar_width * fraction)
    filled_char = ("￭" if app.use_unicode else "#")
    empty_char = ("･" if app.use_unicode else "-")
    bar_str = filled_char * filled + empty_char * (bar_width - filled)
    sys.stdout.write(f"\r{prefix} [{bar_str}] {percent:>5}% {suffix[:49]}")
    sys.stdout.flush()


@register_hook
def erase_progress_bar(app):
    sys.stdout.write(f"\r{' ' * _num_terminal_cols()}\r")
    sys.stdout.flush()


@register_hook
def prompt_yes_or_no(message, default_to_yes):
    yes_no = ("([y]/n)" if default_to_yes else "(y/[n])")
    while True:
        try:
            answer = input(f"{message} {yes_no} ")
        except EOFError:
            raise Error("\nStandard input is not available, aborting."
                " Consider passing --yes.")

        if not answer:
            return default_to_yes
        if answer.lower() in ("y", "yes"):
            return True
        if answer.lower() in ("n", "no"):
            return False


def _extract_zip(zip_path, to_dir):
    # We can't call zip_file.extractall(to_dir) because it does not preserve
    # file modification times, and we need those times to match what's in
    # the manifest.
    to_dir = Path(to_dir)
    dirs_and_times = []
    with zipfile.ZipFile(zip_path, "r") as zip_file:
        for zip_member in zip_file.infolist():
            # Compute the modification timestamp.  We need to extend the zip
            # file's date_time to add dummy values for DST and timezone.
            mod_timestamp = int(
                time.mktime(zip_member.date_time + (0, 0, -1)))

            # Adding a file to a directory will clobber the directory's
            # modification time, so we need to set times on the directories at
            # the very end.
            to_file_path = to_dir / zip_member.filename
            if zip_member.is_dir():
                to_file_path.mkdir(parents=True, exist_ok=True)
                dirs_and_times.append((to_file_path, mod_timestamp))
            else:
                to_file_path.parent.mkdir(parents=True, exist_ok=True)
                with zip_file.open(zip_member) as source, \
                        to_file_path.open("wb") as target:
                    shutil.copyfileobj(source, target)
                os.utime(to_file_path, (mod_timestamp, mod_timestamp))

    # Set directory timestamps, doing the deepest ones first.
    dirs_and_times.sort(key=lambda dir_and_time: -len(str(dir_and_time[0])))
    for to_dir_path, mod_timestamp in dirs_and_times:
        os.utime(to_dir_path, (mod_timestamp, mod_timestamp))


def _md5_checksum_file(file_path):
    hasher = hashlib.md5()
    with open(file_path, "rb") as open_file:
        for chunk in iter(lambda: open_file.read(CHUNK_SIZE), b""):
            hasher.update(chunk)
    return hasher.hexdigest()


def _crc32_checksum_file(file_path):
    checksum = 0
    with open(file_path, "rb") as open_file:
        for chunk in iter(lambda: open_file.read(CHUNK_SIZE), b""):
            checksum = binascii.crc32(chunk, checksum)
    return checksum


def _num_terminal_cols():
    return shutil.get_terminal_size(fallback=(80, 20)).columns


def print_error(app, message, word_wrap=True):
    app.hooks.print_info("Error: " + message, word_wrap)


def print_warning(app, message, verbosity=1, word_wrap=True):
    if app.verbosity < verbosity:
        return
    app.hooks.print_info("Warning: " + message, word_wrap=word_wrap)


@register_hook
def print_info(message, word_wrap=True):
    if word_wrap:
        message = "\n".join(
            textwrap.fill(
                line, _num_terminal_cols(), break_long_words=False,
                replace_whitespace=False)
            for line in message.splitlines())
    if not message.endswith("\n"):
        message += "\n"
    sys.stderr.write(message)


def _can_access_file(file_path, mode):
    assert mode in (os.R_OK, os.W_OK)
    path = Path(file_path)
    if path.exists():
        return path.is_file() and os.access(path, mode)

    return (mode == os.W_OK and path.parent.is_dir() and
        os.access(path.parent, os.W_OK))


def _can_read_file(file_path):
    return _can_access_file(file_path, os.R_OK)


def _b64encode_text(text):
    return base64.b64encode(text.encode()).decode()
