# Copyright 2015-2016 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import difflib
import fnmatch
import getpass
import hashlib
import logging
import os
import random
import re
import socket
import subprocess
from collections import defaultdict
from shlex import quote
from typing import Callable
from typing import Collection
from typing import Generator
from typing import Iterable
from typing import List
from typing import Mapping
from typing import NamedTuple
from typing import Optional
from typing import Sequence
from typing import Set
from typing import Tuple

import ephemeral_port_reserve
from botocore.credentials import InstanceMetadataFetcher
from botocore.credentials import InstanceMetadataProvider
from mypy_extensions import NamedArg

from paasta_tools import remote_git
from paasta_tools.adhoc_tools import load_adhoc_job_config
from paasta_tools.cassandracluster_tools import load_cassandracluster_instance_config
from paasta_tools.eks_tools import EksDeploymentConfig
from paasta_tools.eks_tools import load_eks_service_config
from paasta_tools.flink_tools import load_flink_instance_config
from paasta_tools.flinkeks_tools import load_flinkeks_instance_config
from paasta_tools.kafkacluster_tools import load_kafkacluster_instance_config
from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
from paasta_tools.kubernetes_tools import load_kubernetes_service_config
from paasta_tools.long_running_service_tools import LongRunningServiceConfig
from paasta_tools.monkrelaycluster_tools import load_monkrelaycluster_instance_config
from paasta_tools.nrtsearchservice_tools import load_nrtsearchservice_instance_config
from paasta_tools.nrtsearchserviceeks_tools import (
    load_nrtsearchserviceeks_instance_config,
)
from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
from paasta_tools.tron_tools import load_tron_instance_config
from paasta_tools.utils import _log
from paasta_tools.utils import _run
from paasta_tools.utils import compose_job_id
from paasta_tools.utils import DEFAULT_SOA_CONFIGS_GIT_URL
from paasta_tools.utils import DEFAULT_SOA_DIR
from paasta_tools.utils import get_service_instance_list
from paasta_tools.utils import INSTANCE_TYPE_TO_K8S_NAMESPACE
from paasta_tools.utils import INSTANCE_TYPES
from paasta_tools.utils import InstanceConfig
from paasta_tools.utils import list_all_instances_for_service
from paasta_tools.utils import list_clusters
from paasta_tools.utils import list_services
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import PAASTA_K8S_INSTANCE_TYPES
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import SystemPaastaConfig
from paasta_tools.utils import validate_service_instance
from paasta_tools.vitesscluster_tools import load_vitess_instance_config

try:
    from vault_tools.paasta_secret import get_client as get_vault_client
    from vault_tools.paasta_secret import get_vault_url
    from vault_tools.paasta_secret import get_vault_ca
except ImportError:

    def get_vault_client(url: str, capath: str) -> None:
        pass

    def get_vault_url(ecosystem: str) -> str:
        return ""

    def get_vault_ca(ecosystem: str) -> str:
        return ""


log = logging.getLogger(__name__)


def is_file_in_dir(file_name, path):
    """Recursively search path for file_name.

    :param file_name: a string of a file name to find
    :param path: a string path
    :param file_ext: a string of a file extension
    :return: a boolean
    """
    for root, dirnames, filenames in os.walk(path):
        for filename in filenames:
            if fnmatch.fnmatch(filename, file_name):
                return os.path.join(root, filename)
    return False


def get_file_contents(path):
    """Open a file for reading

    :param path: path of file to read
    """
    with open(path) as p:
        return p.read()


def check_mark():
    """
    :return: string that can print a checkmark
    """
    return PaastaColors.green("\u2713")


def x_mark():
    """
    :return: string that can print an x-mark
    """
    return PaastaColors.red("\u2717")


def info_mark() -> str:
    """
    :return: string that can print an info symbol
    """
    return PaastaColors.blue("\u2139")


def success(msg):
    """Format a paasta check success message.

    :param msg: a string
    :return: a beautiful string
    """
    return "{} {}".format(check_mark(), msg)


def failure(msg, link):
    """Format a paasta check failure message.

    :param msg: a string
    :return: a beautiful string
    """
    return "{} {} {}".format(x_mark(), msg, PaastaColors.blue(link))


def info_message(msg: str) -> str:
    return f"{info_mark()} {msg}"


class PaastaCheckMessages:

    """Collection of message printed out by 'paasta check'.
    Helpful as it avoids cumbersome maintenance of the unit tests.
    """

    DEPLOY_YAML_FOUND = success("deploy.yaml exists for a Jenkins pipeline")

    DEPLOY_YAML_MISSING = failure(
        "No deploy.yaml exists, so your service cannot be deployed.\n  "
        "Please push a deploy.yaml.\n  "
        "More info:",
        "http://y/yelpsoa-configs",
    )

    DEPLOY_SECURITY_FOUND = success(
        "Found a security-check entry in your deploy pipeline"
    )
    DEPLOY_SECURITY_MISSING = failure(
        "No 'security-check' entry was found in your deploy.yaml.\n"
        "Please add a security-check entry *AFTER* the itest entry in deploy.yaml\n"
        "so your docker image can be checked against known security vulnerabilities.\n"
        "More info:",
        "http://paasta.readthedocs.io/en/latest/generated/paasta_tools.cli.cmds.security_check.html",
    )

    DOCKERFILE_FOUND = success("Found Dockerfile")

    DOCKERFILE_MISSING = failure(
        "Dockerfile not found. Create a Dockerfile and try again.\n  " "More info:",
        "http://y/paasta-runbook-dockerfile",
    )

    DOCKERFILE_YELPCORP = success(
        "Your Dockerfile pulls from the standard Yelp images."
    )

    DOCKERFILE_NOT_YELPCORP = failure(
        "Your Dockerfile does not use the standard Yelp images.\n  "
        "This is bad because your `docker pulls` will be slow and you won't be "
        "using the local mirrors.\n"
        "More info:",
        "http://y/base-docker-images",
    )

    GIT_REPO_FOUND = success("Git repo found in the expected location.")

    ADHOC_YAML_FOUND = success("Found adhoc.yaml file.")

    MAKEFILE_FOUND = success("A Makefile is present")
    MAKEFILE_MISSING = failure(
        "No Makefile available. Please make a Makefile that responds\n"
        "to the proper targets. More info:",
        "http://paasta.readthedocs.io/en/latest/about/contract.html",
    )
    MAKEFILE_RESPONDS_BUILD_IMAGE = success(
        "The Makefile responds to `make cook-image`"
    )
    MAKEFILE_RESPONDS_BUILD_IMAGE_FAIL = failure(
        "The Makefile does not have a `make cook-image` target. local-run needs\n"
        "this and expects it to build your docker image. More info:",
        "http://paasta.readthedocs.io/en/latest/about/contract.html",
    )
    MAKEFILE_RESPONDS_ITEST = success("The Makefile responds to `make itest`")
    MAKEFILE_RESPONDS_ITEST_FAIL = failure(
        "The Makefile does not have a `make itest` target. Jenkins needs\n"
        "this and expects it to build and itest your docker image. More info:",
        "http://paasta.readthedocs.io/en/latest/about/contract.html",
    )
    MAKEFILE_RESPONDS_TEST = success("The Makefile responds to `make test`")
    MAKEFILE_RESPONDS_TEST_FAIL = failure(
        "The Makefile does not have a `make test` target. Jenkins needs\n"
        "this and expects it to run unit tests. More info:",
        "http://paasta.readthedocs.io/en/latest/about/contract.html",
    )
    MAKEFILE_HAS_A_TAB = success("The Makefile contains a tab character")
    MAKEFILE_HAS_NO_TABS = failure(
        "The Makefile contains no tab characters. Make sure you\n"
        "didn't accidentally paste spaces (which `make` does not respect)\n"
        "instead of a tab.",
        "http://paasta.readthedocs.io/en/latest/about/contract.html",
    )
    MAKEFILE_HAS_DOCKER_TAG = success("The Makefile contains a docker tag")
    MAKEFILE_HAS_NO_DOCKER_TAG = failure(
        "The Makefile contains no reference to DOCKER_TAG. Make sure you\n"
        "specify a DOCKER_TAG and that your itest tags your docker image with $DOCKER_TAG.",
        "http://paasta.readthedocs.io/en/latest/about/contract.html",
    )

    SENSU_MONITORING_FOUND = success("monitoring.yaml found for Sensu monitoring")

    SENSU_MONITORING_MISSING = failure(
        "Your service is not using Sensu (monitoring.yaml).\n  "
        "Please setup a monitoring.yaml so we know where to send alerts.\n  "
        "More info:",
        "http://y/monitoring-yaml",
    )

    SENSU_TEAM_MISSING = failure(
        "Cannot get team name. Ensure 'team' field is set in monitoring.yaml.\n"
        "  More info:",
        "http://y/monitoring-yaml",
    )

    SMARTSTACK_YAML_FOUND = success("Found smartstack.yaml file")

    SMARTSTACK_PORT_MISSING = failure(
        "Could not determine port. "
        "Ensure 'proxy_port' is set in smartstack.yaml.\n  "
        "More info:",
        "http://y/smartstack-cep323",
    )

    @staticmethod
    def git_repo_missing(git_url):
        git_url = PaastaColors.cyan(git_url)
        return failure(
            "Could not find Git repo %s. "
            "Your service must be there.\n"
            "  More info:" % git_url,
            "http://y/yelpsoa-configs",
        )

    @staticmethod
    def sensu_team_found(team_name):
        return success(
            "Your service uses Sensu and team '%s' will get alerts." % team_name
        )

    @staticmethod
    def smartstack_port_found(instance, port):
        return success(
            "Instance '%s' of your service is using smartstack port %d "
            "and will be automatically load balanced" % (instance, port)
        )

    @staticmethod
    def service_dir_found(service, soa_dir):
        message = "yelpsoa-config directory for {} found in {}".format(
            PaastaColors.cyan(service), soa_dir
        )
        return success(message)

    @staticmethod
    def service_dir_missing(service, soa_dir):
        message = (
            "Failed to locate yelpsoa-config directory for %s in %s.\n"
            "  Please follow the guide linked below to get boilerplate."
            % (service, soa_dir)
        )
        return failure(message, "http://y/paasta-deploy")


class NoSuchService(Exception):

    """Exception to be raised in the event that the service
    name can not be guessed.
    """

    GUESS_ERROR_MSG = (
        "Could not determine service name.\n"
        "Please run this from the root of a copy "
        "(git clone) of your service.\n"
        "Alternatively, supply the %s name you wish to "
        "inspect with the %s option."
        % (PaastaColors.cyan("SERVICE"), PaastaColors.cyan("-s"))
    )

    CHECK_ERROR_MSG = (
        "not found.  Please provide a valid service name.\n"
        "Ensure that a directory of the same name exists in %s."
        % PaastaColors.green("/nail/etc/services")
    )

    def __init__(self, service):
        self.service = service

    def __str__(self):
        if self.service:
            return "SERVICE: {} {}".format(
                PaastaColors.cyan(self.service), self.CHECK_ERROR_MSG
            )
        else:
            return self.GUESS_ERROR_MSG


def guess_service_name():
    """Deduce the service name from the pwd
    :return : A string representing the service name
    """
    return os.path.basename(os.getcwd())


def validate_service_name(service, soa_dir=DEFAULT_SOA_DIR):
    """Determine whether directory named service exists in the provided soa_dir
    :param service: a string of the name of the service you wish to check exists
    :param soa_dir: directory to look for service names
    :return : boolean True
    :raises: NoSuchService exception
    """
    if not service or not os.path.isdir(os.path.join(soa_dir, service)):
        raise NoSuchService(service)
    return True


def list_paasta_services(soa_dir: str = DEFAULT_SOA_DIR):
    """Returns a sorted list of services that happen to have at
    least one service.instance, which indicates it is on PaaSTA
    """
    the_list = []
    for service in list_services(soa_dir):
        if list_all_instances_for_service(service, soa_dir=soa_dir):
            the_list.append(service)
    return the_list


def list_service_instances(soa_dir: str = DEFAULT_SOA_DIR):
    """Returns a sorted list of service<SPACER>instance names"""
    the_list = []
    for service in list_services(soa_dir):
        for instance in list_all_instances_for_service(
            service=service, soa_dir=soa_dir
        ):
            the_list.append(compose_job_id(service, instance))
    return the_list


def list_instances(**kwargs):
    """Returns a sorted list of all possible instance names
    for tab completion. We try to guess what service you might be
    operating on, otherwise we just provide *all* of them
    """
    all_instances: Set[str] = set()
    service = guess_service_name()
    try:
        validate_service_name(service)
        all_instances = set(list_all_instances_for_service(service))
    except NoSuchService:
        for service in list_services():
            for instance in list_all_instances_for_service(service):
                all_instances.add(instance)
    return sorted(all_instances)


def calculate_remote_masters(
    cluster: str, system_paasta_config: SystemPaastaConfig
) -> Tuple[List[str], str]:
    """Given a cluster, do a DNS lookup of that cluster (which
    happens to point, eventually, to the Mesos masters in that cluster).
    Return IPs of those Mesos masters.
    """

    cluster_fqdn = system_paasta_config.get_cluster_fqdn_format().format(
        cluster=cluster
    )
    try:
        _, _, ips = socket.gethostbyname_ex(cluster_fqdn)
        output = None
    except socket.gaierror as e:
        output = f"ERROR while doing DNS lookup of {cluster_fqdn}:\n{e.strerror}\n "
        ips = []
    return (ips, output)


def find_connectable_master(
    masters: Sequence[str],
) -> Tuple[Optional[str], Optional[str]]:
    """For each host in the iterable 'masters', try various connectivity
    checks. For each master that fails, emit an error message about which check
    failed and move on to the next master.

    If a master passes all checks, return a tuple of the connectable master and
    None. If no masters pass all checks, return a tuple of None and the output
    from the DNS lookup.
    """
    timeout = 6.0  # seconds

    connectable_master = None
    for master in masters:
        rc, output = check_ssh_on_master(master, timeout=timeout)
        if rc is True:
            connectable_master = master
            output = None
            break
    return (connectable_master, output)


class NoMasterError(Exception):
    pass


def connectable_master(cluster: str, system_paasta_config: SystemPaastaConfig) -> str:
    masters, output = calculate_remote_masters(cluster, system_paasta_config)
    if masters == []:
        raise NoMasterError("ERROR: %s" % output)

    random.shuffle(masters)

    master, output = find_connectable_master(masters)
    if not master:
        raise NoMasterError(
            f"ERROR: could not find connectable master in cluster {cluster}\nOutput: {output}"
        )

    return master


def check_ssh_on_master(master, timeout=10):
    """Given a master, attempt to ssh to the master and run a simple command
    with sudo to verify that ssh and sudo work properly. Return a tuple of the
    success status (True or False) and any output from attempting the check.
    """
    check_command = "ssh -A -n -o StrictHostKeyChecking=no %s /bin/true" % master
    rc, output = _run(check_command, timeout=timeout)
    if rc == 0:
        return (True, None)
    if rc == 255:  # ssh error
        reason = "Return code was %d which probably means an ssh failure." % rc
        hint = "HINT: Are you allowed to ssh to this machine %s?" % master
    if rc == 1:  # sudo error
        reason = "Return code was %d which probably means a sudo failure." % rc
        hint = "HINT: Is your ssh agent forwarded? (ssh-add -l)"
    if rc == -9:  # timeout error
        reason = (
            "Return code was %d which probably means ssh took too long and timed out."
            % rc
        )
        hint = "HINT: Is there network latency? Try running somewhere closer to the cluster."
    else:  # unknown error
        reason = "Return code was %d which is an unknown failure." % rc
        hint = "HINT: Talk to #paasta and pastebin this output"
    output = (
        "ERROR cannot run check command %(check_command)s\n"
        "%(reason)s\n"
        "%(hint)s\n"
        "Output from check command: %(output)s"
        % {
            "check_command": check_command,
            "reason": reason,
            "hint": hint,
            "output": output,
        }
    )
    return (False, output)


def run_on_master(
    cluster,
    system_paasta_config,
    cmd_parts,
    timeout=None,
    err_code=-1,
    graceful_exit=False,
    stdin=None,
):
    """Find connectable master for :cluster: and :system_paasta_config: args and
    invoke command from :cmd_parts:, wrapping it in ssh call.

    :returns (exit code, output)

    :param cluster: cluster to find master in
    :param system_paasta_config: system configuration to lookup master data
    :param cmd_parts: passed into paasta_tools.utils._run as command along with
        ssh bits
    :param timeout: see paasta_tools.utils._run documentation (default: None)
    :param err_code: code to return along with error message when something goes
        wrong (default: -1)
    :param graceful_exit: wrap command in a bash script that waits for input and
        kills the original command; trap SIGINT and send newline into stdin
    """
    try:
        master = connectable_master(cluster, system_paasta_config)
    except NoMasterError as e:
        return (err_code, str(e))

    if graceful_exit:
        # Signals don't travel over ssh, kill process when anything lands on stdin instead
        # The procedure here is:
        # 1. send process to background and capture it's pid
        # 2. wait for stdin with timeout in a loop, exit when original process finished
        # 3. kill original process if loop finished (something on stdin)
        cmd_parts.append(
            "& p=$!; "
            + "while ! read -t1; do ! kill -0 $p 2>/dev/null && kill $$; done; "
            + "kill $p; wait"
        )
        stdin = subprocess.PIPE
        stdin_interrupt = True
        popen_kwargs = {"preexec_fn": os.setsid}
    else:
        stdin_interrupt = False
        popen_kwargs = {}

    cmd_parts = [
        "ssh",
        "-q",
        "-t",
        "-t",
        "-A",
        master,
        "sudo /bin/bash -c %s" % quote(" ".join(cmd_parts)),
    ]

    log.debug("Running %s" % " ".join(cmd_parts))

    return _run(
        cmd_parts,
        timeout=timeout,
        stream=True,
        stdin=stdin,
        stdin_interrupt=stdin_interrupt,
        popen_kwargs=popen_kwargs,
    )


def lazy_choices_completer(list_func):
    def inner(prefix, **kwargs):
        options = list_func(**kwargs)
        return [o for o in options if o.startswith(prefix)]

    return inner


def figure_out_service_name(args, soa_dir=DEFAULT_SOA_DIR):
    """Figures out and validates the input service name"""
    service = args.service or guess_service_name()
    try:
        validate_service_name(service, soa_dir=soa_dir)
    except NoSuchService as service_not_found:
        print(service_not_found)
        exit(1)
    return service


def get_jenkins_build_output_url():
    """Returns the URL for Jenkins job's output.
    Returns None if it's not available.
    """
    build_output = os.environ.get("BUILD_URL")
    if build_output:
        build_output = build_output + "console"
    return build_output


InstanceListerSig = Callable[
    [
        NamedArg(str, "service"),
        NamedArg(Optional[str], "cluster"),
        NamedArg(str, "instance_type"),
        NamedArg(str, "soa_dir"),
    ],
    List[Tuple[str, str]],
]

InstanceLoaderSig = Callable[
    [
        NamedArg(str, "service"),
        NamedArg(str, "instance"),
        NamedArg(str, "cluster"),
        NamedArg(bool, "load_deployments"),
        NamedArg(str, "soa_dir"),
    ],
    InstanceConfig,
]

LongRunningServiceListerSig = Callable[
    [
        NamedArg(str, "service"),
        NamedArg(Optional[str], "cluster"),
        NamedArg(str, "instance_type"),
        NamedArg(str, "soa_dir"),
    ],
    List[Tuple[str, str]],
]

LongRunningServiceLoaderSig = Callable[
    [
        NamedArg(str, "service"),
        NamedArg(str, "instance"),
        NamedArg(str, "cluster"),
        NamedArg(bool, "load_deployments"),
        NamedArg(str, "soa_dir"),
    ],
    LongRunningServiceConfig,
]


class InstanceTypeHandler(NamedTuple):
    lister: InstanceListerSig
    loader: InstanceLoaderSig


class LongRunningInstanceTypeHandler(NamedTuple):
    lister: LongRunningServiceListerSig
    loader: LongRunningServiceLoaderSig


INSTANCE_TYPE_HANDLERS: Mapping[str, InstanceTypeHandler] = defaultdict(
    lambda: InstanceTypeHandler(None, None),
    adhoc=InstanceTypeHandler(get_service_instance_list, load_adhoc_job_config),
    kubernetes=InstanceTypeHandler(
        get_service_instance_list, load_kubernetes_service_config
    ),
    eks=InstanceTypeHandler(get_service_instance_list, load_eks_service_config),
    tron=InstanceTypeHandler(get_service_instance_list, load_tron_instance_config),
    flink=InstanceTypeHandler(get_service_instance_list, load_flink_instance_config),
    flinkeks=InstanceTypeHandler(
        get_service_instance_list, load_flinkeks_instance_config
    ),
    cassandracluster=InstanceTypeHandler(
        get_service_instance_list, load_cassandracluster_instance_config
    ),
    kafkacluster=InstanceTypeHandler(
        get_service_instance_list, load_kafkacluster_instance_config
    ),
    vitesscluster=InstanceTypeHandler(
        get_service_instance_list, load_vitess_instance_config
    ),
    nrtsearchservice=InstanceTypeHandler(
        get_service_instance_list, load_nrtsearchservice_instance_config
    ),
    nrtsearchserviceeks=InstanceTypeHandler(
        get_service_instance_list, load_nrtsearchserviceeks_instance_config
    ),
    monkrelays=InstanceTypeHandler(
        get_service_instance_list, load_monkrelaycluster_instance_config
    ),
)

LONG_RUNNING_INSTANCE_TYPE_HANDLERS: Mapping[
    str, LongRunningInstanceTypeHandler
] = defaultdict(
    lambda: LongRunningInstanceTypeHandler(None, None),
    kubernetes=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_kubernetes_service_config
    ),
    flink=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_flink_instance_config
    ),
    flinkeks=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_flinkeks_instance_config
    ),
    cassandracluster=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_cassandracluster_instance_config
    ),
    kafkacluster=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_kafkacluster_instance_config
    ),
    vitesscluster=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_vitess_instance_config
    ),
    nrtsearchservice=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_nrtsearchservice_instance_config
    ),
    nrtsearchserviceeks=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_nrtsearchserviceeks_instance_config
    ),
    monkrelays=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_monkrelaycluster_instance_config
    ),
    eks=LongRunningInstanceTypeHandler(
        get_service_instance_list, load_eks_service_config
    ),
)


def get_instance_config(
    service: str,
    instance: str,
    cluster: str,
    soa_dir: str = DEFAULT_SOA_DIR,
    load_deployments: bool = False,
    instance_type: Optional[str] = None,
) -> InstanceConfig:
    """Returns the InstanceConfig object for whatever type of instance
    it is. (kubernetes)"""
    if instance_type is None:
        instance_type = validate_service_instance(
            service=service, instance=instance, cluster=cluster, soa_dir=soa_dir
        )

    instance_config_loader = INSTANCE_TYPE_HANDLERS[instance_type].loader
    if instance_config_loader is None:
        raise NotImplementedError(
            "instance is %s of type %s which is not supported by paasta"
            % (instance, instance_type)
        )

    return instance_config_loader(
        service=service,
        instance=instance,
        cluster=cluster,
        load_deployments=load_deployments,
        soa_dir=soa_dir,
    )


def get_namespaces_for_secret(
    service: str, cluster: str, secret_name: str, soa_dir: str = DEFAULT_SOA_DIR
) -> Set[str]:
    secret_to_k8s_namespace = set()

    k8s_instance_type_classes = {
        "kubernetes": KubernetesDeploymentConfig,
        "eks": EksDeploymentConfig,
    }
    for instance_type in INSTANCE_TYPES:
        if instance_type in PAASTA_K8S_INSTANCE_TYPES:
            config_loader = PaastaServiceConfigLoader(service, soa_dir)
            for service_instance_config in config_loader.instance_configs(
                cluster=cluster,
                instance_type_class=k8s_instance_type_classes[instance_type],
            ):
                secret_to_k8s_namespace.add(service_instance_config.get_namespace())
        else:
            instances = get_service_instance_list(
                service=service,
                instance_type=instance_type,
                cluster=cluster,
                soa_dir=soa_dir,
            )

            for serv, instance in instances:
                config = get_instance_config(serv, instance, cluster, soa_dir)
                if secret_name in config.get_env():
                    secret_to_k8s_namespace.add(
                        INSTANCE_TYPE_TO_K8S_NAMESPACE[instance_type]
                    )

    return secret_to_k8s_namespace


def select_k8s_secret_namespace(namespaces: Set[str]) -> Optional[str]:
    namespaces_count = len(namespaces)

    if not namespaces_count:
        return None

    if namespaces_count == 1:
        return namespaces.pop()

    # prioritise paasta, tron namespaces when found
    for namespace in namespaces:
        if namespace.startswith("paasta"):
            return namespace
        if namespace == "tron":
            return namespace

    # only experimental k8s namespaces
    return namespaces.pop()


def extract_tags(paasta_tag: str) -> Mapping[str, str]:
    """Returns a dictionary containing information from a git tag"""
    regex = r"^refs/tags/(?:paasta-){1,2}(?P<deploy_group>[a-zA-Z0-9._-]+)(?:\+(?P<image_version>.*)){0,1}-(?P<tstamp>\d{8}T\d{6})-(?P<tag>.*?)$"
    regex_match = re.match(regex, paasta_tag)
    return regex_match.groupdict() if regex_match else {}


def list_deploy_groups(
    service: Optional[str], soa_dir: str = DEFAULT_SOA_DIR, parsed_args=None, **kwargs
) -> Set:
    return set(
        filter(
            None,
            {
                config.get_deploy_group()
                for config in get_instance_configs_for_service(
                    service=service
                    if service is not None
                    else parsed_args.service or guess_service_name(),
                    soa_dir=soa_dir,
                )
            },
        )
    )


def validate_given_deploy_groups(
    all_deploy_groups: Collection[str], args_deploy_groups: Collection[str]
) -> Tuple[Set[str], Set[str]]:
    """Given two lists of deploy groups, return the intersection and difference between them.

    :param all_deploy_groups: instances actually belonging to a service
    :param args_deploy_groups: the desired instances
    :returns: a tuple with (common, difference) indicating deploy groups common in both
        lists and those only in args_deploy_groups
    """
    invalid_deploy_groups: Set[str]
    valid_deploy_groups = set(args_deploy_groups).intersection(all_deploy_groups)
    invalid_deploy_groups = set(args_deploy_groups).difference(all_deploy_groups)

    return valid_deploy_groups, invalid_deploy_groups


def short_to_full_git_sha(short, refs):
    """Converts a short git sha to a full SHA

    :param short: A short Git SHA represented as a string
    :param refs: A list of refs in the Git repository
    :return: The full Git SHA or None if one can't be found
    """
    return [sha for sha in set(refs.values()) if sha.startswith(short)]


def validate_short_git_sha(value):
    pattern = re.compile("[a-f0-9]{4,40}")
    if not pattern.match(value):
        raise argparse.ArgumentTypeError("%s is not a valid Git SHA" % value)
    return value


def validate_full_git_sha(value: str) -> str:
    pattern = re.compile("[a-f0-9]{40}")
    if not pattern.match(value):
        raise argparse.ArgumentTypeError(
            "%s is not a full Git SHA, and PaaSTA needs the full SHA" % value
        )
    return value


def validate_git_sha(sha, git_url):
    try:
        validate_full_git_sha(sha)
        return sha
    except argparse.ArgumentTypeError:
        refs = remote_git.list_remote_refs(git_url)
        commits = short_to_full_git_sha(short=sha, refs=refs)
        if len(commits) != 1:
            raise ValueError(
                "%s matched %d Git SHAs (with refs pointing at them). Must match exactly 1."
                % (sha, len(commits))
            )
        return commits[0]


def get_subparser(subparsers, function, command, help_text, description):
    new_parser = subparsers.add_parser(
        command,
        help=help_text,
        description=(description),
        epilog=(
            "Note: This command requires SSH and sudo privileges on the remote PaaSTA "
            "nodes."
        ),
    )
    new_parser.add_argument(
        "-s",
        "--service",
        help="The name of the service you wish to inspect",
        required=True,
    ).completer = lazy_choices_completer(list_services)
    new_parser.add_argument(
        "-c",
        "--cluster",
        help="Cluster on which the service is running"
        "For example: --cluster pnw-prod",
        required=True,
    ).completer = lazy_choices_completer(list_clusters)
    new_parser.add_argument(
        "-i",
        "--instance",
        help="The instance that you wish to inspect" "For example: --instance main",
        required=True,
        default="main",
    )  # No completer because we need to know service first and we can't until some other stuff has happened
    new_parser.add_argument(
        "-H",
        "--host",
        dest="host",
        default=None,
        help="Specify a specific host on which to run. Defaults to"
        " one that is running the service chosen at random",
    )
    new_parser.add_argument(
        "-m",
        "--mesos-id",
        dest="mesos_id",
        default=None,
        help="A specific mesos task ID, must match a task "
        "running on the specified host. If not specified we "
        "will pick a task at random",
    )
    new_parser.set_defaults(command=function)
    return new_parser


def get_instance_configs_for_service(
    service: str,
    soa_dir: str,
    type_filter: Optional[Iterable[str]] = None,
    clusters: Optional[Sequence[str]] = None,
    instances: Optional[Sequence[str]] = None,
) -> Generator[InstanceConfig, None, None]:
    if not clusters:
        clusters = list_clusters(service=service, soa_dir=soa_dir)

    if type_filter is None:
        type_filter = INSTANCE_TYPE_HANDLERS.keys()

    for cluster in clusters:
        for instance_type, instance_handlers in INSTANCE_TYPE_HANDLERS.items():
            if instance_type not in type_filter:
                continue

            instance_lister, instance_loader = instance_handlers

            for _, instance in instance_lister(
                service=service,
                cluster=cluster,
                soa_dir=soa_dir,
                instance_type=instance_type,
            ):
                if instances and instance not in instances:
                    continue

                yield instance_loader(
                    service=service,
                    instance=instance,
                    cluster=cluster,
                    soa_dir=soa_dir,
                    load_deployments=False,
                )


def get_container_name(task):
    container_name = "mesos-{}".format(task.executor["container"])
    return container_name


def pick_random_port(service_name):
    """Return a random port.

    Tries to return the same port for the same service each time, when
    possible.
    """
    hash_key = f"{service_name},{getpass.getuser()}".encode("utf8")
    hash_number = int(hashlib.sha1(hash_key).hexdigest(), 16)
    preferred_port = 33000 + (hash_number % 25000)
    return ephemeral_port_reserve.reserve("0.0.0.0", preferred_port)


def trigger_deploys(
    service: str,
    system_config: Optional["SystemPaastaConfig"] = None,
) -> None:
    """Connects to the deploymentsd watcher on sysgit, which is an extremely simple
    service that listens for a service string and then generates a service deployment"""
    logline = f"Notifying soa-configs primary to generate a deployment for {service}"
    _log(service=service, line=logline, component="deploy", level="event")
    if not system_config:
        system_config = load_system_paasta_config()
    server = system_config.get_git_repo_config("yelpsoa-configs").get(
        "deploy_server",
        DEFAULT_SOA_CONFIGS_GIT_URL,
    )

    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        client.connect((server, 5049))
        client.send(f"{service}\n".encode("utf-8"))
    finally:
        client.close()


def verify_instances(
    args_instances: str,
    service: str,
    clusters: Sequence[str],
    soa_dir: str = DEFAULT_SOA_DIR,
) -> Sequence[str]:
    """Verify that a list of instances specified by user is correct for this service.

    :param args_instances: a list of instances.
    :param service: the service name
    :param cluster: a list of clusters
    :returns: a list of instances specified in args_instances without any exclusions.
    """
    unverified_instances = args_instances.split(",")
    service_instances: Set[str] = list_all_instances_for_service(
        service, clusters=clusters, soa_dir=soa_dir
    )

    misspelled_instances: Sequence[str] = [
        i for i in unverified_instances if i not in service_instances
    ]

    if len(misspelled_instances) == 0:
        return misspelled_instances

    # Check for instances with suffixes other than Tron instances (i.e. Flink instances)
    instances_without_suffixes = [x.split(".")[0] for x in unverified_instances]

    misspelled_instances = [
        i for i in instances_without_suffixes if i not in service_instances
    ]

    if misspelled_instances:
        suggestions: List[str] = []
        for instance in misspelled_instances:
            matches = difflib.get_close_matches(
                instance, service_instances, n=5, cutoff=0.5
            )
            suggestions.extend(matches)  # type: ignore
        suggestions = list(set(suggestions))

        if clusters:
            message = "{} doesn't have any instances matching {} on {}.".format(
                service,
                ", ".join(sorted(misspelled_instances)),
                ", ".join(sorted(clusters)),
            )
        else:
            message = "{} doesn't have any instances matching {}.".format(
                service, ", ".join(sorted(misspelled_instances))
            )

        print(PaastaColors.red(message))

        if suggestions:
            print("Did you mean any of these?")
            for instance in sorted(suggestions):
                print("  %s" % instance)

    return misspelled_instances


def get_paasta_oapi_api_clustername(cluster: str, is_eks: bool) -> str:
    """
    We'll be doing a tiny bit of lying while we have both EKS and non-EKS
    clusters: these will generally share the same PaaSTA name (i.e., the
    soaconfigs suffix will stay the same) - but we'll need a way to route API
    requests to the correct place. To do so, we'll add "fake" entries to our
    api_endpoints SystemPaastaConfig that are the PaaSTA clustername with an
    "eks-" prefix
    """
    return f"eks-{cluster}" if is_eks else cluster


def get_current_ecosystem() -> str:
    """Get current ecosystem from host configs, defaults to dev if no config is found"""
    try:
        with open("/nail/etc/ecosystem") as f:
            return f.read().strip()
    except IOError:
        pass
    return "devc"


def get_service_auth_token() -> str:
    """Uses instance profile to authenticate with Vault and generate token for service authentication"""
    ecosystem = get_current_ecosystem()
    vault_client = get_vault_client(get_vault_url(ecosystem), get_vault_ca(ecosystem))
    vault_role = load_system_paasta_config().get_service_auth_vault_role()
    metadata_provider = InstanceMetadataProvider(
        iam_role_fetcher=InstanceMetadataFetcher(),
    )
    instance_credentials = metadata_provider.load().get_frozen_credentials()
    vault_client.auth.aws.iam_login(
        instance_credentials.access_key,
        instance_credentials.secret_key,
        instance_credentials.token,
        mount_point="aws-iam",
        role=vault_role,
        use_token=True,
    )
    response = vault_client.secrets.identity.generate_signed_id_token(name=vault_role)
    return response["data"]["token"]
