# pylint: disable=too-many-lines

import argparse
import ConfigParser
import imp
import inspect
import logging
import os
import sys
import ast

from functools import partial

import enum
import jinja2

from .color import Colors, switch as switch_colors
from .help import LineWrapRawTextHelpFormatter, option_help, docstring_to_help, trim_docstring, eval_context_help
from .log import Logging, ContextAdapter, ModuleAdapter, log_dict, VERBOSE

# Type annotations
# pylint: disable=unused-import,wrong-import-order
from typing import TYPE_CHECKING, cast, overload, Any, Callable, Dict, Iterable, List, Optional, Sequence  # noqa
from typing import Tuple, Type, Union  # noqa
from .log import LoggingFunctionType, LoggingWarningFunctionType, ExceptionInfoType  # noqa

if TYPE_CHECKING:
    import gluetool.color  # noqa
    import gluetool.utils  # noqa

# Type definitions
# pylint: disable=invalid-name
SharedType = Callable[..., Any]


DEFAULT_MODULE_CONFIG_PATHS = [
    '/etc/gluetool.d/config',
    os.path.expanduser('~/.gluetool.d/config'),
    os.path.abspath('./.gluetool.d/config')
]

DEFAULT_DATA_PATH = '{}/data'.format(os.path.dirname(os.path.abspath(__file__)))

DEFAULT_MODULE_PATHS = [
    '{}/gluetool_modules'.format(sys.prefix)
]


class DryRunLevels(enum.IntEnum):
    """
    Dry-run levels.

    :cvar int DEFAULT: Default level - everything is allowed.
    :cvar int DRY: Well-known "dry-run" - no changes to the outside world are allowed.
    :cvar int ISOLATED: No interaction with the outside world is allowed (networks connections, reading files, etc.)
    """

    DEFAULT = 0
    DRY = 1
    ISOLATED = 2


class GlueError(Exception):
    """
    Generic ``gluetool`` exception.

    :param str message: Exception message, describing what happened.
    :param tuple caused_by: If set, contains tuple as returned by :py:func:`sys.exc_info`, describing
        the exception that caused this one to be born. If not set, constructor will try to auto-detect
        this information, and if there's no such information, instance property ``caused_by`` will be
        set to ``None``.

    :ivar str message: Exception message, describing what happened.
    :ivar tuple caused_by: If set, contains tuple as returned by :py:func:`sys.exc_info`, describing
        the exception that caused this one to be born. ``None`` otherwise.
    """

    no_sentry_exceptions = []  # type: List[str]

    def __init__(self, message, caused_by=None, **kwargs):
        # type: (str, Optional[ExceptionInfoType], **Any) -> None

        super(GlueError, self).__init__(message, **kwargs)  # type: ignore  # too many arguments but it's fine

        self.message = message

        # if not told explicitly, try to detect the cause
        if caused_by is None:
            caused_by = sys.exc_info()

        # if there's no cause, use None to signal that fact to the rest of the world
        if caused_by == (None, None, None):
            caused_by = None

        self.caused_by = caused_by

    @property
    def submit_to_sentry(self):
        # type: () -> bool

        # pylint: disable=no-self-use

        """
        Decide whether the exception should be submitted to Sentry or not. By default,
        all exceptions are submitted. Exception listed in `no_sentry_exceptions` are not submitted.

        :rtype: bool
        :returns: ``True`` when the exception should be submitted to Sentry, ``False`` otherwise.
        """

        exception_name = '.'.join([type(self).__module__, type(self).__name__])
        if exception_name in self.no_sentry_exceptions:
            return False

        return True

    def sentry_fingerprint(self, current):
        # type: (List[str]) -> List[str]

        # pylint: disable=no-self-use
        """
        Default grouping of events into issues might be too general for some cases.
        This method gives users a chance to provide custom fingerprint Sentry could
        use to group events in a more suitable way.

        E.g. user might be interested in some sort of connection issues but they would
        like to have them grouped not by a traceback (which is the default method) but
        per remote host IP. For that, the Sentry integration code will call ``sentry_fingerprint``
        method of a raised exception, and the method should return new fingerprint,
        let's say ``[<exception class name>, <remote IP>]``, and Sentry will group events
        using this fingerprint.

        :param list(str) current: current fingerprint. Usually ``['{{ default }}']`` telling
            Sentry to use its default method, but it could already be more specific.
        :rtype: list(str)
        :returns: new fingerprint, e.g. ``['FailedToConnectToAPI', '10.20.30.40']``
        """

        return current

    def sentry_tags(self, current):
        # type: (Dict[str, str]) -> Dict[str, str]

        # pylint: disable=no-self-use
        """
        Add, modify or remove tags attached to a Sentry event, reported when the exception
        was raised.

        Most common usage would be an addition of tags, e.g. ``remote-host`` to allow search
        for events related to the same remote address.

        :param dict(str, str) current: current set of tags and their values.
        :rtype: dict(str, str)
        :returns: new set of tags. It is possible to add tags directly into ``current`` and
            then return it.
        """

        return current


class SoftGlueError(GlueError):
    """
    **Soft** errors are errors Glue Ops and/or developers shouldn't be bothered with, things that
    are up to the user to fix, e.g. empty set of tests. **Hard** errors are supposed to warn Ops/Devel
    teams about important infrastructure issues, code deficiencies, bugs and other issues that are
    fixable only by actions of Glue staff.

    However, we still must provide notification to user(s), and since we expect them to fix the issues
    that led to raising the soft error, we must provide them with as much information as possible.
    Therefore modules dealing with notifications are expected to give these exceptions a chance
    to influence the outgoing messages, e.g. by letting them provide an e-mail body template.
    """


class GlueRetryError(GlueError):
    """ Retry gluetool exception """


class GlueCommandError(GlueError):
    """
    Exception raised when external command failes.

    :param list cmd: Command as passed to gluetool.utils.run_command helper.
    :param gluetool.utils.ProcessOutput output: Process output data.

    :ivar list cmd: Command as passed to gluetool.utils.run_command helper.
    :ivar gluetool.utils.ProcessOutput output: Process output data.
    """

    def __init__(self, cmd, output, **kwargs):
        # type: (List[str], gluetool.utils.ProcessOutput, **Any) -> None

        super(GlueCommandError, self).__init__("Command '{}' failed with exit code {}".format(cmd, output.exit_code),
                                               **kwargs)

        self.cmd = cmd
        self.output = output


class Failure(object):
    # pylint: disable=too-few-public-methods

    """
    Bundles exception related info. Used to inform modules in their ``destroy()`` phase
    that ``gluetool`` session was killed because of exception raised by one of modules.

    :param gluetool.glue.Module module: module in which the error happened, or ``None``.
    :param tuple exc_info: Exception information as returned by :py:func:`sys.exc_info`.

    :ivar gluetool.glue.Module module: module in which the error happened, or ``None``.
    :ivar Exception exception: Shortcut to ``exc_info[1]``, if available, or ``None``.
    :ivar tuple exc_info: Exception information as returned by :py:func:`sys.exc_info`.
    :ivar str sentry_event_id: If set, the failure was reported to the Sentry under this ID.
    """

    def __init__(self, module, exc_info):
        # type: (Optional[Module], ExceptionInfoType) -> None

        self.module = module
        self.exc_info = exc_info

        self.sentry_event_id = None  # type: Optional[str]
        self.sentry_event_url = None  # type: Optional[str]

        if exc_info:
            self.exception = exc_info[1]
            self.soft = isinstance(self.exception, SoftGlueError)

        else:
            self.exception = None
            self.soft = False


def retry(*args):
    # type: (*Any) -> Any

    """ Retry decorator
    This decorator catches given exceptions and returns
    libRetryError exception instead.

    usage: @retry(exception1, exception2, ..)
    """
    def wrap(func):
        # type: (Any) -> Any

        def func_wrapper(obj, *fargs, **fkwargs):
            # type: (Any, *Any, **Any) -> Any

            try:
                func(obj, *fargs, **fkwargs)
            except args as e:
                if isinstance(e, GlueError):
                    raise GlueRetryError(e.value)  # type: ignore
                else:
                    raise GlueRetryError(e)
        return func_wrapper
    return wrap


class PipelineStep(object):
    # pylint: disable=too-few-public-methods
    """
    Step of ``gluetool``'s  pipeline - which is basically just a list of steps.

    :param str module: name to give to the module instance. This name is used e.g. in logging or when
        searching for module's config file.
    :param str actual_module: The actual module class the step uses. Usually it is same as ``module``
        but may differ, ``module`` is then a mere "alias". ``actual_module`` is used to locate
        a module class, whose instance is then given name ``module``.
    :param list(str) argv: list of options to be given to the module, in a form similar
        to :py:data:`sys.argv`.
    """

    def __init__(self, module, actual_module=None, argv=None):
        # type: (str, Optional[str], Optional[List[str]]) -> None

        self.module = module
        self.actual_module = actual_module or module
        self.argv = argv or []

    def __repr__(self):
        # type: () -> str

        return "PipelineStep('{}', actual_module='{}', argv={})".format(self.module, self.actual_module, self.argv)

    @property
    def module_designation(self):
        # type: () -> str

        return self.module if self.module == self.actual_module else '{}:{}'.format(self.module, self.actual_module)


class ArgumentParser(argparse.ArgumentParser):
    """
    Pretty much the :py:class:`argparse.ArgumentParser`, it overrides just
    the :py:meth:`argparse.ArgumentParser.error` method, to catch errors and to wrap them
    into nice and common :py:class:`GlueError` instances.

    The original prints (for us) useless message, including the program name, and raises ``SystemExit``
    exception. Such action does not provide necessary information when encountered in Sentry, for example.
    """

    def error(self, message):  # type: ignore  # incompatible with supertype because of unicode
        # type: (str) -> None

        """
        Must not return - raising an exception is a good way to "not return".

        :raises gluetool.glue.GlueError: When argument parser encounters an error.
        """

        raise GlueError('Parsing command-line options failed: {}'.format(message))


class Configurable(object):
    """
    Base class of two main ``gluetool`` classes - :py:class:`gluetool.glue.Glue` and :py:class:`gluetool.glue.Module`.
    Gives them the ability to use `options`, settable from configuration files and/or command-line arguments.

    :ivar dict _config: internal configuration store. Values of all options
      are stored here, regardless of them being set on command-line or by the
      configuration file.
    """

    # Logging type stubs
    # These methods are added dynamically, therefore without intruducing them to mypy, every use of `self.debug`
    # would cause an error when checking types. We cannot simply set them to `None`, that makes pylint go crazy
    # because `None` is apparently not callable, and we're calling `self.debug` often :) So, we use dummy lambdas
    # for initialization, to make pylint happy, but we wrap them by `cast` to proper types to make mypy happy
    # as well :)
    verbose = cast(LoggingFunctionType, lambda s: None)
    debug = cast(LoggingFunctionType, lambda s: None)
    info = cast(LoggingFunctionType, lambda s: None)
    warn = cast(LoggingWarningFunctionType, lambda s: None)
    error = cast(LoggingFunctionType, lambda s: None)
    exception = cast(LoggingFunctionType, lambda s: None)

    options = {}  # type: Union[Dict[Any, Any], List[Any]]
    """
    The ``options`` variable defines options accepted by module, and their properties::

        options = {
            <option name>: {
                <option properties>
            },
            ...
        }

    where

    * ``<option name>`` is used to `name` the option in the parser, and two formats are accepted (don't
      add any leading dashes (``-`` nor ``--``):

      * ``<long name>``
      * ``tuple(<short name>, <long name #1>, <long name #2>, ...)``

    * the first of long names (``long name #1``) is used to identify the option - other long names
      are understood by argument parser but their values are stored under ``long name #1`` option.

    * dictionary ``<option properties>`` is passed to :py:meth:`argparse.ArgumentParser.add_argument` as
      keyword arguments when the option is being added to the parser, therefore any arguments recognized
      by :py:mod:`argparse` can be used.

    It is also possible to use groups::

        options = [
            (<group name>,  <group options>),
            ...
        ]

    where

    * ``<group name>`` is the name of the group, e.g. ``Debugging options``

    * ``<group options>`` is the ``dict`` with all group options, as described above.

    This way, you can split pile of options into conceptualy closer groups of options. A single ``dict`` you would
    have is split into multiple smaller dictionaries, and each one is coupled with the group name in a ``tuple``.
    """

    required_options = []  # type: Iterable[str]
    """Iterable of names of required options."""

    options_note = None  # type: str
    """If set, it will be printed after all options as a help's epilog."""

    supported_dryrun_level = DryRunLevels.DEFAULT
    """Highest supported level of dry-run."""

    name = None  # type: str
    """
    Module name. Usually matches the name of the source file, no suffix.
    """

    unique_name = None  # type: Optional[str]
    """
    Unque name of this (module) instance.

    Used by modules, has no meaning elsewhere, but since dry-run checks are done on this level,
    it must be declared here to make pylint happy :/
    """

    @staticmethod
    def _for_each_option(callback, options):
        # type: (Callable[..., None], Any) -> None

        """
        Given dictionary defining options, call a callback for each of them.

        :param dict options: Dictionary of options, in a form ``option-name: option-params``.
        :param callable callback: Must accept at least 3 parameters: option name (``str``),
            all option names (short and long ones) (``tuple(str, str)``), and option params
            (``dict``).
        """

        # Sort options by their names - no code has a strong option on their order, so force
        # one to all users of this helper.
        option_names = sorted(options.keys(), key=lambda x: x[1] if isinstance(x, tuple) else x)

        for names in option_names:
            params = options[names]

            name = names[1] if isinstance(names, tuple) else names

            callback(name, names, params)

    @staticmethod
    def _for_each_option_group(callback, options):
        # type: (Callable[..., None], Any) -> None

        """
        Given set of options, call a callback for each option group.

        :param options: List of option groups, or a dict listing options directly.
        :param callable callback: Must accept at least 2 parameters: ``options`` (``dict``), listing options
            in the group, and keyword parameter ``group_name`` (``str``), which is set to group name when
            the ``options`` defines an option group.
        """

        if isinstance(options, dict):
            callback(options)

        elif isinstance(options, (list, tuple)):
            for group in options:
                if isinstance(group, dict):
                    callback(group)

                else:
                    group_name, group_options = group
                    callback(group_options, group_name=group_name)

    def __init__(self):
        # type: () -> None

        super(Configurable, self).__init__()

        # Initialize configuration store
        self._config = {}  # type: Dict[str, Any]

        # Initialize values in the store, and make sanity check of option names
        def _fail_name(name):
            # type: (str) -> None

            raise GlueError("Option name must be either a string or (<letter>, <string>), '{}' found".format(name))

        def _verify_option(name, names, params):
            # type: (str, List[str], Dict[str, Any]) -> None

            if isinstance(names, str):
                self._config[name] = None

            elif isinstance(names, tuple):
                if not isinstance(names[0], str) or len(names[0]) != 1:
                    _fail_name(name)

                if not isinstance(names[1], str) or len(names[1]) < 2:
                    _fail_name(name)

                self._config[name] = None

            else:
                _fail_name(name)

            if 'help' not in params:
                return

            # Long help texts can be written using triple quotes and docstring-like
            # formatting. Convert every help string to a single line string.
            params['help'] = option_help(params['help'])

        def _verify_options(options, **kwargs):
            # type: (Dict[str, Dict[str, Any]], **Any) -> None

            # pylint: disable=unused-argument

            Configurable._for_each_option(_verify_option, options)

        Configurable._for_each_option_group(_verify_options, self.options)

    def _parse_config(self, paths):
        # type: (List[str]) -> None

        """
        Parse configuration files. Uses :py:mod:`ConfigParser` for the actual parsing.
        Updates module's configuration store with values found returned by the parser.

        :param list paths: List of paths to possible configuration files.
        """

        log_dict(self.debug, 'Loading configuration from following paths', paths)

        parser = ConfigParser.ConfigParser()
        parsed_paths = parser.read(paths)

        log_dict(self.debug, 'Read configuration files', parsed_paths)

        def _inject_value(name, names, params):
            # type: (str, Tuple[str, ...], Dict[str, Any]) -> None

            # pylint: disable=unused-argument

            try:
                value = parser.get('default', name)

            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
                return

            if 'type' in params:
                try:
                    value = params['type'](value)

                except ValueError as exc:
                    raise GlueError("Value of option '{}' expected to be '{}' but cannot be parsed: '{}'".format(
                        name, params['type'].__name__, exc.message))

            self._config[name] = value
            self.debug("Option '{}' set to '{}' by config file".format(name, value))  # pylint: disable=not-callable

        def _inject_values(options, **kwargs):
            # type: (Any, **Any) -> None

            # pylint: disable=unused-argument

            Configurable._for_each_option(_inject_value, options)

        Configurable._for_each_option_group(_inject_values, self.options)

    @classmethod
    def _create_args_parser(cls, **kwargs):
        # type: (**Any) -> ArgumentParser

        """
        Create an argument parser. Used by Sphinx to document "command-line" options
        of the module - which are, by the way, the module options as well.

        :param dict kwargs: Additional arguments passed to :py:class:`argparse.ArgumentParser`.
        """

        root_parser = ArgumentParser(**kwargs)

        def _add_option(parser, name, names, params):
            # type: (ArgumentParser, str, Tuple[str, ...], Dict[str, Any]) -> None

            if params.get('raw', False) is True:
                final_names = (name,)  # type: Tuple[str, ...]
                del params['raw']

            else:
                if isinstance(names, str):
                    final_names = ('--{}'.format(name),)

                else:
                    final_names = ('-{}'.format(names[0]),) + tuple(['--{}'.format(n) for n in names[1:]])

            parser.add_argument(*final_names, **params)

        def _add_options(group_options, group_name=None):
            # type: (Any, Optional[str]) -> None

            group_parser = None  # type: Optional[Union[argparse.ArgumentParser, argparse._ArgumentGroup]]

            if group_name is None:
                group_parser = root_parser

            else:
                group_parser = root_parser.add_argument_group(group_name)

            assert group_parser is not None

            Configurable._for_each_option(partial(_add_option, group_parser), group_options)

        Configurable._for_each_option_group(_add_options, cls.options)

        return root_parser

    def _parse_args(self, args, **kwargs):
        # type: (Any, **Any) -> None

        """
        Parse command-line arguments. Uses :py:mod:`argparse` for the actual parsing.
        Updates module's configuration store with values returned by parser.

        :param list args: arguments passed to this module. Similar to what :py:data:`sys.argv` provides on
          program level.
        """

        self.debug('Loading configuration from command-line arguments')

        # construct the parser
        parser = self._create_args_parser(**kwargs)

        # parse the added args
        args = parser.parse_args(args)

        # add the parsed args to options
        def _inject_value(name, names, params):
            # type: (str, Tuple[str, ...], Dict[str, Any]) -> None

            # pylint: disable=unused-argument

            dest = params.get('dest', name.replace('-', '_'))

            value = getattr(args, dest)

            # if the option was not specified, skip it
            if value is None and name in self._config:
                return

            # do not replace config options with default command line values
            if name in self._config and self._config[name] is not None:
                # if default parameter used
                if 'default' in params and value == params['default']:
                    return

                # with action store_true, the default is False
                if params.get('action', '') == 'store_true' and value is False:
                    return

                # with action store_false, the default is True
                if params.get('action', '') == 'store_false' and value is True:
                    return

            self._config[name] = value
            self.debug("Option '{}' set to '{}' by command-line".format(name, value))

        def _inject_values(options, **kwargs):
            # type: (Any, **Any) -> None

            # pylint: disable=unused-argument

            Configurable._for_each_option(_inject_value, options)

        Configurable._for_each_option_group(_inject_values, self.options)

    def parse_config(self):
        # type: () -> None

        """
        Public entry point to configuration parsing. Child classes must implement this
        method, e.g. by calling :py:meth:`gluetool.glue.Configurable._parse_config` which
        requires list of paths.
        """

        # E.g. self._parse_config(<list of possible configuration files>)

        raise NotImplementedError('Implement this method to enable the actual parsing')

    def parse_args(self, args):
        # type: (List[str]) -> None

        """
        Public entry point to argument parsing. Child classes must implement this method,
        e.g. by calling :py:meth:`gluetool.glue.Configurable._parse_args` which makes use
        of additional :py:class:`argparse.ArgumentParser` options.
        """

        # E.g. self._parse_args(args, description=...)

        raise NotImplementedError('Implement this method to enable the actual parsing')

    def check_required_options(self):
        # type: () -> None

        if not self.required_options:
            self.debug('skipping checking of required options')
            return

        for name in self.required_options:
            if name not in self._config or not self._config[name]:
                raise GlueError("Missing required '{}' option".format(name))

    # `option()` returns two different types based on the number of positional arguments:
    #
    #   - 1 argument => returns just a single value (Any)
    #   - more than 1 argument => returns a tuple of values
    #
    # `mypy` supports typechecking of such function by using @overload decorators, describing
    # each variant, followed by the actual body of the function. We just need to silence pylint
    # and flake since we're re-defining the method - these checkes does not understand @overload.

    # pylint: disable=function-redefined

    @overload
    def option(self, name):
        # type: (str) -> Any

        pass

    @overload  # noqa
    def option(self, *names):
        # type: (*str) -> List[Any]

        pass

    def option(self, *names):  # type: ignore  # noqa
        """
        Return values of given options from module's configuration store.

        :param str names: names of requested options.
        :returns: either a value or ``None`` if such option does not exist. When multiple options are requested,
            a tuple of their values is returned, for a single option its value is **not** wrapped by a tuple.
        """

        if not names:
            raise GlueError('Specify at least one option')

        values = tuple(
            self._config.get(name, None) for name in names
        )

        return values[0] if len(values) == 1 else values

    @property
    def dryrun_level(self):
        # type: () -> int

        """
        Return current dry-run level. This must be implemented by class descendants
        because each one finds the necessary information in different places.
        """

        raise NotImplementedError()

    @property
    def dryrun_enabled(self):
        # type: () -> bool

        """
        ``True`` if dry-run level is enabled, on any level.
        """

        return self.dryrun_level != DryRunLevels.DEFAULT

    def _dryrun_allows(self, threshold, msg):
        # type: (int, str) -> bool

        """
        Check whether current dry-run level allows an action. If the current dry-run level
        is equal of higher than ``threshold``, then the action is not allowed.

        E.g. when action's ``threshold`` is :py:attr:`DryRunLevels.ISOLATED`, and the current
        level is :py:attr:`DryRunLevels.DRY`, the action is allowed.

        :param DryRunLevels threshold: Dry-run level the action is not allowed.
        :param str msg: Message logged (as a warning) when the action is deemed not allowed.
        :returns: ``True`` when action is allowed, ``False`` otherwise.
        """

        if self.dryrun_level >= threshold:
            self.warn('{} is not allowed by current dry-run level'.format(msg))
            return False

        return True

    def dryrun_allows(self, msg):
        # type: (str) -> bool

        """
        Checks whether current dry-run level allows an action which is disallowed on
        :py:attr:`DryRunLevels.DRY` level.

        See :py:meth:`Configurable._dryrun_allows` for detailed description.
        """

        return self._dryrun_allows(DryRunLevels.DRY, msg)

    def isolatedrun_allows(self, msg):
        # type: (str) -> bool

        """
        Checks whether current dry-run level allows an action which is disallowed on
        :py:attr:`DryRunLevels.ISOLATED` level.
        """

        return self._dryrun_allows(DryRunLevels.ISOLATED, msg)

    def check_dryrun(self):
        # type: () -> None

        """
        Checks whether this object supports current dry-run level.
        """

        if not self.dryrun_enabled:
            return

        if self.dryrun_level > self.supported_dryrun_level:
            dryrun_level_name = self.dryrun_level.name  # type: ignore  # `dryrun_level` is not pure int but Enum
            raise GlueError("Module '{}' does not support current dry-run level of '{}'".format(self.unique_name,
                                                                                                dryrun_level_name))

    @property
    def eval_context(self):
        # type: () -> Dict[str, Any]

        """
        Return "evaluation context" - a dictionary of variable names (usually in uppercase)
        and their values, which is supposed to be used in various "evaluate *this*" operations
        like rendering of templates.

        To provide nice and readable documentation of variables, returned by a module's ``eval_context``
        property, assign a dictionary, describing these variables, to a local variable named
        ``__content__``:

        .. code-block:: python

           ...

           @property
           def eval_context(self):
               __content__ = {
                   'FOO': 'This is an important variable, extracted from clouds.'
                }

                return {
                    'FOO': 42
                }

        ``gluetool`` core will extract this information and will use it to generate different help
        texts like your module's help or a list of all known context variables.

        :rtype: dict
        """

        return {}


class Module(Configurable):
    """
    Base class of all ``gluetool`` modules.

    :param gluetool.glue.Glue glue: ``Glue`` instance owning the module.

    :ivar gluetool.glue.Glue glue: ``Glue`` instance owning the module.
    :ivar dict _config: internal configuration store. Values of all module options
      are stored here, regardless of them being set on command-line or in the
      configuration file.
    :ivar dict _overloaded_shared_functions: If a shared function added by this module
        overloades an older function of the same name, registered by a previous module,
        the overloaded one is added into this dictionary. The module can then call this
        saved function - using :py:meth:`overloaded_shared` - to implement a "chain" of
        shared functions, when one calls another, implementing the same operation.
    """

    # logging type stubs
    verbose = None  # type: LoggingFunctionType
    debug = None  # type: LoggingFunctionType
    info = None  # type: LoggingFunctionType
    warn = None  # type: LoggingWarningFunctionType
    error = None  # type: LoggingFunctionType
    exception = None  # type: LoggingFunctionType

    description = None  # type: str
    """Short module description, displayed in ``gluetool``'s module listing."""

    shared_functions = []  # type: List[str]
    """Iterable of names of shared functions exported by the module."""

    def _paths_with_module(self, roots):
        # type: (List[str]) -> List[str]

        """
        Return paths cretaed by joining roots with module's unique name.

        :param list(str) roots: List of root directories.
        """

        assert self.unique_name is not None

        return [os.path.join(root, self.unique_name) for root in roots]

    def __init__(self, glue, name):
        # type: (Glue, str) -> None

        super(Module, self).__init__()

        # we need to save the unique name in case there are more aliases available
        self.unique_name = name

        self.glue = glue

        # initialize logging helpers
        self.logger = ModuleAdapter(glue.logger, self)
        self.logger.connect(self)

        # initialize data path if exists, else it will be None
        self.data_path = None

        for path in self._paths_with_module(self.glue.module_data_paths):
            if not os.path.exists(path):
                continue

            self.data_path = path
            self.debug('data file is {}'.format(path))
            break

        else:
            self.debug('no data file found')

        self._overloaded_shared_functions = {}  # type: Dict[str, SharedType]

    @property
    def dryrun_level(self):
        # type: () -> int

        return self.glue.dryrun_level

    def parse_config(self):
        # type: () -> None

        self._parse_config(self._paths_with_module(self.glue.module_config_paths))

    def _generate_shared_functions_help(self):
        # type: () -> str

        """
        Generate help for shared functions provided by the module.

        :returns: Formatted help, describing module's shared functions.
        """

        if not self.shared_functions:
            return ''

        from .help import functions_help

        functions = []

        for name in self.shared_functions:
            if not hasattr(self, name):
                raise GlueError("No such shared function '{}' of module '{}'".format(name, self.unique_name))

            functions.append((name, getattr(self, name)))

        return jinja2.Template(trim_docstring("""
        {{ '** Shared functions **' | style(fg='yellow') }}

        {{ FUNCTIONS }}
        """)).render(FUNCTIONS=functions_help(functions)).encode('ascii')

    def parse_args(self, args):
        # type: (Any) -> None

        epilog = [
            '' if self.options_note is None else docstring_to_help(self.options_note),
            self._generate_shared_functions_help(),
            eval_context_help(self)
        ]

        self._parse_args(args,
                         usage='{} [options]'.format(Colors.style(self.unique_name, fg='cyan')),
                         description=docstring_to_help(self.__doc__ or ''),
                         epilog='\n'.join(epilog).strip(),
                         formatter_class=LineWrapRawTextHelpFormatter)

    def destroy(self, failure=None):
        # type: (Optional[Failure]) -> None

        # pylint: disable-msg=no-self-use,unused-argument
        """
        Here should go any code that needs to be run on exit, like job cleanup etc.

        :param gluetool.glue.Failure failure: if set, carries information about failure that made
          ``gluetool`` to destroy the whole session. Modules might want to take actions based
          on provided information, e.g. send different notifications.
        """

        return None

    def add_shared(self):
        # type: () -> None

        """
        Register module's shared functions with Glue, to allow other modules
        to use them.
        """

        for funcname in self.shared_functions:
            if self.has_shared(funcname):
                original_shared = self.get_shared(funcname)
                assert original_shared is not None

                self._overloaded_shared_functions[funcname] = original_shared

            self.glue.add_shared(funcname, self)

    def del_shared(self, funcname):
        # type: (str) -> None

        self.glue.del_shared(funcname)

    def has_shared(self, funcname):
        # type: (str) -> bool

        return self.glue.has_shared(funcname)

    def require_shared(self, *names, **kwargs):
        # type: (*str, **str) -> bool

        return self.glue.require_shared(*names, **kwargs)

    def get_shared(self, funcname):
        # type: (str) -> Optional[SharedType]

        return self.glue.get_shared(funcname)

    def execute(self):
        # type: () -> None

        # pylint: disable-msg=no-self-use
        """
        In this method, modules can perform any work they deemed necessary for
        completing their purpose. E.g. if the module promises to run some tests,
        this is the place where the code belongs to.

        By default, this method does nothing. Reimplement as needed.
        """

    def sanity(self):
        # type: () -> None

        # pylint: disable-msg=no-self-use
        """
        In this method, modules can define additional checks before execution.

        Some examples:

        * Advanced checks on passed options
        * Check for additional requirements (tools, data, etc.)

        By default, this method does nothing. Reimplement as needed.
        """

    def shared(self, *args, **kwargs):
        # type: (*Any, **Any) -> Any

        return self.glue.shared(*args, **kwargs)

    def overloaded_shared(self, funcname, *args, **kwargs):
        # type: (str, *Any, **Any) -> Any

        """
        Call a shared function overloaded by the one provided by this module. This way,
        a module can give chance to other implementations of its action, e.g. to publish
        messages on a different message bus.
        """

        if funcname not in self._overloaded_shared_functions:
            return None

        self.debug("calling overloaded shared function '{}'".format(funcname))

        return self._overloaded_shared_functions[funcname](*args, **kwargs)

    def run_module(self, module, args=None):
        # type: (str, Optional[List[str]]) -> None

        self.glue.run_module(module, args or [])


class Glue(Configurable):
    # pylint: disable=too-many-public-methods

    """
    Main workhorse of the ``gluetool``. Manages modules, their instances and runs them as requested.

    :param gluetool.tool.Tool tool: If set, it's an ``gluetool``-like tool that created this instance.
        Some functionality may need it to gain access to bits like its command-name.
    :param gluetool.sentry.Sentry sentry: If set, it provides interface to Sentry.
    """
    # logging type stubs
    verbose = None  # type: LoggingFunctionType
    debug = None  # type: LoggingFunctionType
    info = None  # type: LoggingFunctionType
    warn = None  # type: LoggingWarningFunctionType
    error = None  # type: LoggingFunctionType
    exception = None  # type: LoggingFunctionType

    name = 'gluetool core'

    options = [
        ('Global options', {
            ('l', 'list-modules'): {
                'help': 'List all available modules. If a GROUP is set, limits list to the given module group.',
                'action': 'append',
                'nargs': '?',
                'metavar': 'GROUP',
                'const': True
            },
            ('L', 'list-shared'): {
                'help': 'List all available shared functions.',
                'action': 'store_true',
                'default': False
            },
            ('E', 'list-eval-context'): {
                'help': 'List all available variables provided by modules in their evaluation contexts.',
                'action': 'store_true',
                'default': False,
            },
            ('r', 'retries'): {
                'help': 'Number of retries',
                'type': int,
                'default': 0
            },
            ('V', 'version'): {
                'help': 'Print version',
                'action': 'store_true'
            },
            'no-sentry-exceptions': {
                'help': 'List of exception names, which are not reported to Sentry (Default: none)',
                'action': 'append',
                'default': []
            }
        }),
        ('Output control', {
            ('c', 'colors'): {
                'help': 'Colorize logging on the terminal',
                'action': 'store_true'
            },
            ('d', 'debug'): {
                'help': 'Log debugging messages to the terminal (WARNING: very verbose!).',
                'action': 'store_true'
            },
            ('i', 'info'): {
                'help': 'Print command-line that would re-run the gluetool session',
                'action': 'store_true'
            },
            ('j', 'json-file'): {
                'help': """
                        If set, all log messages (including ``VERBOSE``) are stored in this file
                        in a form of JSON structures (default: %(default)s).
                        """,
                'default': None
            },
            ('o', 'debug-file', 'output'): {
                'help': 'Log messages with at least ``DEBUG`` level are sent to this file.'
            },
            'verbose-file': {
                'help': 'Log messages with ``VERBOSE`` level sent to this file.',
                'default': None
            },
            ('p', 'pid'): {
                'help': 'Log PID of gluetool process',
                'action': 'store_true'
            },
            ('q', 'quiet'): {
                'help': 'Silence info messages',
                'action': 'store_true'
            },
            'show-traceback': {
                'help': """
                        Display exception tracebacks on terminal (besides the debug file when ``--debug-file``
                        is used) (default: %(default)s).
                        """,
                'action': 'store_true',
                'default': False
            },
            ('v', 'verbose'): {
                'help': 'Log **all** messages to the terminal (WARNING: even more verbose than ``-d``!).',
                'action': 'store_true'
            }
        }),
        ('Directories', {
            'module-path': {
                'help': 'Specify directory with modules.',
                'metavar': 'DIR',
                'action': 'append',
                'default': []
            },
            'module-data-path': {
                'help': 'Specify directory with module data files.',
                'metavar': 'DIR',
                'action': 'append',
                'default': []
            },
            'module-config-path': {
                'help': 'Specify directory with module configuration files.',
                'metavar': 'DIR',
                'action': 'append',
                'default': []
            }
        }),
        ('Dry run options', {
            'dry-run': {
                'help': 'Modules that support this option will make no changes to the outside world.',
                'action': 'store_true',
                'default': False
            },
            'isolated-run': {
                'help': 'Modules that support this option will not interact with the outside world.',
                'action': 'store_true',
                'default': False
            }
        }),
        {
            'pipeline': {
                'raw': True,
                'help': 'List of modules and their options, passed after gluetool options.',
                'nargs': argparse.REMAINDER
            }
        }
    ]

    @property
    def module_paths(self):
        # type: () -> List[str]

        """
        List of paths in which modules reside.
        """

        from .utils import normalize_path_option
        return normalize_path_option(self.option('module-path')) or DEFAULT_MODULE_PATHS

    @property
    def module_data_paths(self):
        # type: () -> List[str]

        """
        List of paths in which module data files reside.
        """

        from .utils import normalize_path_option
        return normalize_path_option(self.option('module-data-path')) or [DEFAULT_DATA_PATH]

    @property
    def module_config_paths(self):
        # type: () -> List[str]

        """
        List of paths in which module config files reside.
        """

        from .utils import normalize_path_option
        return normalize_path_option(self.option('module-config-path')) or DEFAULT_MODULE_CONFIG_PATHS

    # pylint: disable=method-hidden
    def sentry_submit_exception(self, *args, **kwargs):
        # type: (*Any, **Any) -> None

        """
        Submits exceptions to the Sentry server. Does nothing by default, unless this instance
        is initialized with a :py:class:`gluetool.sentry.Sentry` instance which actually does
        the job.

        See :py:meth:`gluetool.sentry.Sentry.submit_exception`.
        """

    # pylint: disable=method-hidden
    def sentry_submit_warning(self, *args, **kwargs):
        # type: (*Any, **Any) -> None

        """
        Submits warnings to the Sentry server. Does nothing by default, unless this instance
        is initialized with a :py:class:`gluetool.sentry.Sentry` instance which actually does
        the job.

        See :py:meth:`gluetool.sentry.Sentry.submit_warning`.
        """

    def _add_shared(self, funcname, module, func):
        # type: (str, Configurable, SharedType) -> None

        """
        Register a shared function. Overwrite previously registered function
        with the same name, if there was any such.

        This is a helper method for easier testability. It is not a part of public API of this class.

        :param str funcname: Name of the shared function.
        :param gluetool.glue.Module module: Module instance providing the shared function.
        :param callable func: Shared function.
        """

        self.debug("registering shared function '{}' of module '{}'".format(funcname, module.unique_name))

        self.shared_functions[funcname] = (module, func)

    def add_shared(self, funcname, module):
        # type: (str, Module) -> None

        """
        Register a shared function. Overwrite previously registered function
        with the same name, if there was any such.

        :param str funcname: Name of the shared function.
        :param gluetool.glue.Module module: Module instance providing the shared function.
        """

        if not hasattr(module, funcname):
            raise GlueError("No such shared function '{}' of module '{}'".format(funcname, module.name))

        self._add_shared(funcname, module, getattr(module, funcname))

    # delete a shared function if exists
    def del_shared(self, funcname):
        # type: (str) -> None

        if funcname not in self.shared_functions:
            return

        del self.shared_functions[funcname]

    def has_shared(self, funcname):
        # type: (str) -> bool

        return funcname in self.shared_functions

    def require_shared(self, *names, **kwargs):
        # type: (*str, **str) -> bool

        warn_only = kwargs.get('warn_only', False)

        def _check(name):
            # type: (str) -> bool

            if self.has_shared(name):
                return True

            # pylint: disable=line-too-long
            msg = "Shared function '{}' is required. See `gluetool -L` to find out which module provides it.".format(name)  # Ignore PEP8Bear

            if warn_only is True:
                self.warn(msg, sentry=True)
                return False

            raise GlueError(msg)

        return all([_check(name) for name in names])

    def get_shared(self, funcname):
        # type: (str) -> Optional[SharedType]

        if not self.has_shared(funcname):
            return None

        return self.shared_functions[funcname][1]

    def shared(self, funcname, *args, **kwargs):
        # type: (str, *Any, **Any) -> Any

        if funcname not in self.shared_functions:
            return None

        return self.shared_functions[funcname][1](*args, **kwargs)

    @property
    def eval_context(self):
        # type: () -> Dict[str, Any]

        """
        Returns "global" evaluation context - some variables that are nice to have in all contexts.
        """

        # pylint: disable=unused-variable
        __content__ = {  # noqa
            'ENV': 'Dictionary representing environment variables.'
        }

        return {
            'ENV': dict(os.environ)
        }

    def _eval_context_module_caller(self):
        # type: () -> Optional[Module]

        """
        Infere module instance calling the eval context shared function.

        :rtype: gluetool.glue.Module
        """

        stack = inspect.stack()
        log_dict(self.verbose, 'stack', stack)

        # When being called as a regular shared function, the call stack layout should be
        # as follows:
        #
        # Glue._eval_context_module_caller - this helper method, caller of inspect.stack()
        # Glue._eval_context - our caller, the actual shared function body
        # Glue.shared - shared function call dispatcher of Glue class, calls Glue._eval_context
        # Module.shared - shared function call dispatcher of Module class, calls Glue.shared internally

        # pylint: disable=too-many-boolean-expressions
        if len(stack) < 4 \
           or stack[0][3] != '_eval_context_module_caller' \
           or stack[1][3] != '_eval_context' \
           or stack[2][3] != 'shared' \
           or stack[3][3] != 'shared' \
           or 'self' not in stack[3][0].f_locals:
            self.warn('Cannot infer calling module of eval_context')
            return None

        return cast(Module, stack[3][0].f_locals['self'])

    def _eval_context(self):
        # type: () -> Dict[str, Any]

        """
        Gather contexts of all modules in a pipeline and merge them together.

        **Always** returns a unique dictionary object, therefore it is safe for caller
        to update it. The return value is not cached in any way, therefore the change
        if its content won't affect future callers.

        Provided as a shared function, registered by the Glue instance itself.

        :rtype: dict
        """

        self.debug('gather pipeline eval context')

        context = {
            'MODULE': self._eval_context_module_caller()
        }

        # first "module" is this instance - it provides eval_context as well.
        for module in [self] + self._module_instances:  # type: ignore  # types are compatible
            # cache the context for logging
            module_context = module.eval_context

            log_dict(module.verbose, 'eval context', module_context)

            context.update(module_context)

        return context

    #
    # Module loading
    #
    def _check_module_file(self, mfile):
        # type: (str) -> bool

        """
        Make sure the file looks like a ``gluetool`` module:

        - can be processed by Python parser,
        - imports :py:class:`gluetool.glue.Glue` and :py:class:`gluetool.glue.Module`,
        - contains child class of :py:class:`gluetool.glue.Module`.

        :param str mfile: path to a file.
        :returns: ``True`` if file contains ``gluetool`` module, ``False`` otherwise.
        :raises gluetool.glue.GlueError: when it's not possible to finish the check.
        """

        self.debug("check possible module file '{}'".format(mfile))

        try:
            with open(mfile) as f:
                node = ast.parse(f.read())

            # check for gluetool import
            def imports_gluetool(item):
                # type: (Any) -> bool

                """
                Return ``True`` if item is an ``import`` statement, and imports ``gluetool``.
                """

                class_name = item.__class__.__name__

                is_import = class_name == 'Import' and item.names[0].name == 'gluetool'
                is_import_from = class_name == 'ImportFrom' and item.module == 'gluetool'

                return cast(bool, is_import or is_import_from)

            if not any((imports_gluetool(item) for item in node.__dict__['body'])):
                self.debug("  no 'import gluetool' found")
                return False

            # check for gluetool.Module class definition
            def has_module_class(item):
                # type: (Any) -> bool

                """
                Return ``True`` if item is a class definition, and any of the base classes
                is gluetool.glue.Module.
                """

                if item.__class__.__name__ != 'ClassDef':
                    return False

                for base in item.bases:
                    if (hasattr(base, 'id') and base.id == 'Module') \
                            or (hasattr(base, 'attr') and base.attr == 'Module'):
                        return True

                return False

            if not any((has_module_class(item) for item in node.__dict__['body'])):
                self.debug('  no child of gluetool.Module found')
                return False

            return True

        # pylint: disable=broad-except
        except Exception as e:
            raise GlueError("Unable to check check module file '{}': {}".format(mfile, str(e)))

    def _import_module(self, import_name, filename):
        # type: (str, str) -> Any

        """
        Attempt to import a Python module from a file.

        :param str import_name: name assigned to the imported module.
        :param str filepath: path to a file.
        :returns: imported Python module.
        :raises gluetool.glue.GlueError: when import failed.
        """

        self.debug("try to import module '{}' from file '{}'".format(import_name, filename))

        try:
            return imp.load_source(import_name, filename)

        # pylint: disable=broad-except
        except Exception as e:
            raise GlueError("Unable to import module '{}' from '{}': {}".format(import_name, filename, str(e)))

    def _load_python_module(self, group, module_name, filepath):
        # type: (str, str, str) -> Any

        """
        Load Python module from a file, if it contains ``gluetool`` modules. If the
        file does not look like it contains ``gluetool`` modules, or when it's not
        possible to import the Python module successfully, method simply warns
        user and ignores the file.

        :param str import_name: name assigned to the imported module.
        :param str filepath: path to a file.
        :returns: loaded Python module.
        :raises gluetool.glue.GlueError: when import failed.
        """

        # Check content of the file, look for Glue and Module stuff
        try:
            if not self._check_module_file(filepath):
                return

        except GlueError as e:
            self.warn("ignoring file '{}': {}".format(module_name, e.message))
            return

        # Try to import file as a Python module
        import_name = 'gluetool.glue.{}-{}'.format(group, module_name)

        try:
            module = self._import_module(import_name, filepath)

        except GlueError as e:
            self.warn("ignoring module '{}': {}".format(module_name, e.message))
            return

        return module

    def _load_gluetool_modules(self, group, module_name, filepath):
        # type: (str, str, str) -> List[Tuple[str, Type[Module]]]

        """
        Load ``gluetool`` modules from a file. Method attempts to import the file
        as a Python module, and then checks its content and adds all `gluetool`
        modules to internal module registry.

        :param str group: module group.
        :param str module_name: name assigned to the imported Python module.
        :param str filepath: path to a file.
        :rtype: [(module_group, module_class), ...]
        :returns: list of loaded ``gluetool`` modules
        """

        module = self._load_python_module(group, module_name, filepath)

        loaded_modules = []  # type: List[Tuple[str, Type[Any]]]

        # Look for gluetool modules in imported stuff, and add them to our module registry
        for name in dir(module):
            cls = getattr(module, name)

            if not isinstance(cls, type) or not issubclass(cls, Module) or cls == Module:
                continue

            assert issubclass(cls, Module)

            module_names = getattr(cls, 'name', None)  # type: Union[None, List[str], Tuple[str]]

            if not module_names:
                raise GlueError('No name specified by module class {}:{}'.format(filepath, cls.__name__))

            def add_module(mname, cls):
                # type: (str, Type[Module]) -> None

                if mname in self.modules:
                    raise GlueError("Name '{}' of module '{}' from '{}' is a duplicate module name".format(
                        mname, cls.__name__, filepath))

                self.debug("found module '{}', group '{}', in module '{}' from '{}'".format(
                    mname, group, module_name, filepath))

                self.modules[mname] = {
                    'class': cls,
                    'description': cls.description,
                    'group': group
                }

                loaded_modules.append((group, cls))

            # if name is a list, add more aliases to the same module
            if isinstance(module_names, (list, tuple)):
                for mname in module_names:
                    add_module(mname, cls)

            else:
                add_module(module_names, cls)

        return loaded_modules

    def _load_module_path(self, ppath):
        # type: (str) -> None

        """
        Search and load ``gluetool`` modules from a directory.

        In essence, it scans every file with ``.py`` suffix, and searches for
        classes derived from :py:class:`gluetool.glue.Module`.

        :param str ppath: directory to search for `gluetool` modules.
        """

        for root, _, files in os.walk(ppath):
            for filename in sorted(files):
                if not filename.endswith('.py'):
                    continue

                group = root.replace(ppath + '/', '')
                module_name, _ = os.path.splitext(filename)
                module_file = os.path.join(root, filename)

                self._load_gluetool_modules(group, module_name, module_file)

    def load_modules(self):
        # type: () -> None

        """
        Load all available `gluetool` modules.
        """

        log_dict(self.debug, 'loading modules from these paths', self.module_paths)

        for path in self.module_paths:
            self._load_module_path(path)

    def __init__(self, tool=None, sentry=None):
        # type: (Optional[Any], Optional[Any]) -> None

        # Initialize logging methods before doing anything else.
        # Right now, we don't know the desired log level, or if
        # output file is in play, just get simple logger before
        # the actual configuration is known.
        self._sentry = sentry

        if sentry is not None:
            self.sentry_submit_exception = sentry.submit_exception  # type: ignore
            self.sentry_submit_warning = sentry.submit_warning  # type: ignore

        logger = Logging.create_logger(sentry=sentry, sentry_submit_warning=self.sentry_submit_warning)

        self.logger = ContextAdapter(logger)
        self.logger.connect(self)

        super(Glue, self).__init__()

        self.tool = tool

        self._dryrun_level = DryRunLevels.DEFAULT

        self.current_module = None  # type: Optional[Module]

        # module types dictionary
        self.modules = {}  # type: Dict[str, Dict[str, Any]]

        # Materialized pipeline
        self._module_instances = []  # type: List[Configurable]

        #: Shared function registry.
        #: funcname: (module, fn)
        self.shared_functions = {}  # type: Dict[str, Tuple[Configurable, SharedType]]

        self._add_shared('eval_context', self, self._eval_context)

    # pylint: disable=arguments-differ
    def parse_config(self, paths):  # type: ignore  # signature differs on purpose
        # type: (List[str]) -> None

        self._parse_config(paths)

    def parse_args(self, args):
        # type: (Any) -> None

        module_dirs = '\n'.join(['        - {}'.format(directory) for directory in DEFAULT_MODULE_PATHS])
        data_dirs = '\n'.join(['        - {}'.format(directory) for directory in [DEFAULT_DATA_PATH]])
        module_config_dirs = '\n'.join(['        - {}'.format(directory) for directory in DEFAULT_MODULE_CONFIG_PATHS])

        epilog = trim_docstring("""
        Default paths:

            * modules are searched under (--module-path):
        {}

            * module data files are searched under (--module-data-path):
        {}

            * module configuration files are searched under (--module-config-path):
        {}

        """).format(module_dirs, data_dirs, module_config_dirs)

        self._parse_args(args,
                         usage='%(prog)s [options] [module1 [module1 options] module2 [module2 options] ...]',
                         epilog=epilog,
                         formatter_class=LineWrapRawTextHelpFormatter)

        # re-create logger - now we have all necessary configuration
        if self.option('verbose'):
            level = VERBOSE

        elif self.option('debug'):
            level = logging.DEBUG

        elif self.option('quiet'):
            level = logging.WARNING

        else:
            level = logging.INFO

        from .utils import normalize_bool_option
        switch_colors(normalize_bool_option(self.option('colors')))

        debug_file = self.option('debug-file')
        verbose_file = self.option('verbose-file')
        json_file = self.option('json-file')

        if debug_file and not verbose_file:
            verbose_file = '{}.verbose'.format(debug_file)

        show_traceback = normalize_bool_option(self.option('show-traceback'))

        logger = Logging.create_logger(level=level,
                                       debug_file=debug_file,
                                       verbose_file=verbose_file,
                                       json_file=json_file,
                                       sentry=self._sentry,
                                       show_traceback=show_traceback)

        self.logger = ContextAdapter(logger)
        self.logger.connect(self)

        if level == logging.DEBUG and not verbose_file:
            # pylint: disable=line-too-long
            self.warn('Debug output enabled but no verbose destination set.')
            self.warn('Either use ``-v`` to display verbose messages on screen, or ``--verbose-file`` to store them in a file.')

        if self.option('isolated-run'):
            self._dryrun_level = DryRunLevels.ISOLATED

        elif self.option('dry-run'):
            self._dryrun_level = DryRunLevels.DRY

    @property
    def dryrun_level(self):
        # type: () -> int

        return self._dryrun_level

    def _for_each_module(self, modules, callback, *args, **kwargs):
        # type: (Iterable[Module], Callable[..., None], *Any, **Any) -> None

        for module in modules:
            self.current_module = module

            callback(module, *args, **kwargs)

    def destroy_modules(self, failure=None):
        # type: (Optional[Failure]) -> Any

        if not self._module_instances:
            return

        # we will destroy modules in reverse order, which makes more sense
        self.debug('destroying all modules in reverse order')

        def _destroy(module):
            # type: (Module) -> None

            try:
                module.debug('destroying myself')
                module.destroy(failure=failure)

            # pylint: disable=broad-except
            except Exception as exc:
                destroy_failure = Failure(module=module, exc_info=sys.exc_info())

                msg = "Exception raised while destroying module '{}': {}".format(module.unique_name, exc.message)
                self.error(msg, exc_info=destroy_failure.exc_info)
                self.sentry_submit_exception(destroy_failure, logger=self.logger)

        self._for_each_module(reversed(cast(Sequence[Module], self._module_instances)), _destroy)

        self.current_module = None
        self._module_instances = []

    def init_module(self, module_name, actual_module_name=None):
        # type: (str, Optional[str]) -> Module

        """
        Given a name of the module, create its instance and give it a name.

        :param str module_name: Name under which will be the module instance known.
        :param str actual_module_name: Name of the module to instantiate. It does not have to match
            ``module_name`` - ``actual_module_name`` refers to the list of known ``gluetool`` modules
            while ``module_name`` is basically an arbitrary name new instance calls itself. If it's
            not set, which is the most common situation, it defaults to ``module_name``.
        :returns: A :py:class:`Module` instance.
        """

        actual_module_name = actual_module_name or module_name
        klass = self.modules[actual_module_name]['class']  # type: Type[Module]

        return klass(self, module_name)

    def run_modules(self, pipeline_desc, register=False):
        # type: (List[PipelineStep], Optional[bool]) -> None

        """
        Run a pipeline, consisting of multiple modules.

        :param list(PipelineStep) pipeline_desc: List of pipeline steps.
        :param bool register: If ``True``, module instance is added to a list of modules in this
            ``Glue`` instance, and it will be collected when :py:meth:`destroy_modules` gets called.
        """

        log_dict(self.debug, 'running a pipeline', pipeline_desc)

        modules = []

        for step in pipeline_desc:
            module = self.init_module(step.module, actual_module_name=step.actual_module)
            modules.append(module)

            if register is True:
                self._module_instances.append(module)

            module.parse_config()
            module.parse_args(step.argv)
            module.check_dryrun()

        def _sanity(module):
            # type: (Module) -> None

            module.sanity()
            module.check_required_options()

        self._for_each_module(modules, _sanity)

        def _execute(module):
            # type: (Module) -> None

            # We want to register module's shared function no matter how its ``execute``
            # finished or crashed. We could use ``try``-``finally`` and call add_shared there
            # but should there be an exception under ``try`` *and* should there be another
            # one under ``finally``, the first one would be lost, replaced by the later.
            # And we cannot guarantee exception-less ``add_shared``, it may have been replaced
            # by module's developer. Therefore resorting to being more verbose.
            try:
                module.execute()

            # pylint: disable=broad-except,unused-variable
            except Exception as exc:  # noqa
                # In case ``add_shared`` crashes, the original exception, raised in ``execute``,
                # is not lost since it's already captured as ``exc``. We will re-raise it when we're
                # done with ``add_shared``, and should ``add_shared`` crash, ``exc`` would be added
                # to a chain anyway.
                module.add_shared()
                raise exc.__class__, exc, sys.exc_info()[2]

            else:
                module.add_shared()

        self._for_each_module(modules, _execute)

        self.current_module = None

    def run_module(self, module_name, module_argv=None, actual_module_name=None, register=False):
        # type: (str, Optional[List[str]], Optional[str], Optional[bool]) -> Any

        """
        Syntax sugar for :py:meth:`run_modules`, in the case you want to run just a one-shot module.

        :param str module_name: Name under which will be the module instance known.
        :param list(str) module_argv: Arguments of the module.
        :param str actual_module_name: Name of the module to instantiate. It does not have to match
            ``module_name`` - ``actual_module_name`` refers to the list of known ``gluetool`` modules
            while ``module_name`` is basically an arbitrary name new instance calls itself. If it's
            not set, which is the most common situation, it defaults to ``module_name``.
        :param bool register: If ``True``, module instance is added to a list of modules
            in this ``Glue`` instance, and it will be collected when :py:meth:`destroy_modules` gets
            called.
        """

        step = PipelineStep(module_name, actual_module=actual_module_name, argv=module_argv)

        return self.run_modules([step], register=register)

    def module_list(self):
        # type: () -> List[str]

        return sorted(self.modules)

    def module_list_usage(self, groups):
        # type: (List[str]) -> str

        """ Returns a string with modules description """

        if groups:
            usage = [
                'Available modules in {} group(s)'.format(', '.join(groups))
            ]
        else:
            usage = [
                'Available modules'
            ]

        # get module list
        plist = self.module_group_list()
        if not plist:
            usage.append('')
            usage.append('  -- no modules found --')
        else:
            for group in sorted(plist):
                # skip groups that are not in the list
                # note that groups is [] if all groups should be shown
                if groups and group not in groups:
                    continue
                usage.append('')
                usage.append('%-2s%s' % (' ', group))
                for key, val in sorted(plist[group].iteritems()):
                    usage.append('%-4s%-32s %s' % ('', key, val))

        return '\n'.join(usage)

    def module_group_list(self):
        # type: () -> Dict[str, Dict[str, str]]

        """ Returns a dictionary of groups of modules with description """
        module_groups = {}  # type: Dict[str, Dict[str, str]]

        for module in self.module_list():
            group = self.modules[module]['group']
            try:
                module_groups[group].update({
                    module: self.modules[module]['description']
                })
            except KeyError:
                module_groups[group] = {
                    module: self.modules[module]['description']
                }
        return module_groups
