from past.builtins import basestring
from builtins import object
from itertools import chain
from collections import defaultdict
import sys
import os.path as op
import types
from copy import copy
from logging import getLogger
from nipype.interfaces.utility import IdentityInterface
from arcana.exceptions import (
    ArcanaMissingInputError, ArcanaNoConverterError, ArcanaDesignError,
    ArcanaCantPickleStudyError, ArcanaUsageError,
    ArcanaMissingDataException, ArcanaNameError,
    ArcanaOutputNotProducedException, ArcanaInputMissingMatchError,
    ArcanaInputError)
from arcana.pipeline import Pipeline
from arcana.data import (
    BaseData, BaseInput, BaseInputSpec, InputFilesets, InputFields,
    BaseFileset)
from nipype.pipeline import engine as pe
from .parameter import Parameter, SwitchSpec
from arcana.repository.interfaces import RepositorySource
from arcana.repository import BasicRepo
from arcana.processor import SingleProc
from arcana.environment import StaticEnv
from arcana.utils import get_class_info

logger = getLogger('arcana')


class Study(object):
    """

    Parameters
    ----------
    name : str
        The name of the study.
    repository : Repository
        An Repository object that provides access to a DaRIS, XNAT or local file
        system
    processor : Processor
        A Processor to process the pipelines required to generate the
        requested derived filesets.
    inputs : Dict[str, InputFilesets | FilesetSpec | InputFields | FieldSpec] | List[InputFilesets | FilesetSpec | InputFields | FieldSpec]
        Either a list or a dictionary containing InputFilesets,
        InputFields, FilesetSpec, or FieldSpec objects, which specify the
        names of input filesets to the study, i.e. those that won't
        be generated by this study (although can be derived by the parent
        MultiStudy)
    environment : Environment
        An Environment within which to process the pipelines. Handles the
        version management + loading/unloading of software requirements
    parameters : List[Parameter] | Dict[str, (int|float|str)]
        Parameters that are passed to pipelines when they are constructed
        either as a dictionary of key-value pairs or as a list of
        'Parameter' objects. The name and dtype must match ParamSpecs in
        the _param_spec class attribute (see 'add_param_specs').
    subject_ids : List[(int|str)]
        List of subject IDs to restrict the analysis to
    visit_ids : List[(int|str)]
        List of visit IDs to restrict the analysis to
    enforce_inputs : bool (default: True)
        Whether to check the inputs to see if any acquired filesets
        are missing
    fill_tree : bool
        Whether to fill the tree of the destination repository with the
        provided subject and/or visit IDs. Intended to be used when the
        destination repository doesn't contain any of the the input
        filesets/fields (which are stored in external repositories) and
        so the sessions will need to be created in the destination
        repository.


    Class Attrs
    -----------
    add_data_specs : List[FilesetSpec|FieldSpec]
        Adds specs to the '_data_specs' class attribute,
        which is a dictionary that maps the names of filesets that are
        used and generated by the study to (Fileset|Field)Spec objects.
    add_param_specs : List[ParamSpec]
        Adds specs to the '_param_specs' class attribute,
        which is a dictionary that maps the names of parameters that are
        provided to pipelines in the study
    """

    _data_specs = {}
    _param_specs = {}

    implicit_cls_attrs = ['_data_specs', '_param_specs']

    SUBJECT_ID = 'subject_id'
    VISIT_ID = 'visit_id'
    ITERFIELDS = (SUBJECT_ID, VISIT_ID)
    FREQUENCIES = {
        'per_study': (),
        'per_subject': (SUBJECT_ID,),
        'per_visit': (VISIT_ID,),
        'per_session': (SUBJECT_ID, VISIT_ID)}

    def __init__(self, name, repository, processor, inputs,
                 environment=None, parameters=None, subject_ids=None,
                 visit_ids=None, enforce_inputs=True, fill_tree=False,
                 clear_caches=True):
        try:
            # This works for PY3 as the metaclass inserts it itself if
            # it isn't provided
            metaclass = type(self).__dict__['__metaclass__']
            if not issubclass(metaclass, StudyMetaClass):
                raise KeyError
        except KeyError:
            raise ArcanaUsageError(
                "Need to have StudyMetaClass (or a sub-class) as "
                "the metaclass of all classes derived from Study")
        if isinstance(repository, basestring):
            repository = BasicRepo(repository, depth=None)
        if isinstance(processor, basestring):
            processor = SingleProc(processor)
        if environment is None:
            environment = StaticEnv()
        self._name = name
        self._repository = repository
        self._processor = processor.bind(self)
        self._environment = environment
        self._inputs = {}
        self._subject_ids = (tuple(subject_ids)
                             if subject_ids is not None else None)
        self._visit_ids = tuple(visit_ids) if visit_ids is not None else None
        self._fill_tree = fill_tree
        # Initialise caches for data collection and pipeline objects
        if clear_caches:
            self.clear_caches()
        # Set parameters
        if parameters is None:
            parameters = {}
        elif not isinstance(parameters, dict):
            # Convert list of parameters into dictionary
            parameters = {o.name: o for o in parameters}
        self._parameters = {}
        for param_name, param in list(parameters.items()):
            if not isinstance(param, Parameter):
                param = Parameter(param_name, param)
            try:
                param_spec = self._param_specs[param_name]
            except KeyError:
                raise ArcanaNameError(
                    param_name,
                    "Provided parameter '{}' is not present in the "
                    "allowable parameters for {} classes ('{}')"
                    .format(param_name, type(self).__name__,
                            "', '".join(self.param_spec_names())))
            param_spec.check_valid(param, context=' {}(name={})'.format(
                type(self).__name__, name))
            self._parameters[param_name] = param
        # Convert inputs to a dictionary if passed in as a list/tuple
        if not isinstance(inputs, dict):
            inputs = {i.name: i for i in inputs}
        else:
            # Convert string patterns into Input objects
            for inpt_name, inpt in list(inputs.items()):
                if isinstance(inpt, basestring):
                    spec = self.data_spec(inpt_name)
                    if spec.is_fileset:
                        inpt = InputFilesets(inpt_name, pattern=inpt,
                                             is_regex=True)
                    else:
                        inpt = InputFields(inpt_name, pattern=inpt,
                                             dtype=spec.dtype,
                                             is_regex=True)
                    inputs[inpt_name] = inpt
        # Check validity of study inputs
        for inpt_name, inpt in inputs.items():
            try:
                spec = self.data_spec(inpt_name)
            except ArcanaNameError:
                raise ArcanaNameError(
                    inpt.name,
                    "Input name '{}' isn't in data specs of {} ('{}')"
                    .format(
                        inpt.name, self.__class__.__name__,
                        "', '".join(self._data_specs)))
            else:
                if spec.is_fileset:
                    if inpt.is_field:
                        raise ArcanaUsageError(
                            "Passed field ({}) as input to fileset spec"
                            " {}".format(inpt, spec))
                elif not inpt.is_field:
                    raise ArcanaUsageError(
                        "Passed fileset ({}) as input to field spec {}"
                        .format(inpt, spec))
        # "Bind" input selectors to the current study object, and attempt to
        # match with data in the repository
        input_errors = []
        with self.repository:
            if not self.subject_ids:
                raise ArcanaUsageError(
                    "No subject IDs provided and destination repository "
                    "is empty")
            if not self.visit_ids:
                raise ArcanaUsageError(
                    "No visit IDs provided and destination repository "
                    "is empty")
            for inpt_name, inpt in list(inputs.items()):
                try:
                    try:
                        self._inputs[inpt_name] = bound_inpt = inpt.bind(
                            self, spec_name=inpt_name)
                    except ArcanaInputMissingMatchError as e:
                        if not inpt.drop_if_missing:
                            raise e
                    else:
                        spec = self.data_spec(inpt_name)
                        if spec.is_fileset:
                            if spec.derived:
                                try:
                                    spec.format.converter_from(
                                        bound_inpt.format)
                                except ArcanaNoConverterError as e:
                                    e.msg += (
                                        ", which is requried to convert:\n{} "
                                        "to\n{}.".format(e, bound_inpt, spec))
                                    raise e
                            else:
                                if bound_inpt.format not in spec.valid_formats:
                                    raise ArcanaUsageError(
                                        "Cannot pass {} as an input to {} as "
                                        "it is not in one of the valid formats"
                                        " ('{}')".format(
                                            bound_inpt, spec,
                                            "', '".join(
                                                f.name
                                                for f in spec.valid_formats)))
                except ArcanaInputError as e:
                    # Collate errors across all inputs into a single error
                    # message
                    input_errors.append(e)
        if input_errors:
            raise ArcanaInputError('\n'.join(str(e) for e in input_errors))
        # Check remaining specs are optional or have default values
        for spec in self.data_specs():
            if spec.name not in self.input_names:
                if not spec.derived and spec.default is None:
                    # Emit a warning if an acquired fileset has not been
                    # provided for an "acquired fileset"
                    msg = (" acquired fileset '{}' was not given as"
                           " an input of {}.".format(spec.name, self))
                    if spec.optional:
                        logger.info('Optional' + msg)
                    else:
                        if enforce_inputs:
                            raise ArcanaMissingInputError(
                                'Non-optional' + msg + " Pipelines "
                                "depending on this fileset will not "
                                "run")

    def data(self, name, subject_ids=None, visit_ids=None, session_ids=None,
             generate=True, **kwargs):
        """
        Returns the Fileset(s) or Field(s) associated with the provided spec
        name(s), generating derived filesets as required. Multiple names in a
        list can be provided, to allow their workflows to be combined into a
        single workflow.

        Parameters
        ----------
        name : str | List[str]
            The name of the FilesetSpec|FieldSpec to retried the
            filesets for
        subject_id : str | None
            The subject ID of the data to return. If provided (including None
            values) the data will be return as a single item instead of a
            collection
        visit_id : str | None
            The visit ID of the data to return. If provided (including None
            values) the data will be return as a single item instead of a
            c ollection
        subject_ids : list[str]
            The subject IDs to include in the returned collection
        visit_ids : list[str]
            The visit IDs to include in the returned collection
        session_ids : list[str]
            The session IDs (i.e. 2-tuples of the form
            (<subject-id>, <visit-id>) to include in the returned collection
        generate : bool
            Whether to generate data that hasn't been generated yet or not

        Returns
        -------
        data : BaseItem | BaseCollection | list[BaseItem | BaseCollection]
            If 'subject_id' or 'visit_id' is provided then the data returned is
            a single Fileset or Field. Otherwise a collection of Filesets or
            Fields are returned. If muliple spec names are provided then a
            list of items or collections corresponding to each spec name.
        """
        if isinstance(name, basestring):
            single_name = True
            names = [name]
        else:
            names = name
            single_name = False
        single_item = 'subject_id' in kwargs or 'visit_id' in kwargs
        filter_items = (subject_ids, visit_ids, session_ids) != (None, None,
                                                                 None)
        specs = [self.spec(n) for n in names]
        if single_item:
            if filter_items:
                raise ArcanaUsageError(
                    "Cannot provide 'subject_id' and/or 'visit_id' in "
                    "combination with 'subject_ids', 'visit_ids' or "
                    "'session_ids'")
            subject_id = kwargs.pop('subject_id', None)
            visit_id = kwargs.pop('visit_id', None)
            iterators = set(chain(self.FREQUENCIES[s.frequency]
                                  for s in specs))
            if subject_id is not None and visit_id is not None:
                session_ids = [(subject_id, visit_id)]
            elif subject_id is not None:
                if self.VISIT_ID in iterators:
                    raise ArcanaUsageError(
                        "Non-None values for visit IDs need to be "
                        "provided to select a single item for each of '{}'"
                        .format("', '".join(names)))
                subject_ids = [subject_id]
            elif visit_id is not None:
                if self.SUBJECT_ID in iterators:
                    raise ArcanaUsageError(
                        "Non-None values for subject IDs need to be "
                        "provided to select a single item for each of '{}'"
                        .format("', '".join(names)))
                visit_ids = [visit_id]
            elif iterators:
                raise ArcanaUsageError(
                    "Non-None values for subject and/or visit IDs need to be "
                    "provided to select a single item for each of '{}'"
                    .format("', '".join(names)))
        if generate:
            # Work out which pipelines need to be run
            pipeline_getters = defaultdict(set)
            for spec in specs:
                if spec.derived or spec.derivable:  # Filter out Study inputs
                    # Add name of spec to set of required outputs
                    pipeline_getters[(spec.pipeline_getter,
                                      spec.pipeline_args)].add(spec.name)
            # Run required pipelines
            if pipeline_getters:
                kwargs = copy(kwargs)
                kwargs.update({'subject_ids': subject_ids,
                               'visit_ids': visit_ids,
                               'session_ids': session_ids})
                pipelines, required_outputs = zip(*(
                    (self.pipeline(getter, pipeline_args=args), req_outs)
                    for (getter, args), req_outs in pipeline_getters.items()))
                kwargs['required_outputs'] = required_outputs
                self.processor.run(*pipelines, **kwargs)
        # Find and return Item/Collection corresponding to requested spec
        # names
        all_data = []
        for name in names:
            spec = self.bound_spec(name)
            data = spec.collection
            if single_item:
                data = data.item(subject_id=subject_id, visit_id=visit_id)
            elif filter_items and spec.frequency != 'per_study':
                if subject_ids is None:
                    subject_ids = []
                if visit_ids is None:
                    visit_ids = []
                if session_ids is None:
                    session_ids = []
                if spec.frequency == 'per_session':
                    data = [d for d in data
                            if (d.subject_id in subject_ids or
                                d.visit_id in visit_ids or
                                d.session_id in session_ids)]
                elif spec.frequency == 'per_subject':
                    data = [d for d in data
                            if (d.subject_id in subject_ids or
                                d.subject_id in [s[0] for s in session_ids])]
                elif spec.frequency == 'per_visit':
                    data = [d for d in data
                            if (d.visit_id in visit_ids or
                                d.visit_id in [s[1] for s in session_ids])]
                if not data:
                    raise ArcanaUsageError(
                        "No matching data found (subject_ids={}, visit_ids={} "
                        ", session_ids={})"
                        .format(subject_ids, visit_ids, session_ids))
                data = spec.CollectionClass(spec.name, data)
            all_data.append(data)
        if single_name:
            all_data = all_data[0]
        return all_data

    def pipeline(self, getter_name, required_outputs=None, pipeline_args=None):
        """
        Returns a pipeline from a study by getting the method corresponding to
        the given name and checking that the required outputs are generated
        given the parameters of the study

        Parameters
        ----------
        getter_name : str
            Name of the method that constructs the pipeline
        required_outputs : list[str] | None
            The list of outputs that are expected of the pipeline
        pipeline_args : dict[str, *] | None
            Any arguments that should be passed to the method to produce the
            pipeline.
        """
        if pipeline_args is None:
            pipeline_args = ()
        elif isinstance(pipeline_args, dict):
            pipeline_args = tuple(pipeline_args.items())
        try:
            pipeline = self._pipelines_cache[(getter_name, pipeline_args)]
        except KeyError:
            try:
                getter = getattr(self, getter_name)
            except AttributeError:
                raise ArcanaDesignError(
                    "There is no pipeline constructor method named '{}' in "
                    "present in '{}' study".format(getter_name, self))
            self._pipeline_to_generate = getter_name
            try:
                pipeline = getter(**dict(pipeline_args))
            except ArcanaMissingDataException as e:
                e.msg += ("{}, which is required as an input when calling the "
                          "pipeline constructor method '{}' to create a "
                          "pipeline to produce '{}'".format(
                              e, getter_name,
                              "', '".join(required_outputs)))
                raise e
            finally:
                self._pipeline_to_generate = None
            if pipeline is None:
                raise ArcanaDesignError(
                    "'{}' pipeline constructor in {} is missing return "
                    "statement (should return a Pipeline object)".format(
                        getter_name, self))
            elif not isinstance(pipeline, Pipeline):
                raise ArcanaDesignError(
                    "'{}' pipeline constructor in {} doesn't return a Pipeline"
                    " object ({})".format(
                        getter_name, self, pipeline))
            # Check to see if the pipeline is equivalent to previously
            # generated pipelines (if two getter methods return equivalent
            # pipelines) and whether any outputs are to be generated twice
            # by different pipelines within the same workflow
            for prev_pipeline in self._pipelines_cache.values():
                if pipeline == prev_pipeline:
                    pipeline = prev_pipeline
                    break
                elif any(o in prev_pipeline.outputs for o in pipeline.outputs):
                    raise ArcanaDesignError(
                        "'{}' outputs are produced by more than one pipeline "
                        "({} and {})".format(
                            set(pipeline.output_names).intersection(
                                prev_pipeline.output_names),
                            prev_pipeline, pipeline))
            self._pipelines_cache[(getter_name, pipeline_args)] = pipeline
        if required_outputs is not None:
            # Check that the required outputs are created with the given
            # parameters
            missing_outputs = (set(required_outputs) -
                               set(pipeline.output_names))
            if missing_outputs:
                raise ArcanaOutputNotProducedException(
                    "Required output(s) '{}', will not be created by the '{}' "
                    "pipeline constructed by '{}' method in {} given the "
                    "missing study inputs:\n{}\n"
                    "and the provided switches:\n{}"
                    .format(
                        "', '".join(missing_outputs),
                        pipeline.name, getter_name, self,
                        "\n".join(self.missing_inputs),
                        '\n'.join('{}={}'.format(s.name, s.value)
                                  for s in self.switches)))
        return pipeline

    def __repr__(self):
        """String representation of the study"""
        return "{}(name='{}')".format(self.__class__.__name__,
                                      self.name)

    def __reduce__(self):
        """
        Control how study classes are pickled to allow some generated
        classes (those that don't define additional methods) to be
        generated
        """
        cls = type(self)
        module = sys.modules[cls.__module__]
        try:
            # Check whether the study class is generated or not by
            # seeing if it exists in its module
            if cls is not getattr(module, cls.__name__):
                raise AttributeError
        except AttributeError:
            cls_dct = {}
            for name, attr in list(cls.__dict__.items()):
                if isinstance(attr, types.FunctionType):
                    try:
                        if not attr.auto_added:
                            raise ArcanaCantPickleStudyError()
                    except (AttributeError, ArcanaCantPickleStudyError):
                        raise ArcanaCantPickleStudyError(
                            "Cannot pickle auto-generated study class "
                            "as it contains non-auto-added method "
                            "{}:{}".format(name, attr))
                elif name not in self.implicit_cls_attrs:
                    cls_dct[name] = attr
            pkld = (pickle_reconstructor,
                    (cls.__metaclass__, cls.__name__, cls.__bases__,
                     cls_dct), self.__dict__)
        else:
            # Use standard pickling if not a generated class
            pkld = object.__reduce__(self)
        return pkld

    @classmethod
    def static_menu(cls):
        def wrap_desc(desc, nchars, indent='\t\t'):
            lines = []
            while desc:
                if len(desc) > nchars:
                    n = desc[:nchars].rfind(' ')
                else:
                    n = nchars
                lines.append(desc[:n])
                desc = desc[(n + 1):]
            return '\n{}'.format(indent).join(lines)
        cls_name = '.'.join([cls.__module__, cls.__name__])
        menu = ("\n{} Menu \n".format(cls_name) +
                '-' * len(cls_name) + "------")
        menu += '\n\nInputs:'
        for spec in cls.acquired_data_specs():
            if isinstance(spec, BaseFileset):
                menu += '\n\t{} ({}){}:\n\t\t{}'.format(
                    spec.name, ', '.join(f.name for f in spec.valid_formats),
                    (' - optional' if spec.optional else ''),
                    wrap_desc(spec.desc, 80))
            else:
                menu += '\n\t{} ({}{}){}: {}'.format(
                    spec.name, spec.dtype.__name__,
                    (' array' if spec.array else ''),
                    (' - optional' if spec.optional else ''),
                    wrap_desc(spec.desc, 80))
        menu += '\n\nDerivatives:'
        for spec in cls.derived_data_specs():
            if isinstance(spec, BaseFileset):
                menu += '\n\t{} ({}):\n\t\t{}'.format(
                    spec.name, spec.format.name,
                    wrap_desc(spec.desc, 80))
            else:
                menu += '\n\t{} ({}{}): {}'.format(
                    spec.name, spec.dtype.__name__,
                    (' array' if spec.array else ''),
                    wrap_desc(spec.desc, 80))
        menu += '\n\nParameters:'
        for spec in cls.param_specs():
            menu += '\n\t{} ({}){}:\n\t\t{}'.format(
                spec.name, spec.dtype.__name__,
                (' [{}]'.format(", ".join(str(c) for c in spec.choices))
                 if spec.choices else ''),
                wrap_desc(spec.desc, 80))
        return menu

    @property
    def tree(self):
        return self.repository.cached_tree(
            subject_ids=self._subject_ids,
            visit_ids=self._visit_ids,
            fill=self._fill_tree)

    def clear_caches(self):
        """
        Called after a pipeline is run against the study to force an update of
        the derivatives that are now present in the repository if a subsequent
        pipeline is run.
        """
        self.repository.clear_cache()
        self._bound_specs = {}
        self._pipelines_cache = {}

    @property
    def processor(self):
        return self._processor

    @property
    def fill_tree(self):
        return self._fill_tree

    @property
    def environment(self):
        return self._environment

    @property
    def inputs(self):
        return list(self._inputs.values())

    @property
    def input_names(self):
        return list(self._inputs.keys())

    def input(self, name):
        try:
            return self._inputs[name]
        except KeyError:
            raise ArcanaNameError(
                name,
                "{} doesn't have an input named '{}'"
                .format(self, name))

    @property
    def missing_inputs(self):
        return (n for n in self.acquired_data_spec_names()
                if n not in self._inputs)

    @property
    def subject_ids(self):
        if self._subject_ids is None:
            return [s.id for s in self.tree.subjects]
        return self._subject_ids

    @property
    def visit_ids(self):
        if self._visit_ids is None:
            return [v.id for v in self.tree.visits]
        return self._visit_ids

    @property
    def num_subjects(self):
        return len(self.subject_ids)

    @property
    def num_visits(self):
        return len(self.visit_ids)

    @property
    def num_sessions(self):
        if self._visit_ids is None and self._subject_ids is None:
            num_sessions = len(list(self.tree.sessions))
        else:
            num_sessions = self.num_subjects * self.num_visits
        return num_sessions

    @property
    def prefix(self):
        """The study name as a prefix for fileset names"""
        return self.name + '_'

    @property
    def name(self):
        """Accessor for the unique study name"""
        return self._name

    @property
    def repository(self):
        "Accessor for the repository member (e.g. Daris, XNAT, MyTardis)"
        return self._repository

    def new_pipeline(self, *args, **kwargs):
        """
        Creates a Pipeline object, passing the study (self) as the first
        argument
        """
        return Pipeline(self, *args, **kwargs)

    def _get_parameter(self, name):
        try:
            parameter = self._parameters[name]
        except KeyError:
            try:
                parameter = self._param_specs[name]
            except KeyError:
                raise ArcanaNameError(
                    name,
                    "Invalid parameter, '{}', in {} (valid '{}')"
                    .format(
                        name, self._param_error_location,
                        "', '".join(self.param_spec_names())))
        return parameter

    def parameter(self, name):
        """
        Retrieves the value of the parameter and registers the parameter
        as being used by this pipeline for use in provenance capture

        Parameters
        ----------
        name : str
            The name of the parameter to retrieve
        """
        return self._get_parameter(name).value

    def branch(self, name, values=None):  # @UnusedVariable @IgnorePep8
        """
        Checks whether the given switch matches the value provided

        Parameters
        ----------
        name : str
            The name of the parameter to retrieve
        value : str | None
            The value(s) of the switch to match if a non-boolean switch
        """
        if isinstance(values, basestring):
            values = [values]
        spec = self.param_spec(name)
        if not isinstance(spec, SwitchSpec):
            raise ArcanaUsageError(
                "{} is standard parameter not a switch".format(spec))
        switch = self._get_parameter(name)
        if spec.is_boolean:
            if values is not None:
                raise ArcanaDesignError(
                    "Should not provide values ({}) to boolean switch "
                    "'{}' in {}".format(
                        values, name, self._param_error_location))
            in_branch = switch.value
        else:
            if values is None:
                raise ArcanaDesignError(
                    "Value(s) need(s) to be provided non-boolean switch"
                    " '{}' in {}".format(
                        name, self._param_error_location))
            values = set(values)
            # Register parameter as being used by the pipeline
            unrecognised_values = values - set(spec.choices)
            if unrecognised_values:
                raise ArcanaDesignError(
                    "Provided value(s) ('{}') for switch '{}' in {} "
                    "is not a valid option ('{}')".format(
                        "', '".join(unrecognised_values), name,
                        self._param_error_location,
                        "', '".join(spec.choices)))
            in_branch = switch.value in values
            if not in_branch:
                try:
                    in_branch = spec.fallbacks[switch.value] in values
                except KeyError:
                    pass
        return in_branch

    def unhandled_branch(self, name):
        """
        Convenient method for raising exception if a pipeline doesn't
        handle a particular switch value

        Parameters
        ----------
        name : str
            Name of the switch
        value : str
            Value of the switch which hasn't been handled
        """
        raise ArcanaDesignError(
            "'{}' value of '{}' switch in {} is not handled"
            .format(self.parameter(name), name,
                    self._param_error_location))

    @property
    def _param_error_location(self):
        return ("generation of '{}' pipeline of {}"
                .format(self._pipeline_to_generate, self))

    @property
    def parameters(self):
        for param_name in self._param_specs:
            yield self._get_parameter(param_name)

    @property
    def switches(self):
        for name in self._param_specs:
            if isinstance(self.spec(name), SwitchSpec):
                yield self._get_parameter(name)

    def save_workflow_graph_for(self, spec_name, fname, full=False,
                                style='flat', **kwargs):
        """
        Saves a graph of the workflow to generate the requested spec_name

        Parameters
        ----------
        spec_name : str
            Name of the spec to generate the graph for
        fname : str
            The filename for the saved graph
        style : str
            The style of the graph, can be one of can be one of
            'orig', 'flat', 'exec', 'hierarchical'
        """
        pipeline = self.spec(spec_name).pipeline
        if full:
            workflow = pe.Workflow(name='{}_gen'.format(spec_name),
                                   base_dir=self.processor.work_dir)
            self.processor._connect_pipeline(
                pipeline, workflow, **kwargs)
        else:
            workflow = pipeline._workflow
        fname = op.expanduser(fname)
        if not fname.endswith('.png'):
            fname += '.png'
        dotfilename = fname[:-4] + '.dot'
        workflow.write_graph(graph2use=style,
                             dotfilename=dotfilename)

    def spec(self, name):
        """
        Returns either the input corresponding to a fileset or field
        field spec or a spec or parameter that has either
        been passed to the study as an input or can be derived.

        Parameters
        ----------
        name : Str | BaseData | Parameter
            A parameter, fileset or field or name of one
        """
        # If the provided "name" is actually a data item or parameter then
        # replace it with its name.
        if isinstance(name, (BaseData, Parameter)):
            name = name.name
        # If name is a parameter than return the parameter spec
        if name in self._param_specs:
            return self._param_specs[name]
        else:
            return self.bound_spec(name)

    def bound_spec(self, name):
        """
        Returns an input selector or derived spec bound to the study, i.e.
        where the repository tree is checked for existing outputs

        Parameters
        ----------
        name : Str
            A name of a fileset or field
        """
        # If the provided "name" is actually a data item or parameter then
        # replace it with its name.
        if isinstance(name, BaseData):
            name = name.name
        # Get the spec from the class
        spec = self.data_spec(name)
        try:
            bound = self._inputs[name]
        except KeyError:
            if not spec.derived and spec.default is None:
                raise ArcanaMissingDataException(
                    "Input (i.e. non-generated) data '{}' "
                    "was not supplied when the study '{}' was "
                    "initiated".format(name, self.name))
            else:
                try:
                    bound = self._bound_specs[name]
                except KeyError:
                    bound = self._bound_specs[name] = spec.bind(self)
        return bound

    @classmethod
    def data_spec(cls, name):
        """
        Return the fileset_spec, i.e. the template of the fileset expected to
        be supplied or generated corresponding to the fileset_spec name.

        Parameters
        ----------
        name : Str
            Name of the fileset_spec to return
        """
        # If the provided "name" is actually a data item or parameter then
        # replace it with its name.
        if isinstance(name, BaseData):
            name = name.name
        try:
            return cls._data_specs[name]
        except KeyError:
            raise ArcanaNameError(
                name,
                "No fileset spec named '{}' in {}, available:\n{}"
                .format(name, cls.__name__,
                        "\n".join(list(cls._data_specs.keys()))))

    @classmethod
    def param_spec(cls, name):
        try:
            return cls._param_specs[name]
        except KeyError:
            raise ArcanaNameError(
                name,
                "No parameter spec named '{}' in {}, available:\n{}"
                .format(name, cls.__name__,
                        "\n".join(list(cls._param_specs.keys()))))

    @classmethod
    def data_specs(cls):
        """Lists all data_specs defined in the study class"""
        return iter(cls._data_specs.values())

    @classmethod
    def param_specs(cls):
        return iter(cls._param_specs.values())

    @classmethod
    def data_spec_names(cls):
        """Lists the names of all data_specs defined in the study"""
        return iter(cls._data_specs.keys())

    @classmethod
    def param_spec_names(cls):
        """Lists the names of all param_specs defined in the study"""
        return iter(cls._param_specs.keys())

    @classmethod
    def spec_names(cls):
        return chain(cls.data_spec_names(),
                     cls.param_spec_names())

    @classmethod
    def acquired_data_specs(cls):
        """
        Lists all data_specs defined in the study class that are
        provided as inputs to the study
        """
        return (c for c in cls.data_specs() if not c.derived)

    @classmethod
    def derived_data_specs(cls):
        """
        Lists all data_specs defined in the study class that are typically
        generated from other data_specs (but can be overridden by input
        filesets)
        """
        return (c for c in cls.data_specs() if c.derived)

    @classmethod
    def derived_data_spec_names(cls):
        """Lists the names of generated data_specs defined in the study"""
        return (c.name for c in cls.derived_data_specs())

    @classmethod
    def acquired_data_spec_names(cls):
        "Lists the names of acquired data_specs defined in the study"
        return (c.name for c in cls.acquired_data_specs())

    def cache_inputs(self):
        """
        Runs the Study's repository source node for each of the inputs
        of the study, thereby caching any data required from remote
        repositorys. Useful when launching many parallel jobs that will
        all try to concurrently access the remote repository, and probably
        lead to timeout errors.
        """
        workflow = pe.Workflow(name='cache_download',
                               base_dir=self.processor.work_dir)
        subjects = pe.Node(IdentityInterface(['subject_id']), name='subjects',
                           environment=self.environment)
        sessions = pe.Node(IdentityInterface(['subject_id', 'visit_id']),
                           name='sessions', environment=self.environment)
        subjects.iterables = ('subject_id', tuple(self.subject_ids))
        sessions.iterables = ('visit_id', tuple(self.visit_ids))
        source = pe.Node(RepositorySource(
            self.bound_spec(i).collection for i in self.inputs), name='source')
        workflow.connect(subjects, 'subject_id', sessions, 'subject_id')
        workflow.connect(sessions, 'subject_id', source, 'subject_id')
        workflow.connect(sessions, 'visit_id', source, 'visit_id')
        workflow.run()

    @classmethod
    def print_specs(cls):
        print('Available data:')
        for spec in cls.data_specs():
            print(spec)
        print('\nAvailable parameters:')
        for spec in cls.param_specs():
            print(spec)

    def provided(self, spec_name, default_okay=True):
        """
        Checks to see whether the corresponding data spec was provided an
        explicit input, as opposed to derivatives or missing optional inputs

        Parameters
        ----------
        spec_name : str
            Name of a data spec
        default_okay : bool
            Whether or not the default value is okay, or whether it needs to
            be explicitly provided as an input. For derivatives the default is
            the derived value
        """
        try:
            spec = self.bound_spec(spec_name)
        except ArcanaMissingDataException:
            return False
        if isinstance(spec, BaseInput):
            return True
        elif default_okay:
            if isinstance(spec, BaseInputSpec):
                return spec.default is not None
            else:
                return True
        else:
            return False

    @classmethod
    def freq_from_iterators(cls, iterators):
        """
        Returns the frequency corresponding to the given iterators
        """
        return {
            set(it): f for f, it in cls.FREQUENCIES.items()}[set(iterators)]

    @property
    def prov(self):
        """
        Extracts provenance information from the study for storage alongside
        generated derivatives. Typically for reference purposes only as only
        the pipeline workflow, inputs and outputs are checked by default when
        determining which sessions require reprocessing.

        Returns
        -------
        prov : dict[str, *]
            A dictionary containing the provenance information to record
            for the study
        """
        # Get list of repositories where inputs to the study are stored
        input_repos = list(set((i.repository for i in self.inputs)))
        inputs = {}
        for input in self.inputs:  # @ReservedAssignment
            inputs[input.name] = {
                'repository_index': input_repos.index(input.repository)}
            if input.frequency == 'per_study':
                inputs[input.name]['names'] = next(input.collection).name
            elif input.frequency == 'per_subject':
                inputs[input.name]['names'] = {i.subject_id: i.name
                                               for i in input.collection}
            elif input.frequency == 'per_visit':
                inputs[input.name]['names'] = {i.visit_id: i.name
                                               for i in input.collection}
            elif input.frequency == 'per_session':
                names = defaultdict(dict)
                for item in input.collection:
                    names[item.subject_id][item.visit_id] = item.name
                # Convert from defaultdict to dict
                inputs[input.name]['names'] = dict(names.items())
        return {
            'name': self.name,
            'type': get_class_info(type(self)),
            'parameters': {p.name: p.value for p in self.parameters},
            'inputs': inputs,
            'environment': self.environment.prov,
            'repositories': [r.prov for r in input_repos],
            'processor': self.processor.prov,
            'subject_ids': tuple(self.subject_ids),
            'visit_ids': tuple(self.visit_ids)}


class StudyMetaClass(type):
    """
    Metaclass for all study classes that collates data specs from
    bases and checks pipeline method names.

    Combines specifications in add_(data|parameter)_specs from
    the class to be created with its base classes, overriding matching
    specs in the order of the bases.
    """

    def __new__(metacls, name, bases, dct):  # @NoSelf @UnusedVariable
        if not any(issubclass(b, Study) for b in bases):
            raise ArcanaUsageError(
                "StudyMetaClass can only be used for classes that "
                "have Study as a base class")
        try:
            add_data_specs = dct['add_data_specs']
        except KeyError:
            add_data_specs = []
        try:
            add_param_specs = dct['add_param_specs']
        except KeyError:
            add_param_specs = []
        combined_attrs = set()
        combined_data_specs = {}
        combined_param_specs = {}
        for base in reversed(bases):
            # Get the combined class dictionary including base dicts
            # excluding auto-added properties for data and parameter specs
            combined_attrs.update(
                a for a in dir(base) if (not issubclass(base, Study) or
                                         a not in base.spec_names()))
            # TODO: need to check that fields are not overridden by filesets
            #       and vice-versa
            try:
                combined_data_specs.update(
                    (d.name, d) for d in base.data_specs())
            except AttributeError:
                pass  # Not a Study class
            try:
                combined_param_specs.update(
                    (p.name, p) for p in base.param_specs())
            except AttributeError:
                pass  # Not a Study class
        combined_attrs.update(list(dct.keys()))
        combined_data_specs.update((d.name, d) for d in add_data_specs)
        combined_param_specs.update(
            (p.name, p) for p in add_param_specs)
        # Check that the pipeline names in data specs correspond to a
        # pipeline method in the class and that if a pipeline is called with
        # arguments (for parameterizing a range of metrics for example) then
        # the same argument names are consistent across every call.
        pipeline_arg_names = {}
        for spec in add_data_specs:
            if spec.derived:
                if spec.pipeline_getter in ('pipeline', 'new_pipeline'):
                    raise ArcanaDesignError(
                        "Cannot use the names 'pipeline' or  'new_pipeline' "
                        "('{}') for the name of a pipeline constructor in "
                        "class {} as it clashes with base method to create "
                        "pipelines"
                        .format(spec.pipeline_getter, name))
                if spec.pipeline_getter not in combined_attrs:
                    raise ArcanaDesignError(
                        "Pipeline to generate '{}', '{}', is not present"
                        " in '{}' class".format(
                            spec.name, spec.pipeline_getter, name))
                try:
                    if pipeline_arg_names[
                            spec.pipeline_getter] != spec.pipeline_arg_names:
                        raise ArcanaDesignError(
                            "Inconsistent pipeline argument names used for "
                            "'{}' pipeline getter {} and {}".format(
                                spec.pipeline_getter,
                                pipeline_arg_names[spec.pipeline_getter],
                                spec.pipeline_arg_names))
                except KeyError:
                    pipeline_arg_names[
                        spec.pipeline_getter] = spec.pipeline_arg_names
        # Check for name clashes between data and parameter specs
        spec_name_clashes = (set(combined_data_specs) &
                             set(combined_param_specs))
        if spec_name_clashes:
            raise ArcanaDesignError(
                "'{}' name both data and parameter specs in '{}' class"
                .format("', '".join(spec_name_clashes), name))
        reserved_clashes = [n for n in combined_data_specs
                            if n in Study.ITERFIELDS]
        if reserved_clashes:
            raise ArcanaDesignError(
                "'{}' data spec names clash with reserved names"
                .format("', '".join(reserved_clashes), name))
        dct['_data_specs'] = combined_data_specs
        dct['_param_specs'] = combined_param_specs
        if '__metaclass__' not in dct:
            dct['__metaclass__'] = metacls
        # Append description of Study parameters to class
        try:
            docstring = dct['__doc__']
        except KeyError:
            docstring = '{} Study class'.format(name)
        if 'Parameters' not in docstring:
            docstring += Study.__doc__
        dct['__doc__'] = docstring
        return type(name, bases, dct)


def pickle_reconstructor(metacls, name, bases, cls_dict):
    obj = DummyObject()
    obj.__class__ = metacls(name, bases, cls_dict)
    return obj


class DummyObject(object):
    pass
