"""The genomic-config base class and constants
"""
from genomic_config import regions
import copy
import urllib.parse
import re
import glob
import os

# Some constants for config location
GENOMIC_CONFIG_ENV = "GENOMIC_CONFIG"
"""The name of the shell environment variable that if present will store
the location to the genomic configuration file (`str`)
"""
GENOMIC_CONFIG_DEFAULT_NAME = ".genomic_data"
"""The default name for a genomic configuration file (`str`)
"""
GENOMIC_CONFIG_EXTENSIONS = [".cnf", ".ini", ".cfg"]
"""The allowed extensions for genomic configuration files (`list` of `str`)
"""

# The section names in the configuration file
GENERAL_SECTION = 'general'
"""The name of the general section in the configuration file (`str`)
"""
CHAIN_FILE_PREFIX = 'chain_files'
"""The name of the chain file section in the configuration file (`str`)
"""
ASSEMBLY_PREFIX = "assembly"
"""The name of the genome assembly section in the configuration file (`str`)
"""
REFERENCE_GENOME_PREFIX = "reference_genome"
"""The prefix for a reference genome section in the configuration file (`str`)
"""
ASSEMBLY_SYN_PREFIX = "assembly_synonyms"
"""The name of the assembly synonyms section in the genomic-config config file
 (`str`)
"""
CHR_NAME_SYN_PREFIX = "chr_name_synonyms"
"""The name of the chromosome name synonyms section in the genomic-config
config file (`str`).
"""
CHR_NAME_SORT_PREFIX = "chr_name_sort"
"""The name of the chromosome name sort section in the genomic-config
config file (`str`).
"""
SPECIES_SYN_SECTION = "species_synonyms"
"""The name of the species synonyms section in the genomic-config config file
 (`str`)
"""
SAMPLE_PREFIX = "samples"
"""The name of the sample section in the configuration file (`str`)
"""
POPULATION_PREFIX = "population"
"""The name of the population section in the configuration file (`str`)
"""
COHORT_PREFIX = "cohort"
"""The name of the cohort section in the configuration file (`str`)
"""
MAPPING_FILE_PREFIX = "mapping_file"
"""The name of the mapping_file section in the configuration file (`str`)
"""

# Section keys
SEED_KEY = 'seed'
"""The name for the seed key in a Genomic-Config configuration section (`str`)
"""
VERSION_KEY = 'version'
"""The name for the version key in a Genomic-Config configuration section (`str`)
"""
DETAIL_DELIMITER = '.'
"""The delimiter that separates detail fields in the genomic-config
 configuration section names (`str`)
"""
REMOTE_SCHEMES = ('ftp', 'http', 'https')
"""The names for the `urlib.urlparse.schemes` that are indicative of a path
 being a remote location (`tuple` of `str`)
"""
REFERENCE_GENOME_INDEXES = ['.fai', '.gzi']
"""The extensions of reference genome index files (`list` of `str`)
"""
VCF_INDEXES = ['.tbi', '.csi']
"""The extensions of VCF index files (`list` of `str`)
"""
BGEN_INDEXES = ['.bgi']
"""The extensions of BGEN index files (`list` of `str`)
"""
FORMAT_KEY = 'format'
"""The name for the file format key in a genomic configuration section (`str`)
"""
KNOWN_FILES = dict(
    vcf=('.vcf', False, VCF_INDEXES),
    bcf=('.bcf', False, VCF_INDEXES),
    faa=('.faa', True, REFERENCE_GENOME_INDEXES),
    bgen=('.bgen', True, BGEN_INDEXES)
)
"""The file types that are known by ``genomic_config``. This allows index
checking. Unknown file types are allowed but indexes are not checked
(`dict` of `tuple`)

Notes
-----
Each tuple has ``[0]`` the file extension ``[1]`` all - do all the indexes have
to be present ``[2]`` The expected index file extensions. If the user wants to
all other files to this then their indexes will be checked also.
"""


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
class _BaseConfig(object):
    """Handles interaction with the genomic configuration options in ``.ini``
    format (reading only). This should not be called directly but should be
    used to build new config file types.

    Parameters
    ----------
    config_file : `str`
        A configuration file location or an existing parser object to use
    use_species_synonyms : `bool`, optional, default: `True`
        When parsing the config file, attempt to map the species definitions
        to any synonyms defined in the config file. Also, when retrieving
        species related configuration, use species synonyms.
    use_assembly_synonyms : `bool`, optional, default: `True`
        When parsing the config file, attempt to map the genome assembly
        definitions to any synonyms defined in the config file. Also, when
        retrieving genome assembly related configuration, use assembly
        synonyms.
    validate_files : `bool`, optional, default: `True`
        For configuration options that have local files on the system. Ensure
        that they are actually accessible. If not exit with the relevant error.
    validate_indexes : `bool`, optional, default: `True`
        For configuration options that should have local index files associated
        with them (but the actual index files are not defined in the
        configuration file). Ensure that the index files actually exist. If not
        exit with the relevant error.

    Raises
    ------
    FileNotFoundError
        If a location for the default config file is not found
    PermissionError
        If a location for the default config file can't be read
    """
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, config_file, use_species_synonyms=True,
                 use_assembly_synonyms=True, validate_files=True,
                 validate_indexes=True):
        # Attempt to locate the configuration file
        self._config_file = config_file
        self.use_species_synonyms = use_species_synonyms
        self.use_assembly_synonyms = use_assembly_synonyms
        self._validate_files = validate_files
        self._validate_indexes = validate_indexes
        self._parsed = False

        self.validate_conifg_file = self._validate_file
        if self._validate_files is False:
            self.validate_conifg_file = self._no_validate_file

        self.validate_file_index = self._validate_index
        if self._validate_indexes is False:
            self.validate_file_index = self._no_validate_file

        # Get the outline empty configuration. This will contain all the
        # species specific data
        self._parser = None
        self._cnf = self.__class__.get_config_defaults()
        self._species = dict()

        # This will hold the species synonyms
        self._species_synonyms = dict()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __enter__(self):
        """Entry into the context manager

        Returns
        -------
        self : `base_config._BaseConfig`
            This class (or any sub-classes)
        """
        # print("ENTER CALLED")
        self.open()
        return self

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __exit__(self, *args):
        """Exit from the context manager.

        Parameters
        ----------
        *args
            The exit condition, this will be a tuple of 3 ``NoneType`` if the
            exit was clean.
        """
        # print("EXIT CALLED")
        self.close()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def open(self):
        """This should be called from any sub-classes and they should respond
        to the return condition.

        Returns
        -------
        has_been_parsed : `bool`
             There is a possibility to open via the context manager or not,
             this is just an indicator that the config has been parsed. So the
             first call will produce True, subsequent calls will produce False.
             In most instances this can be ignored by the user and is mainly
             for internal use or use if sub-classing.
        """
        # If a path to the config file has been given then make sure that it
        # exists
        if isinstance(self._config_file, str):
            open(self._config_file).close()

        # Already parsed (possibly from a context manager call)
        if self._parsed is True:
            return False
        self._parsed = True
        return True

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def close(self):
        """This should be called from any sub-classes.
        """
        self._parsed = False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def as_dict(self):
        """Return the configuration file as a nested dictionary. The returned
        `dict` is a deep copy of the actual configuration `dict`.

        Returns
        -------
        config_dict : `dict`
            A dict of all the configuration parsed from the config file
        """
        return copy.deepcopy(self._cnf)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def seed(self):
        """The ``seed`` defined in the config file (`int` or `NoneType`)
        """
        raise NotImplementedError("override this")

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @property
    def parser(self):
        """Return the underlying parser object that is parsing the config file
        """
        return self._parser

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_species_synonym(self, species):
        """Attempt to return a species synonym defined in the configuration
        file.

        Parameters
        ----------
        species : `str`
            The species to get a synonym for

        Returns
        -------
        species_synonym : `str`

        Notes
        -----
         If the synonym does not exist then no error is raised, rather the
        ``species`` argument is returned as it is assumed it is it's own
        synonym.
        """
        try:
            return self._species_synonyms[species]
        except KeyError:
            return species

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_species_synonyms(self):
        """Get all the species synonyms stored in the configuration file

        Returns
        -------
        species_synonyms : `dict`
            The species synonyms. Note that the returned dictionary is a copy
            of the actual dictionary in the config object.
        """
        return self._species_synonyms.copy()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_assembly_synonym(self, species, assembly):
        """Return a genome assembly synonym for a species.

        Parameters
        ----------
        species : `str`
            The species associated with the assembly synonym.
        assembly : `str`
            The assembly synonym associated with the species synonym.

        Returns
        -------
        assembly_synonym : `str`

        Notes
        -----
         If the synonym does not exist then no error is raised, rather the
        ``assembly`` argument is returned as it is assumed it is it's own
        synonym. The lookup for the species obeys ``use_species_synonym``.
        """
        species_data = self._check_species(species)

        try:
            return species_data[ASSEMBLY_SYN_PREFIX][assembly]
        except KeyError:
            return assembly

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_chr_name_synonyms(self, species, assembly, name):
        """Return all the chromosome name synonyms for a genome
        assembly/species.

        Parameters
        ----------
        species : `str`
            The species associated with the assembly synonym.
        assembly : `str`
            The assembly synonym associated with the species synonym.
        name : `str`
            The name for the chromosome synonym set.

        Returns
        -------
        chromosome_synonyms : `dict`
            The chromosome synonym mappings. Keys are chromosome synonyms and
            values are what they map to.
        """
        if self.use_assembly_synonyms is True:
            assembly = self.get_assembly_synonym(
                species, assembly
            )
        assembly_section = self._check_assembly(species, assembly)

        try:
            return assembly_section[CHR_NAME_SYN_PREFIX][name]
        except KeyError as e:
            raise KeyError(
                "unknown chromosome synonym name: "
                f"{name} ({species}:{assembly})"
            ) from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_chr_name_sort_order(self, species, assembly, name):
        """Return the chromosome in the correct sort order for a genome
        assembly/species.

        Parameters
        ----------
        species : `str`
            The species associated with the chromosome sort order.
        assembly : `str`
            The assembly synonym associated with the species synonym.
        name : `str`
            The name for the chromosome sort order.

        Returns
        -------
        chromosome_sort_order : `list`
            The chromosomes in the correct sort order. This can be used to define
            a key function.
        """
        if self.use_assembly_synonyms is True:
            assembly = self.get_assembly_synonym(
                species, assembly
            )
        assembly_section = self._check_assembly(species, assembly)

        try:
            return assembly_section[CHR_NAME_SORT_PREFIX][name]
        except KeyError as e:
            raise KeyError(
                "unknown chromosome sort order name: "
                f"{name} ({species}:{assembly})"
            ) from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_chr_name_sort_key(self, species, assembly, name, error=True):
        """Return a sort key function for the required chromosome sort order
        for a genome assembly/species.

        Parameters
        ----------
        species : `str`
            The species associated with the chromosome sort order.
        assembly : `str`
            The assembly synonym associated with the species synonym.
        name : `str`
            The name for the chromosome sort order.
        error : `bool`, optional, default: True
            Should the sort function raise an error if the requested
            chromosome is not known tot the sort function. If this is False and
            an unknown chromosome is encountered then it is placed last in the
            sort order after known chromosomes. Note, this may not give expected
            results if unknown chromosomes are not grouped together.

        Returns
        -------
        chromosome_sort_order : `list`
            The chromosomes in the correct sort order. This can be used to define
            a key function.
        """
        sort_order = self.get_chr_name_sort_order(species, assembly, name)
        sort_dict = {i: c for c, i in enumerate(sort_order)}
        default_idx = len(sort_order)

        if error:
            def _sort_error_key(x):
                return sort_dict[x]
            return _sort_error_key
        else:
            def _sort_no_error_key(x):
                try:
                    return sort_dict[x]
                except KeyError:
                    return default_idx
            return _sort_no_error_key

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_assembly_synonyms(self, species):
        """Get all the assembly synonyms stored in the configuration file for
        a species.

        Returns
        -------
        assembly_synonyms : `dict`
            The assembly synonyms. Note that the returned dictionary is a copy
            of the actual dictionary in the config object.

        Raises
        ------
        KeyError
            If the species does not exist
        """
        species_data = self._check_species(species)
        return species_data[ASSEMBLY_SYN_PREFIX].copy()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_available_samples(self):
        """Get all the available sample sets in the configuration file.

        Returns
        -------
        available_samples : `list` of `str`
            The names of the available sample sets in the configuration file.

        Notes
        -----
        Sample definitions are not associated with any species.

        See also
        --------
        genomic_config.base_config._BaseConfig.get_samples
        """
        return list(self._cnf[SAMPLE_PREFIX].keys())

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_samples(self, sample_set_name):
        """Get all the sample IDs associated with a sample set.

        Parameters
        ----------
        sample_set_name : `str`
            The name of the sample set containing the IDs

        Returns
        -------
        sample_ids : `list` of `str`
            The sample IDs associate with ``sample_set``.

        Raises
        ------
        KeyError
            If the sample set name does not exist

        See also
        --------
        genomic_config.base_config._BaseConfig.get_available_samples
        """
        return list(self._cnf[SAMPLE_PREFIX][sample_set_name])

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_available_populations(self):
        """Get all the available sample sets in the configuration file.

        Returns
        -------
        available_samples : `list` of `str`
            The names of the available sample sets in the configuration file.

        Notes
        -----
        Sample definitions are not associated with any species.

        See also
        --------
        genomic_config.base_config._BaseConfig.get_samples
        """
        return list(self._cnf[POPULATION_PREFIX].keys())

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_population_map(self, population_name):
        """Get all the population mappings associated with a population name.

        Parameters
        ----------
        population_name : `str`
            The name of the population you want the mappings for.

        Returns
        -------
        population mappings : `list` of `str`
            The population mappings for the population name.

        Raises
        ------
        KeyError
            If the population name does not exist

        Notes
        -----
        The idea here is to provide a mechanism for the user to define
        population data they have access to and how it related to a
        specific population name. For example, the European population,
        one user might have access to HapMap European samples and
        another 1000g genomes, so they can represent the "European"
        population with either of those mappings.
        """
        return list(self._cnf[POPULATION_PREFIX][population_name])

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_chain_file(self, species, source_assembly, target_assembly):
        """Get the chain file associated with the species mapping between the
        ``source_assembly`` and ``target_assembly``.

        Parameters
        ----------
        species : `str`
            The species name associated with the chain file.
        source_assembly : `str`
            The source assembly that the chain file maps from.
        target_assembly : `str`
            The target assembly that the chain file maps to.

        Returns
        -------
        chain_file_path : `str`
            The path to the chain file.

        Raises
        ------
        KeyError
            If the ``species``, ``source_assembly``, ``target_assembly``
            combination does not have a chain file associated with it.
        """
        assembly_section = self._check_assembly(species, source_assembly)

        if self.use_assembly_synonyms is True:
            target_assembly = self.get_assembly_synonym(
                species, target_assembly
            )
        return assembly_section[CHAIN_FILE_PREFIX][target_assembly]

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_reference_genome(self, species, assembly, name):
        """Get the reference genome associated with the ``species``,
        ``assembly`` and ``name``.

        Parameters
        ----------
        species : `str`
            The species name associated with the reference genome.
        assembly : `str`
            The assembly of the reference genome.
        name : `str`
            The name of the reference genome.

        Returns
        -------
        reference_genome_path : `str`
            The path to the reference genome.

        Raises
        ------
        KeyError
            If the ``species``, ``assembly``, ``name`` combination does not
            have a reference genome file associated with it.
        """
        assembly_section = self._check_assembly(species, assembly)
        return assembly_section[REFERENCE_GENOME_PREFIX][name]

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_mapping_file(self, species, assembly, name):
        """Get the mapping file associated with the ``species``,
        ``assembly`` and ``name``.

        Parameters
        ----------
        species : `str`
            The species name associated with the reference genome.
        assembly : `str`
            The assembly of the reference genome.
        name : `str`
            The name of the reference genome.

        Returns
        -------
        mapping_file_path : `str`
            The path to the mapping file.

        Raises
        ------
        KeyError
            If the ``species``, ``assembly``, ``name`` combination does not
            have a mapping file associated with it.
        """
        assembly_section = self._check_assembly(species, assembly)
        return assembly_section[MAPPING_FILE_PREFIX][name]

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_cohort_region_provider(self, species, assembly, cohort_name,
                                   file_format):
        """Get the region provider associated with the ``species``,
        ``assembly`` and ``cohort_name`` and ``file_format``.

        Parameters
        ----------
        species : `str`
            The parent species name for the assembly section
        assembly : `str`
            The assembly name for the section.
        cohort_name : `str`
            The name of the cohort.
        file_format : `str`
            The format of the files for the cohort.

        Returns
        -------
        region_provider : `` or `NoneType`
            The region provider object. A `NoneType` might be returned in cases
            where a cohort section has been setup but no cohort files are have
            parsed into it. This should not happen in normal operation but
            could possibly happen if the user has called
            `base_config._BaseConfig.set_cohort_section`.

        Raises
        ------
        KeyError
            If any of species, assembly, cohort_name, file_format are not
            defined.
        """
        # First get the assembly section based on species and assembly names
        # this respects the use_species_synonyms and use_assembly_synonyms
        # parameters.
        assembly_section = self._check_assembly(species, assembly)

        try:
            cohort_section = assembly_section[COHORT_PREFIX][cohort_name]
        except KeyError as e:
            raise KeyError(
                "unknown cohort name: {0}".format(cohort_name)
            ) from e

        # Make sure the file format is standardised
        file_format = file_format.lower()

        try:
            return cohort_section[file_format]
        except KeyError as e:
            raise KeyError(
                "unknown file format for cohort '{0}': {1}".format(
                    cohort_name, file_format
                )
            ) from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def add_cohort_files(self, species, assembly, file_format, cohort_name,
                         file_hints, provider_class=regions.RegionFileFetch):
        """Add files to a cohort.
        """
        # First get the assembly section based on species and assembly names
        # this respects the use_species_synonyms and use_assembly_synonyms
        # parameters.
        assembly_section = self._check_assembly(species, assembly)

        try:
            cohort_section = assembly_section[COHORT_PREFIX][cohort_name]
        except KeyError as e:
            raise KeyError(
                "unknown cohort name: {0}".format(cohort_name)
            ) from e

        # Make sure the file format is standardised
        file_format = file_format.lower()

        # Set the region fetcher
        cohort_section[file_format] = provider_class(
            from_list=file_hints
        )

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def set_species_section(self, species, use_species_synonyms=True):
        """Initialise a blank species section in the configuration if it does
        not already exist.

        Parameters
        ----------
        species : `str`
            The species name for the section.
        use_species_synonyms : `bool`, optional, default: `True`
            Lookup a synonym for the species argument.

        Returns
        -------
        species : `str`
            The actual species name that was used to store the default section
            data. This may differ from what is passed if
            ``use_species_synonyms=True``.
        """
        if use_species_synonyms is True:
            species = self.get_species_synonym(species)

        if species not in self._species:
            self._species[species] = self.__class__.get_species_defaults()

        return species

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def set_assembly_section(self, species, assembly,
                             use_species_synonyms=True,
                             use_assembly_synonyms=True):
        """Initialise a blank assembly section in the configuration if it does
        not already exist.

        Parameters
        ----------
        species : `str`
            The parent species name for the assembly section
        assembly : `str`
            The assembly name for the section
        use_species_synonyms : `bool`, optional, default: `True`
            Lookup a synonym for the species before creating the assembly
            section.
        use_assembly_synonyms : `bool`, optional, default: `True`
            Lookup an assembly synonym for the before creating the section.

        Returns
        -------
        species : `str`
            The actual species name that was used to store the default section
            data. This may differ from what is passed if
            ``use_species_synonyms=True``.
        assembly : `str`
            The actual assembly name that was used to store the default section
            data. This may differ from what is passed if
            ``use_assembly_synonyms=True``.
        """
        if use_species_synonyms is True:
            species = self.get_species_synonym(species)

        if use_assembly_synonyms is True:
            assembly = self.get_assembly_synonym(species, assembly)

        species_section = self._check_species(species)

        if assembly not in species_section[ASSEMBLY_PREFIX]:
            species_section[ASSEMBLY_PREFIX][assembly] = \
                self.__class__.get_assembly_defaults()

        return species, assembly

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def set_cohort_section(self, species, assembly, cohort_name, file_format,
                           use_species_synonyms=True,
                           use_assembly_synonyms=True):
        """Initialise a blank assembly section in the configuration if it does
        not already exist.

        Parameters
        ----------
        species : `str`
            The parent species name for the assembly section
        assembly : `str`
            The assembly name for the section.
        cohort_name : `str`
            The name of the cohort.
        file_format : `str`
            The format of the files for the cohort.
        use_species_synonyms : `bool`, optional, default: `True`
            Lookup a synonym for the species before creating the assembly
            section.
        use_assembly_synonyms : `bool`, optional, default: `True`
            Lookup an assembly synonym for the before creating the section.

        Returns
        -------
        species : `str`
            The actual species name that was used to store the default section
            data. This may differ from what is passed if
            ``use_species_synonyms=True``.
        assembly : `str`
            The actual assembly name that was used to store the default section
            data. This may differ from what is passed if
            ``use_assembly_synonyms=True``.
        """
        if use_species_synonyms is True:
            species = self.get_species_synonym(species)

        if use_assembly_synonyms is True:
            assembly = self.get_assembly_synonym(species, assembly)

        assembly_section = self._check_assembly(species, assembly)

        try:
            cohort_section = assembly_section[COHORT_PREFIX][cohort_name]
        except KeyError:
            cohort_section = dict()
            assembly_section[COHORT_PREFIX][cohort_name] = cohort_section

        # Make sure the file format is standardised
        file_format = file_format.lower()

        try:
            cohort_section[file_format]
        except KeyError:
            cohort_section[file_format] = None
        return species, assembly, cohort_section

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_species(self, species):
        """This gets a species section from a configuration file (if it
        exists). This respects the ``use_species_synonyms`` attribute as well

        Parameters
        ----------
        species : `str`
            The name of the species to retrieve the section for.

        Returns
        -------
        species_section : `dict`
            A species section the keys of this dictionary are:
            ``assembly_synonyms`` and ``assembly``

        Raises
        ------
        KeyError
            If the species (or any of the synonyms if
            ``use_species_synonyms=True``) does not exist in the configuration
        """
        if self.use_species_synonyms is True:
            species = self.get_species_synonym(species)

        try:
            return self._species[species]
        except KeyError as e:
            raise KeyError("unknown species: {0}".format(species)) from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_assembly(self, species, assembly):
        """This gets an assembly section from a configuration file (if it
        exists). This respects the ``use_species_synonyms`` attribute as well

        Parameters
        ----------
        species : `str`
            The name of the species to retrieve the section for.
        assembly : `str`
            The name of the assembly associated with the species to retrieve
            the section for.

        Returns
        -------
        assembly_section : `dict`
            An assembly section the keys of this dictionary are:
            ``chain_files``, ``cohorts`` and ``reference_genomes``

        Raises
        ------
        KeyError
            If the species (or any of the synonyms if
            ``use_species_synonyms=True``) does not exist in the configuration
            or if the assembly (or any of the synonyms if
            ``use_assembly_synonyms=True``) does not exist in the configuration
        """
        species_section = self._check_species(species)

        if self.use_assembly_synonyms is True:
            assembly = self.get_assembly_synonym(species, assembly)

        try:
            return species_section[ASSEMBLY_PREFIX][assembly]
        except KeyError as e:
            raise KeyError("unknown assembly: {0}".format(assembly)) from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _validate_file(self, file_path, file_type="genomic file"):
        """Ensure that a file can be opened.

        Parameters
        ----------
        file_path : `str`
            The file path to check
        file_type : `str`, optional, default: ``genomic_file``
            The type of the file. This is used in error messages should the
            file not be accessible.

        Raises
        ------
        This will raise any of the normal python file errors should the file
        not be accessible.

        Notes
        -----
        If the file is a remote file then nothing is checked.
        """
        if self.__class__.is_remote(file_path) is False:
            try:
                open(file_path).close()
            except Exception as e:
                raise e.__class__(
                    "problem with '{0}': {1}".format(file_type, file_path)
                ) from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _validate_index(self, file_path, indexes, file_type="genomic index",
                        all_indexes=True):
        """Ensure that index files that are associated with a genomic file are
        present.

        Parameters
        ----------
        file_path : `str`
            The file path to check
        indexes : `list` of `str`
            The indexes that can be associated with with the ``file_path``.
        file_type : `str`, optional, default: ``genomic_index``
            The type of the file. This is used in error messages should the
            file not be accessible.
        all_indexes : `bool`, optional, default: `True`
            Should all the indexes be present? If `False` then as long as one
            index file is present it is good enough. This is to handle
            situations like VCF file indexes that can be ``.tbi`` or ``.csi``.

        Raises
        ------
        This will raise any of the normal python file errors should the file
        not be accessible.

        Notes
        -----
        If the file is a remote file then nothing is checked.
        """
        present = []
        exceptions = []
        if self.__class__.is_remote(file_path) is False and len(indexes) > 0:
            for i in indexes:
                expected_path = "{0}{1}".format(file_path, i)
                try:
                    open(expected_path).close()
                    present.append(True)
                except Exception as e:
                    present.append(False)
                    exceptions.append((e, expected_path))

            raise_error = False
            if all_indexes is True and not all(present):
                raise_error = True
            if all_indexes is False and not any(present):
                raise_error = True

            if raise_error is True:
                raise exceptions[0][0].__class__(
                    "problem with index: '{0}'".format(
                            exceptions[0][1]
                    )
                )

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _no_validate_file(self, *args, **kwargs):
        """A method that just acts as a pass through should the user not want
        to validate files.

        Parameters
        ----------
        *args
            Any positional arguments that are usually passed to (ignored)
            ``base_config._BaseConfig._validate_file`` or
            ``base_config._BaseConfig._validate_index``
        **kwargs
            Any keyword arguments that are usually passed to (ignored)
            ``base_config._BaseConfig._validate_file`` or
            ``base_config._BaseConfig._validate_index``
        """
        pass

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def get_config_defaults(cls):
        """Get a blank empty dictionary of non-species configuration file
        sections.

        Returns
        -------
        blank_config :`dict` [`str`, `dict`]
            The keys are the parsed config file sections, the values are empty
            dictionaries where parsed content can be placed.
        """
        return {
            GENERAL_SECTION: {SEED_KEY: None},
            SAMPLE_PREFIX: dict(),
            POPULATION_PREFIX: dict()
        }

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def get_species_defaults(cls):
        """Get a blank empty dictionary of configuration file sections for a
        single species.

        Returns
        -------
        blank_config :`dict` [`str`, `dict`]
            The keys are the parsed config file sections, the values are empty
            dictionaries where parsed content can be placed.
        """
        return {
            ASSEMBLY_SYN_PREFIX: dict(),
            ASSEMBLY_PREFIX: dict(),
        }

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def get_assembly_defaults(cls):
        """Get a blank empty dictionary of configuration file sections for a
        single genome assembly.

        Returns
        -------
        blank_config :`dict` [`str`, `dict`]
            The keys are the parsed config file sections, the values are empty
            dictionaries where parsed content can be placed.
        """
        return {
            CHAIN_FILE_PREFIX: dict(),
            CHR_NAME_SYN_PREFIX: dict(),
            CHR_NAME_SORT_PREFIX: dict(),
            COHORT_PREFIX: dict(),
            REFERENCE_GENOME_PREFIX: dict(),
            MAPPING_FILE_PREFIX: dict()
        }

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def get_explicit_hint(cls, hint):
        """Extract hinting tuples from hinting strings.

        Parameters
        ----------
        hint : `str`
            the hinting string to extract the hinting tuple from

        Returns
        -------
        chr_name : `str`
            The extracted chromosome name
        start_pos : `int` or `NoneType`
            The extracted start position
        end_pos : `int` or `NoneType`
            The extracted end position

        Notes
        -----
        The algorithm here is very simple. First, we look for the structure
        of ``<CHR>:<START>-<END>``, for example ``1:12345-23456``. If this is
        matched then a fully filled out tuple is returned. If not then it is
        assumed that we have just <CHR> so the hint that was passed to the
        method is returned back at ``[0]`` with `NoneType` as ``[1]`` and
        ``[2]``.
        """
        pos_match = re.match(
            r'(?P<CHR>[\w\d_-]+):(?P<START>\d+)-(?P<END>\d+)', hint
        )
        if pos_match:
            return pos_match.group('CHR'), int(pos_match.group('START')), \
                int(pos_match.group('END'))
        return hint, None, None

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def get_implicit_hint(cls, path):
        """Get paths to any implicitly hinted files.

        Parameters
        ----------
        path : `str`
            The path to a file for a set of files defined by implicit hinting.

        Returns
        -------
        files : `list` of `tuple`
            The `tuple` has a hinting tuple at ``[0]`` and the file full
            unhinted file path at ``[1]``. The hinting `tuple` has the
            structure `(chr_name, start_pos, end_pos)`. If there was no
            `start_pos`, `end_pos` hinting then these will be `NoneType`. Note
            if the `path` contains no hinting then the hinting `tuple` will
            have the structure `(NoneType, NoneType, NoneType)`.

        Notes
        -----
        The hinting in the path should be in the format of {CHR}, {START},
        {END} placeholders where these attributes should be expected.

        Examples
        --------
        hinting matching a single path of ``{CHR}``, ``{START}``, ``{END}``:

        >>> from merit import config
        >>> import os
        <BLANKLINE>
        >>> # Home directory
        >>> home = os.environ['HOME']
        >>> # Create a file to work on
        >>> realpath = os.path.join(home, "fileX_from12400_to12345.vcf.gz")
        >>> open(realpath, 'w').close()
        <BLANKLINE>
        >>> hintpath = os.path.join(home, "file{CHR}_from{START}_to{END}.vcf.gz")
        >>> config.MeritConfig.get_implicit_hint(hintpath)
        [(('X', 12400, 12345), '/home/user/fileX_from12400_to12345.vcf.gz')]
        <BLANKLINE>
        >>> os.unlink(realpath)
        """
        # Make sure the path is a full path
        path = os.path.abspath(os.path.expanduser(path))

        if cls.is_remote(path) is True:
            raise TypeError("paths should not be remote locations")

        # The implicit hinting will be represented as {CHR}, {START}, {END}
        # embedded in the full file path or just {CHR}. So, any of these are
        # substituted with wildcards and then we glob that path to get any
        # matches
        test_path = path.format(CHR='*', START='*', END='*')
        files = glob.glob(test_path)

        # Will store all the full file names to the implicitly hinted files
        # if we do not find any hinted files then this empty list is returned
        hinted_files = []
        if len(files) > 0:
            # Here we have found some files that match the hinting structure
            # Not that even a file path with no hinting will glob back a single
            # matching file.

            # We do not know how they match yet. So the hinted path is
            # turned into a regexp so we can pull out the actual chr_name,
            # start_pos and end_pos if they exist.

            # This escapes the provided path but also reverses the escaping on
            # the {CHR}, {START}, {END} fields as we want to substitute regexps
            # into these.
            test_path = re.sub(
                r'\\{(CHR|START|END)\\}', '{\\1}', re.escape(path)
            )

            # Substitute regexp named capture groups into the placeholders.
            # Note that if the placeholders do not exist then this will not
            # fail
            test_path = test_path.format(
                CHR=r'(?P<CHR>.+?)',
                START=r'(?P<START>\d+?)',
                END=r'(?P<END>\d+?)'
            )

            # Now test all the files that we have found by globbing
            for i in files:
                match = re.match(test_path, i)

                # Is there a match
                if match:
                    hint = [None, None, None]

                    # Attempt to pull out the components that have matched
                    # the hinting should either be {CHR} in it's own or
                    # {CHR}, {START}, {END}, we do not support any other
                    # combinations (the presence of {CHR}, {START} is
                    # checked below)
                    for idx, pos in enumerate(['CHR', 'START', 'END']):
                        try:
                            hint[idx] = match.group(pos)
                        except IndexError:
                            # raise
                            break

                    # Expand to [{CHR}, None, None]in the case of [{CHR}]
                    for j in range(1, 3):
                        try:
                            hint[j] = int(hint[j])
                        except TypeError:
                            pass
                    # Ensure we do not have [{CHR}, {START}, None]
                    if hint[2] is None:
                        hint[1] = None
                    hinted_files.append((tuple(hint), i))
                else:
                    # No match, so there is not any hinting on the path
                    hinted_files.append((None, i))
        return hinted_files

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def get_hinting(cls, key, value):
        """Extract the hinting from a key value pair.

        Parameters
        ----------
        key : `str`
            The hinting key. If value is ``NoneType`` then the ``key`` will be
            a file path and not a hint. However, if the value is defined then
            key will be an explicit hint that will either be of the structure
            <chromosome_name> or <chromosome_name>:<start_pos>-<end_pos>.

        Returns
        -------
        """
        # No explicit hinting and remote, this is suspicious so we will
        # make sure there is no implicit hinting in there, if we find some
        # we will issue a warning only to alert the user to the possibility
        # of an error
        if value is None and cls.is_remote(key) is True:
            if '{CHR}' in key:
                raise TypeError(
                    "implicit hinting in remote location: {}".format(key)
                )
                # Add to the no hind set
                return [(None, True, key)]
        elif cls.is_remote(value) is True:
            # Remote with hinting
            hint = cls.get_explicit_hint(key)
            return [(hint, True, value)]
        elif value is None:
            # No exclicit hinting, so either, all implicit or no hinting
            # either way we should get at least one file returned
            return [(hint, False, path)
                    for hint, path in cls.get_implicit_hint(key)]
        else:
            # Local file with explicit hinting
            hint = cls.get_explicit_hint(key)
            return [(hint, False, value)]

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def is_remote(cls, path):
        """Determine if a file path points to a remote location

        Parameters
        ----------
        path : `str`
            The file path to test

        Returns
        -------
        remote_file : `bool`
            `True` if the path is a remote location `False` if not.

        Notes
        -----
        A path is determine to be a remote location if a urlparse scheme
        returns either ``ftp``, ``http``, or ``https``. If it returns ``""``
        then it is not determined to be a remote location.
        """
        return urllib.parse.urlparse(path).scheme in REMOTE_SCHEMES

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _raise_config_parse_error(cls, error):
        """Re-raise an exception with a configuration specific error prefixed
        onto the original error message.

        This is designed to help the user identify errors that have occurred
        during configuration file parsing.

        Parameters
        ----------
        error : `Exception`
            A caught exception that you want to re-raise

        Raises
        ------
        Exception
            An exception of the same type that was passed to the method but
            with the error message prefixed with a configuration specific
            message.
        """
        raise error.__class__(
            "Problem with configuration file: {0}".format(error.args[0])
        ) from error

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def _raise_option_defined(cls, section, option):
        """

        Parameters
        ----------
        section : `str`
            A section where the option is already defined
        option : `str`
            The option that has already been defined

        Raises
        ------
        KeyError
            A KeyError detailing the section and option that has clashed.
        """
        raise KeyError(
            "option already defined (check for other synonyms?):"
            " {0}:{1}".format(section, option)
        )
