# Only a single class is defined so hardly makes sense to split.
# pylint: disable=too-many-lines
"""Define a class for working with HDF5 files."""

from abc import ABC, abstractmethod
from io import BytesIO
import os
import os.path
from sys import exc_info
from itertools import count
from time import sleep

# from ast import literal_eval
from traceback import format_exception
import logging

from lxml import etree
import h5py
import numpy
from astropy.io import fits

from autowisp.pipeline_exceptions import HDF5LayoutError

git_id = "$Id: b770969695b6cf8082b19c75e543f6bdcb370717 $"


# This is a h5py issue not an issue with this module
# pylint: disable=too-many-ancestors
# pylint: disable=too-many-public-methods
class HDF5File(ABC, h5py.File):
    """
    Base class for HDF5 pipeline products.

    The actual structure of the file has to be defined by a class inheriting
    from this one, by overwriting the relevant properties and
    :meth:`_get_root_tag_name`.

    Implements backwards compatibility for different versions of the structure
    of files.

    Attributes:
        _file_structure:    See the first entry returned by get_file_structure.

        _file_structure_version:    See the second entry returned by
            get_file_structure.

        _hat_id_prefixes (numpy.array):    A list of the currently recognized
            HAT-ID prefixes, with the correct data type ready for adding as a
            dataset.
    """

    @classmethod
    @abstractmethod
    def _get_root_tag_name(cls):
        """The name of the root tag in the layout configuration."""

    @classmethod
    @abstractmethod
    def _product(cls):
        """The pipeline key of the product held in this type of HDF5 files."""

    @property
    def _layout_version_attribute(self):
        """
        Return path, name of attribute in the file holding the layout version.
        """

        return "/", "LayoutVersion"

    @property
    @abstractmethod
    def elements(self):
        """
        Identifying strings for the recognized elements of the HDF5 file.

        Shoul be a dictionary-like object with values being a set of strings
        containing the identifiers of the HDF5 elements and keys:

            * dataset: Identifiers for the data sets that could be included in
                the file.

            * attribute: Identifiers for the attributes that could be included
                in the file.

            * link: Identifiers for the links that could be included in
                the file.
        """

    @classmethod
    @abstractmethod
    def get_file_structure(cls, version=None):
        """
        Return the layout structure with the given version of the file.

        Args:
            version:    The version number of the layout structure to set. If
                None, it should provide the default structure for new files
                (presumably the latest version).

        Returns:
            (dict, str):

                The dictionary specifies how to include elements in the HDF5
                file. The keys for the dictionary should be one in one of the
                lists in self.elements and the value is an object with
                attributes decsribing how to include the element. See classes in
                :mod:database.data_model for the provided attributes and their
                meining.

                The string is the actual file structure version returned. The
                same as version if version is not None.
        """

    def _flag_required_attribute_parents(self):
        """
        Flag attributes whose parents must exist when adding the attribute.

        The file structure must be fully configured before calling this method!

        If the parent is a group, it is safe to create it and then add the
        attribute, however, this in is not the case for attributes to datasets.

        Add an attribute named 'parent_must_exist' to all attribute
        configurations in self._file_structure set to False if and only if the
        attribute parent is a group.
        """

        dataset_paths = [
            self._file_structure[dataset_key].abspath
            for dataset_key in self.elements["dataset"]
        ]

        for attribute_key in self.elements["attribute"]:
            attribute = self._file_structure[attribute_key]
            attribute.parent_must_exist = attribute.parent in dataset_paths

    def _write_text_to_dataset(
        self, dataset_key, text, if_exists="overwrite", **substitutions
    ):
        r"""
        Adds ASCII text/file as a dateset to an HDF5 file.

        Args:
            dataset_key:    The key identifying the dataset to add.

            text:    The text or file to add. If it is an open file, the
                contents is dumped, if it is a python2 string or a python3
                bytes, the value is stored.

            if_exists:    See add_dataset().

            substitututions:    Any arguments that should be substituted in the
                dataset path.

        Returns:
            None
        """

        if isinstance(text, bytes):
            data = numpy.frombuffer(text, dtype="i1")
        elif isinstance(text, numpy.ndarray) and text.dtype == "i1":
            data = text
        else:
            data = numpy.fromfile(text, dtype="i1")

        self.add_dataset(
            dataset_key, data, if_exists=if_exists, **substitutions
        )

    def write_fitsheader_to_dataset(self, dataset_key, fitsheader, **kwargs):
        r"""
        Adds a FITS header to an HDF5 file as a dataset.

        Args:
            dataset_key(str):    The key identifying the dataset to add the
                header to.

            fitsheader(fits.Header):    The header to save.

            kwargs:    Passed directly to :meth:`_write_text_to_dataset`\ .

        Returns:
            None
        """

        if isinstance(fitsheader, str):
            # pylint false positive
            # pylint: disable=no-member
            with fits.open(fitsheader, "readonly") as fitsfile:
                header = fitsfile[0].header
                if header["NAXIS"] == 0:
                    header = fitsfile[1].header
                fitsheader_string = b"".join(map(bytes, header.cards))
            # pylint: enable=no-member
        else:
            fitsheader_string = b"".join(
                card.image.encode("ascii") for card in fitsheader.cards
            )
        fitsheader_array = numpy.frombuffer(fitsheader_string, dtype="i1")
        self._write_text_to_dataset(dataset_key, fitsheader_array, **kwargs)

    def read_fitsheader_from_dataset(self, dataset_key, **substitutions):
        """
        Reads a FITS header from an HDF5 dataset.

        The inverse of :meth:`write_fitsheader_to_dataset`.

        Args:
            h5dset:    The dataset containing the header to read.

        Returns:
            fits.Header:
                The FITS header contained in the given dataset.
        """

        fitsheader_array = self.get_dataset(dataset_key, **substitutions)
        return fits.Header.fromfile(
            BytesIO(fitsheader_array.data), endcard=False, padding=False
        )

    def check_for_dataset(self, dataset_key, must_exist=True, **substitutions):
        """
        Check if the given key identifies a dataset and it actually exists.

        Args:
            dataset_key:    The key identifying the dataset to check for.

            must_exist:    If True, and the dataset does not exist, raise
                IOError.

            substitutions:    Any arguments that should be substituted in the
                path. Only required if must_exist == True.

        Returns:
            None

        Raises:
            KeyError:
                If the specified key is not in the currently set file structure
                or does not identify a dataset.

            IOError:
                If the dataset does not exist but the must_exist argument is
                True.
        """

        if dataset_key not in self._file_structure:
            raise KeyError(
                f"The key '{dataset_key:s}' does not exist in the list of "
                f"configured {self._product()!s} file entries."
            )

        if (
            dataset_key not in self.elements["dataset"]
            and dataset_key not in self.elements["link"]
        ):
            raise KeyError(
                f"The key '{dataset_key!s}' does not identify a dataset or "
                f"link in '{self.filename!s}'"
            )

        if must_exist:
            dataset_path = (
                self._file_structure[dataset_key].abspath % substitutions
            )
            if dataset_path not in self:
                raise IOError(
                    f"Requried dataset ('{dataset_key}') '{dataset_path}' does "
                    f"not exist in '{self.filename}'"
                )

    @classmethod
    def get_element_type(cls, element_id):
        """
        Return the type of HDF5 entry that corresponds to the given ID.

        Args:
            element_id:    The identifying string for an element present in the
                HDF5 file.

        Returns:
            hdf5_type:    The type of HDF5 structure to create for this element.
                One of: 'group', 'dataset', 'attribute', 'link'.
        """

        # All implementations of _elemnts are required to make them dict-like.
        # pylint: disable=no-member
        for element_type, recognized in cls.elements.items():
            if element_id.rstrip(".") in recognized:
                return element_type
        # pylint: enable=no-member

        raise KeyError("Unrecognized element: " + repr(element_id))

    def get_element_path(self, element_id, **substitutions):
        """
        Return the path to the given element (.<attr> for attributes).

        Args:
            substitutions:    Arguments that should be substituted in the path.
                If none are given, the path is returned without substitutions.

        Returns:
            str:
                A string giving the path the element does/will have in the file.
        """

        path_template = None
        for element_type, recognized in self.elements.items():
            if element_id.rstrip(".") in recognized:
                if element_type == "attribute":
                    attribute_config = self._file_structure[element_id]
                    path_template = (
                        attribute_config.parent + "." + attribute_config.name
                    )
                else:
                    path_template = self._file_structure[element_id].abspath

        assert path_template is not None

        if substitutions:
            return path_template % substitutions
        return path_template

    def layout_to_xml(self):
        """Create an etree.Element decsribing the currently defined layout."""

        root = etree.Element(
            "group",
            {
                "name": self._get_root_tag_name(),
                "version": self._file_structure_version,
            },
        )

        def require_parent(path, must_be_group):
            """
            Return group element at the given path creating groups as needed.

            Args:
                path ([str]):    The path for the group element required. Each
                    entry in the list is the name of a sub-group of the previous
                    entry.

            Returns:
                etree.Element:
                    The element holding the group at the specified path. If it
                    does not exist, it is created along with any parent groups
                    required along the way.

            Raises:
                TypeError:
                    If an element anywhere along the given path already exists,
                    but is not a group.
            """

            parent = root
            if len(path) == 1 and path[0] == "":
                return parent
            current_path = ""
            for group_name in path:
                found = False
                current_path += "/" + group_name
                for element in parent.iterfind("./*"):
                    if element.attrib["name"] == group_name:
                        if element.tag != "group" and (
                            must_be_group or element.tag != "dataset"
                        ):
                            raise TypeError(
                                "Element "
                                + repr(current_path)
                                + " exists, but is of type "
                                + element.tag
                                + ", expected group"
                                + ("" if must_be_group else " or dataset")
                                + "!"
                            )
                        parent = element
                        found = True
                        break
                if not found:
                    parent = etree.SubElement(parent, "group", name=group_name)
            return parent

        def add_dataset(parent, dataset):
            """
            Add the given dataset as a SubElement to the given parent.

            Args:
                parent (etree.Element):    The group element in the result
                    tree to add the dataset under.

                dataset:    The dataset to add (object with attributes
                    specifying how the dataset should be added to the file).
            """

            etree.SubElement(
                parent,
                "dataset",
                name=dataset.abspath.rsplit("/", 1)[1],
                key=dataset.pipeline_key,
                dtype=dataset.dtype,
                compression=(
                    (dataset.compression or "")
                    + ":"
                    + (dataset.compression_options or "")
                ),
                scaleoffset=str(dataset.scaleoffset),
                shuffle=str(dataset.shuffle),
                fill=repr(dataset.replace_nonfinite),
                description=dataset.description,
            )

        def add_attribute(parent, attribute):
            """Add the given attribute as a SubElement to the given parent."""

            etree.SubElement(
                parent,
                "attribute",
                name=attribute.name,
                key=attribute.pipeline_key,
                dtype=dataset.dtype,
                description=attribute.description,
            )

        def add_link(parent, link):
            """Add the given link as a SubElement to the given parent."""

            etree.SubElement(
                parent,
                "link",
                name=link.abspath.rsplit("/", 1)[1],
                key=link.pipeline_key,
                target=link.target,
                description=link.description,
            )

        for dataset_key in self.elements["dataset"]:
            dataset = self._file_structure[dataset_key]
            path = dataset.abspath.lstrip("/").split("/")[:-1]
            add_dataset(require_parent(path, True), dataset)

        for attribute_key in self.elements["attribute"]:
            attribute = self._file_structure[attribute_key]
            path = attribute.parent.lstrip("/").split("/")
            add_attribute(require_parent(path, False), attribute)

        for link_key in self.elements["link"]:
            link = self._file_structure[link_key]
            path = link.abspath.lstrip("/").split("/")[:-1]
            add_link(require_parent(path, True), link)

        return root

    def get_dtype(self, element_key):
        """Return numpy data type for the element with by the given key."""

        result = self._file_structure[element_key].dtype

        if result == "manual":
            return None

        # Used only on input defined by us.
        # pylint: disable=eval-used
        result = eval(result)
        # pylint: enable=eval-used

        if isinstance(result, str):
            result = numpy.dtype(result)

        return result

    # The path_substitutions arg is used by overloading functions.
    # pylint: disable=unused-argument
    # The point of this function is to handle many cases
    # pylint: disable=too-many-branches
    def get_dataset_creation_args(self, dataset_key, **path_substitutions):
        """
        Return all arguments to pass to create_dataset() except the content.

        Args:
            dataset_key:    The key identifying the dataset to delete.

            path_substitutions:    In theory the dataset creation arguments can
                depend on the full dataset path (c.f. srcextract.sources).

        Returns:
            dict:
                All arguments to pass to create_dataset() or require_dataset()
                except: name, shape and data.
        """

        self.check_for_dataset(dataset_key, False)

        dataset_config = self._file_structure[dataset_key]
        result = {"shuffle": dataset_config.shuffle}

        dtype = self.get_dtype(dataset_key)
        if dtype is not None:
            result["dtype"] = dtype

        if dataset_config.compression is not None:
            result["compression"] = dataset_config.compression
            if (
                dataset_config.compression == "gzip"
                and dataset_config.compression_options is not None
            ):
                result["compression_opts"] = int(
                    dataset_config.compression_options
                )

        if dataset_config.scaleoffset is not None:
            result["scaleoffset"] = dataset_config.scaleoffset

        if dataset_config.replace_nonfinite is not None:
            result["fillvalue"] = dataset_config.replace_nonfinite

        if dataset_key in ["catalogue.columns", "srcproj.columns"]:
            column = path_substitutions[
                dataset_key.split(".")[0] + "_column_name"
            ]
            if column in [
                "hat_id_prefix",
                "hat_id_field",
                "hat_id_source",
                "objtype",
                "doublestar",
                "sigRA",
                "sigDec",
                "phqual",
                "magsrcflag",
                "enabled",
                "DESIGNATION",
                "phot_variable_flag",
                "datalink_url",
                "epoch_photometry_url",
                "libname_gspphot",
                "pmra",
                "pmdec",
                "phot_bp_mean_mag",
                "phot_rp_mean_mag",
                "phot_bp_mean_flux",
                "phot_rp_mean_flux",
                "phot_bp_mean_flux_error",
                "phot_rp_mean_flux_error",
                "phot_bp_rp_excess_factor",
            ]:
                result["compression"] = "gzip"
                result["compression_opts"] = 9
                result["shuffle"] = True
            elif column in ["RA", "Dec", "RA_orig", "Dec_orig"]:
                del result["compression"]
                result["scaleoffset"] = 7
            elif column in ["xi", "eta", "x", "y"]:
                del result["compression"]
                result["scaleoffset"] = 6
            elif column in [
                "J",
                "H",
                "K",
                "B",
                "V",
                "R",
                "I",
                "u",
                "g",
                "r",
                "i",
                "z",
            ] or column.endswith("mag"):
                del result["compression"]
                result["scaleoffset"] = 3
            elif column in [
                "dist",
                "epochRA",
                "epochDec",
                "sigucacmag",
                "errJ",
                "errH",
                "errK",
            ]:
                del result["compression"]
                result["scaleoffset"] = 2
            elif column in "source_id" or column.endswith("_n_obs"):
                del result["compression"]
                result["dtype"] = numpy.dtype("uint64")
                result["scaleoffset"] = 0
            else:
                del result["compression"]
                result["scaleoffset"] = 1

        return result

    # pylint: enable=unused-argument
    # pylint: enable=too-many-branches

    @staticmethod
    def hdf5_class_string(hdf5_class):
        """Return a string identifier of the given hdf5 class."""

        if issubclass(hdf5_class, h5py.Group):
            return "group"
        if issubclass(hdf5_class, h5py.Dataset):
            return "dataset"
        if issubclass(hdf5_class, h5py.HardLink):
            return "hard link"
        if issubclass(hdf5_class, h5py.SoftLink):
            return "soft link"
        if issubclass(hdf5_class, h5py.ExternalLink):
            return "external link"
        raise ValueError(
            "Argument to hdf5_class_string does not appear to be a class or"
            " a child of a class defined by h5py!"
        )

    def add_attribute(
        self,
        attribute_key,
        attribute_value,
        if_exists="overwrite",
        **substitutions,
    ):
        """
        Adds a single attribute to a dateset or a group.

        Args:
            attribute_key:    The key in _destinations that corresponds to the
                attribute to add. If the key is not one of the recognized keys,
                h5file is not modified and the function silently exits.

            attribute_value:    The value to give the attribute.

            if_exists:    What should be done if the attribute exists? Possible
                values are:

                * ignore:
                    do not update but return the attribute's value.

                * overwrite:
                    Change the value to the specified one.

                * error:
                    raise an exception.

            substitutions:    variables to substitute in HDF5 paths and names.

        Returns:
            unknown:
                The value of the attribute. May differ from attribute_value if
                the attribute already exists, if type conversion is performed,
                or if the file structure does not specify a location for the
                attribute. In the latter case the result is None.
        """

        if attribute_key not in self._file_structure:
            return None

        assert attribute_key in self.elements["attribute"]

        attribute_config = self._file_structure[attribute_key]
        parent_path = attribute_config.parent % substitutions
        if parent_path not in self:
            parent = self.create_group(parent_path)
        else:
            parent = self[parent_path]

        attribute_name = attribute_config.name % substitutions
        if attribute_name in parent.attrs:
            # TODO: handle  multi-valued attributes correctly.
            if (
                if_exists == "ignore"
                or (
                    parent.attrs[attribute_name]
                    == numpy.asarray(attribute_value)
                ).all()
            ):
                return parent.attrs[attribute_name]
            if if_exists == "error":
                raise HDF5LayoutError(
                    "Attribute "
                    f"'{self.filename}/{parent_path}.{attribute_name}' "
                    "already exists!"
                )
            assert if_exists == "overwrite"

        if isinstance(attribute_value, (str, bytes, numpy.string_)):
            parent.attrs.create(
                attribute_name,
                (
                    attribute_value.encode("ascii")
                    if isinstance(attribute_value, str)
                    else attribute_value
                ),
            )
        else:
            parent.attrs.create(
                attribute_name,
                attribute_value,
                dtype=self.get_dtype(attribute_key),
            )

        return parent.attrs[attribute_name]

    def delete_attribute(self, attribute_key, **substitutions):
        """Delete the given attribute."""

        attribute_config = self._file_structure[attribute_key]
        parent_path = attribute_config.parent % substitutions
        if parent_path in self:
            parent = self[parent_path]
            attribute_name = attribute_config.name % substitutions
            try:
                del parent.attrs[attribute_name]
            except KeyError:
                pass

    def add_link(self, link_key, if_exists="overwrite", **substitutions):
        """
        Adds a soft link to the HDF5 file.

        Args:
            link_key:    The key identifying the link to create.

            if_exists:    See same name argument to :meth:`add_attribute`.

            substitutions:    variables to substitute in HDF5 paths and names of
                both where the link should be place and where it should point
                to.

        Returns:
            str:
                The path the identified link points to. See if_exists argument
                for how the value con be determined or None if the link was not
                created (not defined in current file structure).

        Raises:
            IOError:    if an object with the same name as the link exists,
                but is not a link or is a link, but does not point to the
                configured target and if_exists == 'error'.
        """

        if link_key not in self._file_structure:
            return None

        assert link_key in self.elements["link"]

        link_config = self._file_structure[link_key]

        link_path = link_config.abspath % substitutions
        target_path = link_config.target % substitutions

        if link_path in self:
            existing_class = self.get(link_path, getclass=True, getlink=True)
            if issubclass(existing_class, h5py.SoftLink):
                existing_target_path = self[link_path].path
                if if_exists == "ignore" or existing_target_path == target_path:
                    return existing_target_path

                raise IOError(
                    f"Unable to create link with key {link_key}: a link at "
                    f"'{link_path}' already exists in '{self.filename}', and "
                    f"points to '{existing_target_path}' instead of "
                    f"'{target_path}'!"
                )
            raise IOError(
                f"Unable to create link with key {link_key}: a "
                f"{self.hdf5_class_string(existing_class)} at '{link_path}' "
                f"already exists in '{self.filename}'!"
            )
        self[link_path] = h5py.SoftLink(target_path)
        return target_path

    def delete_link(self, link_key, **substitutions):
        """Delete the link corresponding to the given key."""

        link_path = self._file_structure[link_key].abspath % substitutions
        if link_path in self:
            del self[link_path]

    def _add_repack_dataset(self, dataset_path):
        """Add the given dataset to the list of datasets to repack."""

        if "repack" not in self._file_structure:
            return
        repack_attribute_config = self._file_structure["repack"]
        if repack_attribute_config.parent not in self:
            self.create_group(repack_attribute_config.parent)
        repack_parent = self[repack_attribute_config.parent]
        self._logger.debug(
            "Adding %s to repack datasets (dtype: %s) of %s.",
            repr(dataset_path.encode("ascii")),
            repr(self.get_dtype("repack")),
            self.filename,
        )
        if repack_attribute_config.name in repack_parent.attrs:
            repack_parent.attrs[repack_attribute_config.name] = (
                repack_parent.attrs[repack_attribute_config.name]
                + ","
                + dataset_path
            ).encode("ascii")
        else:
            repack_parent.attrs.create(
                repack_attribute_config.name, dataset_path.encode("ascii")
            )

    def delete_dataset(self, dataset_key, **substitutions):
        """
        Delete obsolete HDF5 dataset if it exists and update repacking flag.

        Args:
            dataset_key:    The key identifying the dataset to delete.

        Returns:
            bool:
                Was a dataset actually deleted?

        Raises:
            Error.HDF5:
                if an entry already exists at the target dataset's location
                but is not a dataset.
        """

        if dataset_key not in self._file_structure:
            return False

        self.check_for_dataset(dataset_key, False)

        dataset_config = self._file_structure[dataset_key]
        dataset_path = dataset_config.abspath % substitutions

        if dataset_path in self:
            self._add_repack_dataset(dataset_path)
            print(f'Deleting {dataset_path} from {self.filename}')
            del self[dataset_path]
            return True

        return False

    def dump_file_or_text(
        self, dataset_key, file_contents, if_exists="overwrite", **substitutions
    ):
        """
        Adds a byte-by-byte dump of a file-like object to self.

        Args:
            dataset_key:    The key identifying the dataset to create for the
                file contents.

            file_contents:    See text argument to
                :meth:`_write_text_to_dataset`. None is also a valid value, in
                which case an empty dataset is created.

            if_exists:    See same name argument to add_attribute.

            substitutions:    variables to substitute in the dataset HDF5 path.
        Returns:
            (bool):
                Was the dataset actually created?
        """

        self._write_text_to_dataset(
            dataset_key=dataset_key,
            text=(
                file_contents
                if file_contents is not None
                else numpy.empty((0,), dtype="i1")
            ),
            if_exists=if_exists,
            **substitutions,
        )
        return True

    def add_file_dump(
        self,
        dataset_key,
        fname,
        if_exists="overwrite",
        delete_original=True,
        **substitutions,
    ):
        """
        Adds a byte by byte dump of a file to self.

        If the file does not exist an empty dataset is created.

        Args:
            fname:    The name of the file to dump.

            dataset_key:    Passed directly to dump_file_like.

            if_exists:    See same name argument to add_attribute.

            delete_original:    If True, the file being dumped is
                deleted (default).

            substitutions:    variables to substitute in the dataset HDF5 path.
        Returns:
            None.
        """

        created_dataset = self.dump_file_or_text(
            dataset_key,
            # Switching to if would result in unnecessarily complicated code
            # pylint: disable=consider-using-with
            (open(fname, "rb") if os.path.exists(fname) else None),
            # pylint: enable=consider-using-with
            if_exists,
            **substitutions,
        )
        if delete_original and os.path.exists(fname):
            if created_dataset:
                os.remove(fname)
            else:
                raise IOError(
                    f"Dataset '{dataset_key}' containing a dump of '{fname}' "
                    f"not created in '{self.filename}' but original deletion "
                    "was requested!"
                )

    def get_attribute(self, attribute_key, default_value=None, **substitutions):
        """
        Returns the attribute identified by the given key.

        Args:
            attribute_key:    The key of the attribute to return. It must be one
                of the standard keys.

            default_value:    If this is not None this values is returned if the
                attribute does not exist in the file, if None, not finding the
                attribute rasies IOError.

            substitutions:    Any keys that must be substituted in the path
                (i.e. ap_ind, config_id, ...).

        Returns:
            value:    The value of the attribute.

        Raises:
            KeyError:
                If no attribute with the given key is defined in the current
                files structure or if it does not correspond to an attribute.

            IOError:
                If the requested dataset is not found and no default value was
                given.
        """

        if attribute_key not in self._file_structure:
            raise KeyError(
                f"The key '{attribute_key}' does not exist in the list of "
                "configured HDF5 file structure."
            )
        if attribute_key not in self.elements["attribute"]:
            raise KeyError(
                f"The key '{attribute_key}' does not correspond to an attribute"
                " in the configured HDF5 file structure."
            )

        attribute_config = self._file_structure[attribute_key]

        parent_path = attribute_config.parent % substitutions
        attribute_name = attribute_config.name % substitutions

        if parent_path not in self:
            if default_value is not None:
                return default_value
            raise IOError(
                f"Requested attribute ({attribute_key}) '{attribute_name}' from"
                f" a non-existent path: '{parent_path}' in '{self.filename}'!"
            )
        parent = self[parent_path]
        if attribute_name not in parent.attrs:
            if default_value is not None:
                return default_value
            raise IOError(
                f"The attribute ({attribute_key}) '{attribute_name}' is not "
                f"defined for '{parent_path}' in '{self.filename}'!"
            )
        return parent.attrs[attribute_name]

    def get_dataset(
        self,
        dataset_key,
        expected_shape=None,
        default_value=None,
        **substitutions,
    ):
        """
        Return a dataset as a numpy float or int array.

        Args:
            dataset_key:    The key in self._destinations identifying the
                dataset to read.

            expected_shape:    The shape to use for the dataset if an empty
                dataset is found. If None, a zero-sized array is returned.

            default_value:    If the dataset does not exist, this value is
                returned.

            substitutions:    Any arguments that should be substituted in the
                path.

        Returns:
            numpy.array:
                A numpy int/float array containing the identified dataset from
                the HDF5 file.

        Raises:
            KeyError:
                If the specified key is not in the currently set file structure
                or does not identify a dataset.

            IOError:
                If the dataset does not exist, and no default_value was
                specified
        """

        self.check_for_dataset(
            dataset_key, default_value is None, **substitutions
        )

        dataset_config = self._file_structure[dataset_key]
        dataset_path = dataset_config.abspath % substitutions

        if dataset_path not in self:
            return default_value

        dataset = self[dataset_path]
        variable_length_dtype = h5py.check_dtype(vlen=dataset.dtype)
        #        if variable_length_dtype is not None:
        #            result_dtype = variable_length_dtype

        if dataset.size == 0:
            result = numpy.full(
                shape=(
                    dataset.shape if expected_shape is None else expected_shape
                ),
                fill_value=numpy.nan,
            )
        elif variable_length_dtype is not None:
            return dataset[:]
        else:
            result = numpy.empty(
                shape=dataset.shape, dtype=self.get_dtype(dataset_key)
            )
            dataset.read_direct(result)

        if (
            dataset_config.replace_nonfinite is not None
            and result.dtype.kind == "f"
        ):
            result[result == dataset.fillvalue] = numpy.nan

        return result

    def get_dataset_shape(self, dataset_key, **substitutions):
        """Return the shape of the given dataset."""

        dataset_path = self._file_structure[dataset_key].abspath % substitutions

        if dataset_path not in self:
            return None

        return self[dataset_path].shape

    @staticmethod
    def _replace_nonfinite(data, expected_dtype, replace_nonfinite):
        """Return (copy of) data with non-finite values replaced."""

        if (
            data.dtype.kind == "S"
            or data.dtype == numpy.string_
            or data.dtype == numpy.bytes_
        ) and (
            (
                expected_dtype is not None
                and numpy.dtype(expected_dtype).kind == "f"
            )
            or numpy.atleast_1d(numpy.atleast_1d(data) == b"NaN").all()
        ):
            assert (data == b"NaN").all() or (data == b"None").all()
            return numpy.full(
                fill_value=(replace_nonfinite or numpy.nan),
                dtype=numpy.float64,
                shape=data.shape,
            )

        if replace_nonfinite is None:
            return data

        finite = numpy.isfinite(data)
        if finite.all():
            return data
        data_copy = numpy.copy(data)
        data_copy[numpy.logical_not(finite)] = replace_nonfinite
        return data_copy

    def add_dataset(
        self,
        dataset_key,
        data,
        *,
        if_exists="overwrite",
        unlimited=False,
        shape=None,
        dtype=None,
        **substitutions,
    ):
        """
        Adds a single dataset to self.

        If the target dataset already exists, it is deleted first and the
        name of the dataset is added to the root level Repack attribute.

        Args:
            dataset_key:    The key identifying the dataset to add.

            data:    The values that should be written, a numpy array with
                an appropriate data type or None if an empty dataset should be
                created.

            if_exists:    See same name argument to add_attribute.

            unlimited(bool):    Should the first dimension of the dataset be
                unlimited (i.e. data can be added later)?

            shape(tuple(int,...)):    The shape of the dataset to create if data
                is None, otherwise the shape of the data is used. Just like if
                data is specified, the first dimension will be ignored if
                unlimited is True. It is an error to specify both data and
                shape!

            dtype:    The data type for the new dataset if the data is None. It
                is an error to specify both dtype and data!

            substitututions:    Any arguments that should be substituted in the
                dataset path.

        Returns:
            None
        """

        self.check_for_dataset(dataset_key, False)
        dataset_config = self._file_structure[dataset_key]
        dataset_path = dataset_config.abspath % substitutions

        if dataset_path in self:
            print(
                f"Dataset {dataset_path!r} already existis in "
                f'{self.filename!r}: {if_exists.rstrip("e")}ing!'
            )
            if if_exists == "ignore":
                return None
            if if_exists == "error":
                raise IOError(
                    f"Dataset ('{dataset_key}') '{dataset_path}' already exists"
                    f" in '{self.filename}' and overwriting is not allowed!"
                )
            self.delete_dataset(dataset_key, **substitutions)

        creation_args = self.get_dataset_creation_args(
            dataset_key, **substitutions
        )

        if data is None:
            data_copy = None
        else:
            data_copy = self._replace_nonfinite(
                data,
                creation_args.get("dtype"),
                dataset_config.replace_nonfinite,
            )

        if data is not None:
            assert shape is None
            assert dtype is None
            shape = data.shape
            dtype = data_copy.dtype

        if unlimited:
            shape_tail = shape[1:]

            if hasattr(self, "_chunk_size"):
                # pylint: disable=no-member
                creation_args["chunks"] = (self._chunk_size,) + shape_tail
                # pylint: enable=no-member
            else:
                creation_args["chunks"] = True

            creation_args["maxshape"] = (None,) + shape_tail

        if (
            creation_args.get("dtype", dtype) == numpy.string_
            or dtype.kind == "S"
        ):
            assert creation_args.get("dtype", numpy.bytes_) == numpy.bytes_
            creation_args["dtype"] = h5py.special_dtype(vlen=bytes)

        if "scaleoffset" in creation_args:
            assert data is None or numpy.isfinite(data_copy).all()

        self.create_dataset(
            dataset_path, data=data_copy, shape=shape, **creation_args
        )
        return dataset_path

    def __init__(self, fname=None, mode=None, layout_version=None, **kwargs):
        """
        Opens the given HDF5 file in the given mode.

        Args:
            fname:    The name of the file to open.

            mode:    The mode to open the file in (see hdf5.File).

            layout_version:    If the file does not exist, this is the version
                of the layout that will be used for its structure. Leave None
                to use the latest defined.

            kwargs:    Any additional arguments. Passed directly to h5py.File.

        Returns:
            None
        """

        self._logger = logging.getLogger(__name__)
        if fname is None:
            assert mode is None
            super().__init__(
                "memory_only", mode="w", driver="core", backing_store=False
            )
        else:
            old_file = os.path.exists(fname)
            if mode[0] != "r":
                path = os.path.dirname(fname)
                if path:
                    try:
                        os.makedirs(path)
                    except OSError:
                        if not os.path.exists(path):
                            raise

            for retry in count():
                try:
                    super().__init__(fname, mode, **kwargs)
                    break
                except OSError as details:
                    if retry == 10:
                        raise HDF5LayoutError(
                            f"Problem opening {fname:s} in mode={mode:s}"
                            + "".join(format_exception(*exc_info()))
                        ) from details
                    sleep(60)

        layout_version_path, layout_version_attr = (
            self._layout_version_attribute
        )

        if fname is not None and old_file:
            layout_version = self[layout_version_path].attrs[
                layout_version_attr
            ]

        (
            self._defined_elements,
            self._file_structure,
            self._file_structure_version,
        ) = self.get_file_structure(layout_version)

        if fname is not None and not old_file:
            self[layout_version_path].attrs[
                layout_version_attr
            ] = self._file_structure_version

    @staticmethod
    def collect_columns(destination, name_head, name_tail, dset_name, values):
        """
        If dataset is 1D and name starts and ends as given, add to destination.

        This function is intended to be passed to h5py.Group.visititems() after
        fixing the first 3 arguments using functools.partial.

        Args:
            destination(pandas.DataFrame):    The DataFrame to add matching
                datasets to. Datasets are added with column names given by the
                part of the name between `name_head` and `name_tail`.

            name_head(str):    Only datasets whose names start with this will be
                included.

            name_tail(str):    Only datasets whose names end with this will be
                included.

            dset_name(str):    The name of the dataset.

            values(array-like):    The values to potentially add as the new
                column.

        Returns:
            None
        """

        if (
            isinstance(values, h5py.Dataset)
            and dset_name.startswith(name_head)
            and dset_name.endswith(name_tail)
            and len(values.shape) == 1
        ):
            column_name = dset_name[len(name_head) :]
            if name_tail:
                column_name = column_name[: -len(name_tail)]
            enum_transform = h5py.check_enum_dtype(values.dtype)
            if enum_transform is None:
                insert_values = values
            else:
                insert_values = numpy.empty(
                    values.shape,
                    dtype="S" + str(max(map(len, enum_transform.keys()))),
                )
                for new, old in enum_transform.items():
                    insert_values[values[:] == old] = new.encode("ascii")
            destination.insert(
                len(destination.columns), column_name, insert_values
            )

    def delete_columns(self, parent, name_head, name_tail, dset_name):
        """Delete 1D datasets under parent if name starts and ends as given."""

        if (
            isinstance(parent[dset_name], h5py.Dataset)
            and dset_name.startswith(name_head)
            and dset_name.endswith(name_tail)
            and len(parent[dset_name].shape) == 1
        ):
            if dset_name in parent:
                self._logger.debug(
                    "Deleting %s from %s in %s",
                    repr(dset_name),
                    repr(parent.name),
                    repr(self.filename),
                )
                self._add_repack_dataset(parent[dset_name].name)
                del parent[dset_name]


# pylint: enable=too-many-ancestors
# pylint: enable=too-many-public-methods
