autowisp.data_reduction.data_reduction_file — AutoWISP documentation

# Source code for autowisp.data_reduction.data_reduction_file

    """Define a class for worknig with data reduction files."""

    # pylint: disable=too-many-lines

    import string
    from functools import partial
    import logging

    import numpy
    import h5py
    import pandas

    from autowisp.multiprocessing_util import setup_process_map
    from autowisp.hat.file_parsers import parse_anmatch_transformation
    from autowisp.miscellaneous import RECOGNIZED_HAT_ID_PREFIXES

    from autowisp.database.hdf5_file_structure import HDF5FileDatabaseStructure

    git_id = "$Id: cd2a6c9ea1521813bb0f625ac3878ee3291e7f99 $"

    # TODO: Add missed attributes: bg.cfg.annulus, bg.cfg.zero.

    def init_dr_process(configuration):
        """Initialize the process with the given configuration."""

        DataReductionFile.fname_template = configuration["data_reduction_fname"]
        setup_process_map(configuration)

    # The class has to satisfy many needs, hence many public methods.
    # pylint: disable=too-many-public-methods

    # Out of my control (most ancestors come from h5py module).
    # pylint: disable=too-many-ancestors

    class DataReductionFile(HDF5FileDatabaseStructure):
        """
        Interface for working with the pipeline data reduction (DR) files.

        Attributes:
            _product(str):    The pipeline key of the HDF5 product. In this case:
                `'data_reduction'`

            _key_io_tree_to_dr (dict):    A dictionary specifying the correspondence
                between the keys used in astrowisp.IOTree to store quantities and
                the element key in the DR file.

            _dtype_dr_to_io_tree (dict):    A dictionary specifying the
                correspondence between data types for entries in DR files and data
                types in astrowisp.IOTree.
        """

        _logger = logging.getLogger(__name__)
        fname_template = None

        @classmethod
        def _product(cls):
            return "data_reduction"

        @classmethod
        def _get_root_tag_name(cls):
            """The name of the root tag in the layout configuration."""

            return "DataReduction"

        def _prepare_source_iter(
            self, dataset_key, column_substitution_name, **path_substitutions
        ):
            """
            Return required head and tail of paths identifying source collection.

            Args:
                See `get_sources()`.

            Returns:
                str:    The path to the parent group containing all source columns.

                str:    The string that must be in the beginning of each path for it
                    to be considered part of the source collection.

                str:    The string that must be in the end of each path for it to be
                    considered part of the source collection.
            """

            path_substitutions[column_substitution_name] = "{column}"
            self._logger.debug(
                "Parsing source path: %s",
                repr(self._file_structure[dataset_key].abspath),
            )
            parsed_path = string.Formatter().parse(
                self._file_structure[dataset_key].abspath % path_substitutions
            )
            pre_column, verify, _, _ = next(parsed_path)
            self._logger.debug("Pre_column: %s, verify: %s", pre_column, verify)
            assert verify == "column"
            try:
                name_tail = next(parsed_path)
                for i in range(1, 4):
                    assert name_tail[i] is None
                name_tail = name_tail[0]
                try:
                    next(parsed_path)
                    assert False
                except StopIteration:
                    pass
            except StopIteration:
                name_tail = ""
            parent, name_head = pre_column.rsplit("/", 1)
            return parent, name_head, name_tail

        @classmethod
        def get_fname_from_header(cls, header):
            """Return the filename of the DR file for the given header."""

            # TODO: implement filename template from DB ofter DB has been designed.
            # pylint: disable=no-member
            return cls.fname_template.format_map(header)

            # pylint: enable=no-member

        def get_dataset_creation_args(self, dataset_key, **path_substitutions):
            """See HDF5File.get_dataset_creation_args(), but handle srcextract."""

            result = super().get_dataset_creation_args(
                dataset_key, **path_substitutions
            )

            if dataset_key == "srcextract.sources":
                column = path_substitutions["srcextract_column_name"]
                if column.lower() in ["id", "numberpixels", "npix", "nsatpix"]:
                    result["compression"] = "gzip"
                    result["compression_opts"] = 9
                else:
                    del result["compression"]
                    result["scaleoffset"] = 3

            return result

        def add_sources(
            self,
            data,
            dataset_key,
            column_substitution_name,
            *,
            parse_ids=False,
            ascii_columns=(),
            **path_substitutions,
        ):
            """
            Creates datasets out of the fields in an array of sources.

            Args:
                data(structured numpy.array):    The data about the sources to
                    add.

                dataset_key(str):    The pipeline key for the dataset to add.

                column_substitution_name(str):    The %-subsittution variable to
                    distinguish between the column in the array.

                parse_ids(bool):    Should self.parse_hat_source_id() be used to
                    translate string IDs to datasets to insert?

                string_columns([str]):    A list of column names to convert to ascii
                    strings before saving.

            Returns:
                None
            """

            def iter_data():
                """Iterate over (column name, values) of the input data."""

                if hasattr(data, "dtype"):
                    for column_name in data.dtype.names:
                        yield column_name, data[column_name]
                else:
                    yield data.index.name, data.index.array
                    for column_name, series in data.items():
                        yield column_name, series.array

            for column_name, column_data in iter_data():
                if column_name in ascii_columns or column_data.dtype.kind in "SUO":
                    column_data = column_data.astype("string_")
                if parse_ids and column_name == "ID":
                    id_data = self.parse_hat_source_id(column_data)
                    for id_part in ["prefix", "field", "source"]:
                        self.add_dataset(
                            dataset_key=dataset_key,
                            data=id_data[id_part],
                            **{column_substitution_name: "hat_id_" + id_part},
                            **path_substitutions,
                        )
                else:
                    self._logger.debug(
                        "Saving %s dataset of type: %s",
                        repr(column_name),
                        repr(column_data.dtype),
                    )
                    self.add_dataset(
                        dataset_key=dataset_key,
                        data=column_data,
                        **{column_substitution_name: column_name.replace("/", "")},
                        **path_substitutions,
                    )

        def delete_sources(
            self, dataset_key, column_substitution_name, **path_substitutions
        ):
            """Delete all columns of a given source collection."""

            parent, name_head, name_tail = self._prepare_source_iter(
                dataset_key, column_substitution_name, **path_substitutions
            )
            if parent not in self:
                return
            to_delete = []
            self[parent].visit(to_delete.append)
            for dset_name in to_delete:
                self.delete_columns(self[parent], name_head, name_tail, dset_name)

        def get_sources(
            self, dataset_key, column_substitution_name, **path_substitutions
        ):
            """
            Return a collection of sources previously stored in the DR file.

            Args:
                dataset_key(str):    The pipeline key for the dataset to return.

                column_substitution_name(str):    The %-subsittution variable to
                    distinguish between the column in the array.

            Returns:
                dict:
                    The keys are the columns of the sources stored and the values
                    are 1-D numpy arrays containing the data.
            """

            parent, name_head, name_tail = self._prepare_source_iter(
                dataset_key, column_substitution_name, **path_substitutions
            )
            result = pandas.DataFrame()
            self._logger.debug(
                "Collecting columns frcom %s under %s, starting with %s and ending "
                "with %s",
                self.filename,
                parent,
                name_head,
                name_tail,
            )
            self[parent].visititems(
                partial(self.collect_columns, result, name_head, name_tail)
            )
            column_names = [colname.lower() for colname in result.columns]
            for id_colname in ["id", "source_id"]:
                if id_colname in column_names:
                    result.set_index(id_colname, inplace=True)
            return result

        def __init__(self, *args, **kwargs):
            """Open or create a data reduction file.

            Args:
                See HDF5File.__init__() for description of arguments, however
                instead of fname, a DataReductionFile can be specified by the header
                of the frame it corresponds to (or at least a dict-like object
                defining the header keywords required by the DR filename template).
            """

            if "header" in kwargs:
                kwargs["fname"] = self.get_fname_from_header(kwargs["header"])
                del kwargs["header"]

            super().__init__(*args, **kwargs)

            self._hat_id_prefixes = numpy.array(
                RECOGNIZED_HAT_ID_PREFIXES,
                dtype=self.get_dtype("srcproj.recognized_hat_id_prefixes"),
            )

        def get_dtype(self, element_key):
            """Return numpy data type for the element with by the given key."""

            if element_key.endswith(".hat_id_prefix"):
                return h5py.special_dtype(
                    enum=(
                        numpy.ubyte,
                        dict(
                            (prefix, value)
                            for value, prefix in enumerate(self._hat_id_prefixes)
                        ),
                    )
                )

            result = super().get_dtype(element_key)

            return result

        def parse_hat_source_id(self, source_id):
            """Return the prefix ID, field number, and source number."""

            if hasattr(source_id, "dtype") and source_id.shape:
                id_data = {
                    id_part: numpy.empty((len(source_id),), dtype=id_dtype)
                    for id_part, id_dtype in [
                        ("prefix", self.get_dtype(".hat_id_prefix")),
                        ("field", numpy.uint16),
                        ("source", numpy.uint32),
                    ]
                }

                for source_index, this_id in enumerate(source_id):
                    (
                        id_data["prefix"][source_index],
                        id_data["field"][source_index],
                        id_data["source"][source_index],
                    ) = self.parse_hat_source_id(this_id)
                return id_data

            if isinstance(source_id, bytes):
                c_style_end = source_id.find(b"\0")
                if c_style_end >= 0:
                    source_id = source_id[:c_style_end].decode()
                else:
                    source_id = source_id.decode()
            prefix_str, field_str, source_str = source_id.split("-")
            return (
                numpy.where(self._hat_id_prefixes == prefix_str.encode("ascii"))[0][
                    0
                ],
                int(field_str),
                int(source_str),
            )

        def get_source_count(self, **path_substitutions):
            """
            Return the number of sources for the given tool versions.

            Args:
                path_substitutions:    Values to substitute in the paths to the
                    datasets and attributes containing shape fit informaiton
                    (usually versions of various components).

            Returns:
                int:
                    The number of projected sources in the databasets reached by the
                    given substitutions.
            """

            path_substitutions["srcproj_column_name"] = "hat_id_prefix"
            return self[
                self._file_structure["srcproj.columns"].abspath % path_substitutions
            ].len()

        def add_frame_header(self, header, **substitutions):
            """Add the header of the corresponding FITS frame to DR file."""

            self.write_fitsheader_to_dataset("fitsheader", header, **substitutions)

        def get_frame_header(self, **substitutions):
            """Return the header of the corresponding FITS frame."""

            return self.read_fitsheader_from_dataset("fitsheader", **substitutions)

        def get_num_apertures(self, **path_substitutions):
            """Return the number of apertures used for aperture photometry."""

            num_apertures = 0
            while True:
                try:
                    self.check_for_dataset(
                        "apphot.magnitude",
                        aperture_index=num_apertures,
                        **path_substitutions,
                    )
                    num_apertures += 1
                except IOError:
                    return num_apertures

            assert False

        def get_num_magfit_iterations(self, **path_substitutions):
            """
            Return how many magnitude fitting iterations are in the file.

            Args:
                path_substitutions:    See get_source_count().

            Returns:
                int:
                    The number of magnitude fitting iterations performed on the
                    set of photometry measurements identified by the
                    path_substitutions argument.
            """

            path_substitutions["aperture_index"] = 0
            path_substitutions["magfit_iteration"] = 0
            for photometry_mode in ["shapefit", "apphot"]:
                try:
                    self.check_for_dataset(
                        photometry_mode + ".magfit.magnitude", **path_substitutions
                    )
                except IOError:
                    continue

                while True:
                    path_substitutions["magfit_iteration"] += 1
                    try:
                        self.check_for_dataset(
                            photometry_mode + ".magfit.magnitude",
                            **path_substitutions,
                        )
                    except IOError:
                        break

            return path_substitutions["magfit_iteration"]

        def has_shape_fit(self, accept_zeropsf=True, **path_substitutions):
            """True iff shape fitting photometry exists for path_substitutions."""

            try:
                self.check_for_dataset("shapefit.magnitude", **path_substitutions)
                return (
                    accept_zeropsf
                    or min(
                        self.get_attribute(
                            "shapefit.cfg.psf.bicubic.grid.x", **path_substitutions
                        ).size,
                        self.get_attribute(
                            "shapefit.cfg.psf.bicubic.grid.y", **path_substitutions
                        ).size,
                    )
                    > 2
                )
            except IOError:
                return False

        # Could not think of a reasonable way to simplify further.
        # pylint: disable=too-many-locals
        # pylint: disable=too-many-statements

        def get_source_data(
            self,
            *,
            magfit_iterations="all",
            shape_fit=True,
            apphot=True,
            string_source_ids=True,
            all_numeric_source_ids=False,
            background=True,
            **path_substitutions,
        ):
            """
            Extract available photometry from the data reduction file.

            Args:
                magfit_iterations(iterable):    The set of magnitude fitting
                    iterations to include in the result. ``0`` is the raw photometry
                    (i.e. no magnitude fitting), 1 is  single reference frame fit, 1
                    is the first re-fit etc. Use ``'all'`` to get all iterations.
                    Negative numbers have the same interpretation as python list
                    indices. For example ``-1`` is the final iteration.

                shape_fit(bool):    Should the result include shape fit photometry
                    measurements. If ``True`` and no shape fit is present, still
                    excludes shape fit columns.

                apphot(bool):    Should the result include aperture photometry
                    measurements.

                string_source_ids(bool):    Should source IDs be formatted as
                    strings (True) or a set of integers (False)?

                background(bool):    Should the result include information about the
                    background behind the sources?

                path_substitutions:    See get_source_count().

            Returns:
                pandas.Dataframe:
                    The photometry information in the current data reduction file.
                    The columns always included are:

                        * ID(set as index): an array of sources IDs in the given DR
                          file. Either a string (if string_source_ids) or 1- or
                          3-column composite index depending on ID type.

                        * <catalogue quantity> (dtype as needed): one entry for each
                          catalogue column.

                        * x (numpy.float64): The x coordinates of the sources

                        * y (numpy.float64): The y coordinates of the sources

                   The following columns are included if the corresponding input
                   argument is set to True:

                        * bg (numpy.float64): The background estimates for the
                          sources

                        * bg_err (numpy.float64): Error estimate for 'bg'

                        * bg_npix (numpy.uint): The number of pixel background
                          extraction was based on.

                        * mag (2-D numpy.float64 array): measured magnitudes. The
                          first dimension is the index within the
                          ``magfit_iterations`` argument and the second index
                          iterates over photometry, starting with shape fitting (if
                          the ``shape_fit`` argument is True),
                          followed by the aperture photometry measurement for each
                          aperture (if the ``apphot`` argument is True).

                        * mag_err (numpy.float64): Error estimate for ``mag``. Same
                          shape and order.

                        * phot_flag: The quality flag for the photometry. Same
                          shape and order as ``mag``.
            """

            def assemble_hat_id(prefix, field, source):

                return f"{prefix.decode()}-{field:03d}-{source:07d}".encode("ascii")

            def initialize_result():
                """Create the part of the result always included."""

                result = self.get_sources(
                    "srcproj.columns", "srcproj_column_name", **path_substitutions
                )
                self._logger.debug(
                    "Initial source data columns: %s", repr(result.columns)
                )
                hat_id_components = [
                    "hat_id_prefix",
                    "hat_id_field",
                    "hat_id_source",
                ]
                if string_source_ids:
                    if result.index.name == "source_id":
                        result["ID"] = numpy.vectorize(
                            lambda i: str(i).encode("ascii"), otypes=["O"]
                        )(result.index)
                    else:
                        result["ID"] = numpy.vectorize(assemble_hat_id)(
                            *[result[comp] for comp in hat_id_components],
                            otypes=["O"],
                        )
                        for id_component in hat_id_components:
                            del result[id_component]
                    result.set_index("ID", inplace=True)
                elif set(hat_id_components) < set(result.columns):
                    if all_numeric_source_ids:
                        result.insert(
                            0, "hat_id_prefnum", len(self._hat_id_prefixes)
                        )
                        for new_id, old_id in enumerate(self._hat_id_prefixes):
                            result.loc[
                                result["hat_id_prefix"] == old_id, "hat_id_prefnum"
                            ] = new_id
                        assert result["hat_id_prefnum"].max() < len(
                            self._hat_id_prefixes
                        )
                        hat_id_components[0] = "hat_id_prefnum"
                    result.set_index(hat_id_components, inplace=True)

                self._logger.debug(
                    "Source data after formatting ID:\n%s", repr(result)
                )

                return result

            def normalize_magfit_iterations():
                """Make sure ``magfit_iterations`` is a list of positive indices."""

                if magfit_iterations != "all" and (
                    len(magfit_iterations) == 0 or min(magfit_iterations) >= 0
                ):
                    return magfit_iterations

                all_magfit_indices = numpy.array(
                    [0]
                    + list(
                        range(
                            1,
                            self.get_num_magfit_iterations(**path_substitutions)
                            + 1,
                        )
                    )
                )

                if magfit_iterations == "all":
                    return all_magfit_indices

                return all_magfit_indices[magfit_iterations]

            def fill_background(result):
                """Fill the background entries in the result."""

                for result_key, dataset_key in (
                    ("bg", "bg.value"),
                    ("bg_err", "bg.error"),
                    ("bg_npix", "bg.npix"),
                ):
                    result[result_key] = self.get_dataset(
                        dataset_key,
                        expected_shape=result.shape,
                        **path_substitutions,
                    )

            def fill_photometry(result):
                """Fill the photomtric measurements entries in result."""

                for result_key, dataset_key_tail in (
                    ("mag", "magnitude"),
                    ("mag_err", "magnitude_error"),
                    ("phot_flag", "quality_flag"),
                ):
                    for magfit_iter in magfit_iterations:
                        if magfit_iter == 0 or result_key != "mag":
                            dataset_key_middle = ""
                        else:
                            dataset_key_middle = "magfit."
                        path_substitutions["magfit_iteration"] = magfit_iter - 1
                        column_tail = f"_mfit{magfit_iter:03d}"
                        if shape_fit:
                            result["shapefit_" + result_key + column_tail] = (
                                self.get_dataset(
                                    (
                                        "shapefit."
                                        + dataset_key_middle
                                        + dataset_key_tail
                                    ),
                                    expected_shape=result.shape,
                                    **path_substitutions,
                                )
                            )
                        if apphot:
                            num_apertures = self.get_num_apertures(
                                **path_substitutions
                            )
                            for aperture_index in range(num_apertures):
                                result[
                                    f"ap{aperture_index:03d}_"
                                    + result_key
                                    + column_tail
                                ] = self.get_dataset(
                                    (
                                        "apphot."
                                        + dataset_key_middle
                                        + dataset_key_tail
                                    ),
                                    expected_shape=result.shape,
                                    aperture_index=aperture_index,
                                    **path_substitutions,
                                )

            shape_fit = shape_fit and self.has_shape_fit(**path_substitutions)
            magfit_iterations = normalize_magfit_iterations()
            result = initialize_result()

            if background:
                fill_background(result)

            fill_photometry(result)
            return result

        def get_source_ids(self, string_source_ids=True, **path_substitutions):
            """Return the IDs of the sources in the given DR file.

            Args:
                string_source_ids:    Should source IDs be formatted as strings
                    (True) or a set of integers (False)?

                path_substitutions:    See get_source_count().

            Returns:
                numpy.array:
                    See ID field of result in get_source_data().
            """

            return self.get_source_data(
                string_source_ids=string_source_ids,
                magfit_iterations=[],
                shape_fit=False,
                apphot=False,
                shape_map_variables=False,
                background=False,
                position=False,
                **path_substitutions,
            ).index

        def add_magnitude_fitting(
            self,
            *,
            fitted_magnitudes,
            fit_statistics,
            magfit_configuration,
            missing_indices,
            **path_substitutions,
        ):
            """
            Add a magnitude fitting iteration to the DR file.

            Args:
                fitted_magnitudes(numpy.array):   The differential photometry
                    corrected magnitudes of the sources.

                fit_statistics(dict):    Summary statistics about how the fit went.
                    It should define at least the following keys:
                    ``initial_src_count``, ``final_src_count``, and ``residual``.

                magfit_configuration:    The configuration structure with which
                    magnitude fitting was performed.

                missing_indices:    A list of indices within the file of sources
                    for which no entries are included in fitted_magnitudes.

            Returns:
                None
            """

            def pad_missing_magnitudes():
                """Return fitted magnitudes with nans added at missing_indices."""

                if not missing_indices:
                    return fitted_magnitudes

                fitted_magnitudes_shape = list(fitted_magnitudes.shape)
                fitted_magnitudes_shape[0] += len(missing_indices)
                padded_fitted_magnitudes = numpy.empty(
                    shape=fitted_magnitudes_shape, dtype=fitted_magnitudes.dtype
                )
                padded_fitted_magnitudes[missing_indices] = numpy.nan
                padded_fitted_magnitudes[
                    [
                        ind not in missing_indices
                        for ind in range(fitted_magnitudes_shape[0])
                    ]
                ] = fitted_magnitudes
                return padded_fitted_magnitudes

            def add_magfit_datasets(fitted_magnitudes, include_shape_fit):
                """Create the datasets holding the newly fitted magnitudes."""

                def add_dataset(dset_key, dset_data, substitutions):

                    orig_path = self.add_dataset(
                        dset_key,
                        dset_data,
                        if_exists="error",
                        **substitutions,
                    )
                    path_template = self._file_structure[dset_key].abspath
                    for magfit_iter in range(
                        num_magfit_iterations,
                        path_substitutions["magfit_iteration"],
                    ):
                        self[
                            path_template
                            % {
                                **substitutions,
                                "magfit_iteration": magfit_iter,
                            }
                        ] = self[orig_path]

                num_apertures = fitted_magnitudes.shape[1]
                apphot_start = 0
                if include_shape_fit:
                    num_apertures -= 1
                    apphot_start = 1
                    add_dataset(
                        "shapefit.magfit.magnitude",
                        fitted_magnitudes[:, 0],
                        path_substitutions,
                    )
                for aperture_index in range(num_apertures):
                    add_dataset(
                        "apphot.magfit.magnitude",
                        fitted_magnitudes[:, aperture_index + apphot_start],
                        {**path_substitutions, "aperture_index": aperture_index},
                    )

            def add_attributes(include_shape_fit):
                """Add attributes with the magfit configuration."""

                for phot_index in range(fitted_magnitudes.shape[1]):

                    phot_method = (
                        "shapefit"
                        if include_shape_fit and phot_index == 0
                        else "apphot"
                    )

                    if phot_method == "apphot":
                        path_substitutions["aperture_index"] = (
                            path_substitutions.get("aperture_index", -1) + 1
                        )

                    if num_magfit_iterations == 0:
                        self.add_attribute(
                            phot_method + ".magfit.cfg.correction_type",
                            b"linear",
                            if_exists="error",
                            **path_substitutions,
                        )

                        for pipeline_key_end, config_attribute in [
                            ("correction", "correction_parametrization"),
                            ("require", "fit_source_condition"),
                            ("single_photref", "single_photref_dr_fname"),
                        ]:
                            self.add_attribute(
                                phot_method + ".magfit.cfg." + pipeline_key_end,
                                getattr(magfit_configuration, config_attribute),
                                if_exists="error",
                                **path_substitutions,
                            )

                        for config_param in [
                            "noise_offset",
                            "max_mag_err",
                            "rej_level",
                            "max_rej_iter",
                            "error_avg",
                        ]:
                            self.add_attribute(
                                phot_method + ".magfit.cfg." + config_param,
                                getattr(magfit_configuration, config_param),
                                if_exists="error",
                                **path_substitutions,
                            )

                    for pipeline_key_end, statistics_key in [
                        ("num_input_src", "initial_src_count"),
                        ("num_fit_src", "final_src_count"),
                        ("fit_residual", "residual"),
                    ]:
                        self.add_attribute(
                            phot_method + ".magfit." + pipeline_key_end,
                            fit_statistics[statistics_key][phot_index],
                            if_exists="error",
                            **path_substitutions,
                        )

            num_magfit_iterations = self.get_num_magfit_iterations(
                **path_substitutions
            )
            if "magfit_iteration" in path_substitutions:
                assert (
                    path_substitutions["magfit_iteration"] >= num_magfit_iterations
                )
            else:
                path_substitutions["magfit_iteration"] = num_magfit_iterations
            self._logger.debug(
                "Adding magfit iteration %d to %s containing %d prior iterations",
                path_substitutions["magfit_iteration"],
                self.filename,
                num_magfit_iterations,
            )
            include_shape_fit = self.has_shape_fit(
                accept_zeropsf=False, **path_substitutions
            )
            add_magfit_datasets(pad_missing_magnitudes(), include_shape_fit)

            add_attributes(include_shape_fit)

        def add_hat_astrometry(
            self, filenames, configuration, **path_substitutions
        ):
            """
            Add astrometry derived by fistar, and anmatch to the DR file.

            Args:
                filanemes(dict):    The files containing the astrometry results.
                    Should have the following keys: `'fistar'`, `'trans'`,
                    `'match'`, `'catalogue'`.

                configuration:    An object with attributes containing the
                    configuraiton of how astormetry was performed.

                path_substitutions:    See get_source_count()

            Returns:
                None
            """

            def add_match(extracted_sources, catalogue_sources):
                """Create dset of the matched indices from catalogue & extracted."""

                num_cat_columns = len(catalogue_sources.dtype.names)
                match_ids = numpy.genfromtxt(
                    filenames["match"],
                    dtype=None,
                    names=["cat_id", "extracted_id"],
                    usecols=(0, num_cat_columns),
                )
                extracted_sorter = numpy.argsort(extracted_sources["ID"])
                catalogue_sorter = numpy.argsort(catalogue_sources["ID"])
                match = numpy.empty([match_ids.size, 2], dtype=int)
                match[:, 0] = catalogue_sorter[
                    numpy.searchsorted(
                        catalogue_sources["ID"],
                        match_ids["cat_id"],
                        sorter=catalogue_sorter,
                    )
                ]
                match[:, 1] = extracted_sorter[
                    numpy.searchsorted(
                        extracted_sources["ID"],
                        match_ids["extracted_id"],
                        sorter=extracted_sorter,
                    )
                ]
                self.add_dataset(
                    dataset_key="skytoframe.matched",
                    data=match,
                    **path_substitutions,
                )

            def add_trans():
                """Create dsets/attrs describing the sky to frame transformation."""

                transformation, info = parse_anmatch_transformation(
                    filenames["trans"]
                )
                self.add_dataset(
                    dataset_key="skytoframe.coefficients",
                    data=numpy.stack(
                        (transformation["dxfit"], transformation["dyfit"])
                    ),
                    **path_substitutions,
                )
                for entry in ["type", "order", "offset", "scale"]:
                    self.add_attribute(
                        attribute_key="skytoframe." + entry,
                        attribute_value=transformation[entry],
                        **path_substitutions,
                    )
                for entry in ["residual", "unitarity"]:
                    self.add_attribute(
                        attribute_key="skytoframe." + entry,
                        attribute_value=info[entry],
                        **path_substitutions,
                    )
                self.add_attribute(
                    attribute_key="skytoframe.sky_center",
                    attribute_value=numpy.array(
                        [info["2mass"]["RA"], info["2mass"]["DEC"]]
                    ),
                    **path_substitutions,
                )

            def add_configuration():
                """Add the information about the configuration used."""

                for component, config_attribute in [
                    ("srcextract", "binning"),
                    ("catalogue", "name"),
                    ("catalogue", "epoch"),
                    ("catalogue", "filter"),
                    ("catalogue", "fov"),
                    ("catalogue", "orientation"),
                    ("skytoframe", "srcextract_filter"),
                    ("skytoframe", "sky_preprojection"),
                    ("skytoframe", "max_match_distance"),
                    ("skytoframe", "frame_center"),
                    ("skytoframe", "weights_expression"),
                ]:
                    if component == "catalogue":
                        value = getattr(
                            configuration, "astrom_catalogue_" + config_attribute
                        )
                    else:
                        value = getattr(
                            configuration, component + "_" + config_attribute
                        )
                    self.add_attribute(
                        component + ".cfg." + config_attribute,
                        value,
                        **path_substitutions,
                    )

            extracted_sources = numpy.genfromtxt(
                filenames["fistar"],
                names=[
                    "ID",
                    "x",
                    "y",
                    "Background",
                    "Amplitude",
                    "S",
                    "D",
                    "K",
                    "FWHM",
                    "Ellipticity",
                    "PositionAngle",
                    "Flux",
                    "SignalToNoise",
                    "NumberPixels",
                ],
                dtype=None,
            )
            catalogue_sources = numpy.genfromtxt(
                filenames["catalogue"], dtype=None, names=True, deletechars=""
            )
            catalogue_sources.dtype.names = [
                name.split("[", 1)[0] for name in catalogue_sources.dtype.names
            ]

            self.add_sources(
                extracted_sources,
                "srcextract.sources",
                "srcextract_column_name",
                **path_substitutions,
            )
            self.add_sources(
                catalogue_sources,
                "catalogue.columns",
                "catalogue_column_name",
                parse_ids=True,
                **path_substitutions,
            )
            add_match(extracted_sources, catalogue_sources)
            add_trans()
            add_configuration()

        def get_matched_sources(self, **path_substitutions):
            """Get combined catalogue and extracted matched sources."""

            match = self.get_dataset(
                dataset_key="skytoframe.matched", **path_substitutions
            )

            catalogue = (
                self.get_sources(
                    "catalogue.columns",
                    "catalogue_column_name",
                    **path_substitutions,
                )
                .iloc[match[:, 0]]
                .reset_index()
            )
            extracted_sources = (
                self.get_sources(
                    "srcextract.sources",
                    "srcextract_column_name",
                    **path_substitutions,
                )
                .iloc[match[:, 1]]
                .reset_index()
            )
            return pandas.concat([catalogue, extracted_sources], axis=1)

        def save_source_extracted_psf_map(
            self, *, fit_results, fit_configuration, **path_substitutions
        ):
            """Create the datasets and attributes holding the fit results."""

            psf_parameters = fit_results["coefficients"].keys()
            self._logger.debug(
                "Writing the following data to srcextract.psf_map dataset: %s",
                repr(
                    [
                        fit_results["coefficients"][param_name]
                        for param_name in psf_parameters
                    ]
                ),
            )
            self.add_dataset(
                "srcextract.psf_map",
                numpy.stack(
                    [
                        fit_results["coefficients"][param_name]
                        for param_name in psf_parameters
                    ]
                ),
                **path_substitutions,
            )

            for param_key, param_value in [
                (
                    "cfg.psf_params",
                    numpy.array([name.encode("ascii") for name in psf_parameters]),
                ),
                (
                    "cfg.terms",
                    fit_configuration.fit_terms_expression.encode("ascii"),
                ),
                (
                    "cfg.weights",
                    (
                        b"none"
                        if fit_configuration.weights_expression is None
                        else fit_configuration.weights_expression.encode("ascii")
                    ),
                ),
                ("cfg.error_avg", fit_configuration.error_avg.encode("ascii")),
                ("cfg.rej_level", fit_configuration.rej_level),
                ("cfg.max_rej_iter", fit_configuration.max_rej_iter),
                (
                    "residual",
                    numpy.array(
                        [
                            fit_results["fit_res2"][param_name] ** 0.5
                            for param_name in psf_parameters
                        ]
                    ),
                ),
                (
                    "num_fit_src",
                    numpy.array(
                        [
                            fit_results["num_fit_src"][param_name]
                            for param_name in psf_parameters
                        ]
                    ),
                ),
            ]:
                self.add_attribute(
                    "srcextract.psf_map." + param_key,
                    param_value,
                    **path_substitutions,
                )

        # pylint: enable=too-many-locals
        # pylint: enable=too-many-statements

    # pylint: enable=too-many-ancestors
    # pylint: enable=too-many-public-methods