# -*- coding: utf-8 -*-
# Copyright 2025 Matthew Fitzpatrick.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 3.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <https://www.gnu.org/licenses/gpl-3.0.html>.
"""For training machine learning models for distortion estimation in CBED.

"""



#####################################
## Load libraries/packages/modules ##
#####################################

# For timing the execution of different segments of code.
import time



# Contains the majority of the implementation code for this module, which is
# also shared with other modules.
import emicroml.modelling.cbed.distortion._common



##################################
## Define classes and functions ##
##################################

# List of public objects in module.
__all__ = ["DefaultDistortionModelGenerator",
           "DefaultCBEDPatternGenerator",
           "generate_and_save_ml_dataset"
           "combine_ml_dataset_files",
           "split_ml_dataset_file",
           "MLDataset",
           "ml_data_dict_to_distortion_models",
           "ml_data_dict_to_signals",
           "MLDatasetManager",
           "MLModel",
           "normalize_normalizable_elems_in_ml_data_dict",
           "unnormalize_normalizable_elems_in_ml_data_dict",
           "MLModelTrainer",
           "MLModelTester",
           "load_ml_model_from_file",
           "load_ml_model_from_state_dict"]



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_reference_pt = \
    _module_alias._default_reference_pt
_default_rng_seed = \
    _module_alias._default_rng_seed
_default_sampling_grid_dims_in_pixels = \
    _module_alias._default_sampling_grid_dims_in_pixels
_default_least_squares_alg_params = \
    _module_alias._default_least_squares_alg_params
_default_device_name = \
    _module_alias._default_device_name
_default_skip_validation_and_conversion = \
    _module_alias._default_skip_validation_and_conversion



_module_alias = emicroml.modelling.cbed.distortion._common
_cls_alias = _module_alias._DefaultDistortionModelGenerator
class DefaultDistortionModelGenerator(_cls_alias):
    r"""The default class of random distortion model generators.

    The current class is a subclass of
    :class:`fancytypes.PreSerializableAndUpdatable`.

    The current class represents the default random distortion model generators
    used to generate random "fake" CBED patterns. The current class is used in
    the class
    :class:`emicroml.modelling.cbed.distortion.estimation.DefaultCBEDPatternGenerator`,
    with the latter class representing the default random fake CBED pattern
    generators. See the documentation for the latter class for further
    discussion on the default random fake CBED pattern generators.

    A random number generator is used to generate random distortion models. Upon
    construction of an instance of the current class, or core attribute update
    via the method :meth:`~fancytypes.Updatable.update`, a random numpy
    generator ``rng`` is constructed via ``import numpy;
    rng=numpy.random.default_rng(rng_seed)``, where ``rng_seed`` is the
    construction parameter or core attribute that specifies the seed used in the
    random number generator. See the documentation for the class
    :attr:`~fancytypes.Checkable.core_attrs` for a discussion on core
    attributes.

    The distortion models generated by instances of the current class are
    "standard", meaning that the corresponding coordinate transformation
    :math:`T_{⌑;x}\left(u_{x},u_{y}\right)` that describes the optical
    distortions can be specified equivalently by an instance
    ``standard_coord_transform_params`` of
    :class:`distoptica.StandardCoordTransformParams`. The distortion models are
    instances of the class :class:`distoptica.DistortionModel`. See the
    documentation for the class :class:`distoptica.DistortionModel` for further
    discussion on how the optical distortions are modelled.

    The construction parameters of
    :class:`distoptica.StandardCoordTransformParams` that specify
    :math:`T_{⌑;x}\left(u_{x},u_{y}\right)` are ``center``,
    ``quadratic_radial_distortion_amplitude``, ``elliptical_distortion_vector``,
    ``spiral_distortion_amplitude``, and ``parabolic_distortion_vector``. For
    each candidate distortion model, these parameters are calculated by

    .. code-block:: python

        import numpy as np

        r_c_D = rng.normal(loc=0, scale=1/20)
        phi_c_D = rng.uniform(low=0, high=2*np.pi)
        x_c_D = reference_pt[0] + r_c_D*np.cos(phi_c_D)
        y_c_D = reference_pt[1] + r_c_D*np.sin(phi_c_D)
        center = (x_c_D, y_c_D)

        quadratic_radial_distortion_amplitude = rng.uniform(low=-0.5, high=2)
        spiral_distortion_amplitude = rng.uniform(low=-1.5, high=1.5)

        amplitude = rng.uniform(low=0, high=0.2)
        phase = rng.uniform(low=0, high=np.pi)
        elliptical_distortion_vector = (amplitude*np.cos(2*phase),
                                        amplitude*np.sin(2*phase))

        amplitude = rng.uniform(low=0, high=0.4)
        phase = rng.uniform(low=0, high=2*np.pi)
        parabolic_distortion_vector = (amplitude*np.cos(phase),
                                       amplitude*np.sin(phase))

    where ``reference_pt`` is a pair of floating-point numbers that users
    specify as a construction parameter of the current class.

    If the floating-point numbers stored in the instance attribute
    :attr:`distoptica.DistortionModel.mask_frame_of_distorted_then_resampled_images`
    of the resulting candidate distortion model are less than or equal to
    ``1/8``, then the candidate distortion model is accepted as valid and
    returned as output after calling the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.DefaultDistortionModelGenerator.generate`.
    Otherwise, the candidate distortion model is rejected, and new candidate
    distortion models are generated until either: one of them is accepted as
    valid; or 10 candidate distortion models have been rejected in total. In the
    latter case, an exception is raised. See the documentation for the attribute
    :attr:`distoptica.DistortionModel.mask_frame_of_distorted_then_resampled_images`
    for a description of said attribute.

    Parameters
    ----------
    reference_pt : `array_like` (`float`, shape=(2,)), optional
        A reference point from which to randomly generate distortion centers. 
        See the summary documentation above for context.
    rng_seed : `int` | `None`, optional
        ``rng_seed`` specifies the seed used in the random number generator.
    sampling_grid_dims_in_pixels : `array_like` (`int`, shape=(2,)), optional
        The dimensions of the sampling grid, in units of pixels, used for
        all distortion models.
    least_squares_alg_params : :class:`distoptica.LeastSquaresAlgParams` | `None`, optional
        ``least_squares_alg_params`` specifies the parameters of the
        least-squares algorithm to be used to calculate the mappings of
        fractional Cartesian coordinates of distorted images to those of the
        corresponding undistorted images. If ``least_squares_alg_params`` is set
        to ``None``, then the parameter will be reassigned to the value
        ``distoptica.LeastSquaresAlgParams()``. See the documentation for the
        class :class:`distoptica.LeastSquaresAlgParams` for details on the
        parameters of the least-squares algorithm.
    device_name : `str` | `None`, optional
        This parameter specifies the device to be used to perform
        computationally intensive calls to PyTorch functions and where to store
        attributes of the type :class:`torch.Tensor` for each distortion model
        represented by the class :class:`distoptica.DistortionModel`. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.
    skip_validation_and_conversion : `bool`, optional
        Let ``validation_and_conversion_funcs`` and ``core_attrs`` denote the
        attributes :attr:`~fancytypes.Checkable.validation_and_conversion_funcs`
        and :attr:`~fancytypes.Checkable.core_attrs` respectively, both of which
        being `dict` objects.

        Let ``params_to_be_mapped_to_core_attrs`` denote the `dict`
        representation of the constructor parameters excluding the parameter
        ``skip_validation_and_conversion``, where each `dict` key ``key`` is a
        different constructor parameter name, excluding the name
        ``"skip_validation_and_conversion"``, and
        ``params_to_be_mapped_to_core_attrs[key]`` would yield the value of the
        constructor parameter with the name given by ``key``.

        If ``skip_validation_and_conversion`` is set to ``False``, then for each
        key ``key`` in ``params_to_be_mapped_to_core_attrs``,
        ``core_attrs[key]`` is set to ``validation_and_conversion_funcs[key]
        (params_to_be_mapped_to_core_attrs)``.

        Otherwise, if ``skip_validation_and_conversion`` is set to ``True``,
        then ``core_attrs`` is set to
        ``params_to_be_mapped_to_core_attrs.copy()``. This option is desired
        primarily when the user wants to avoid potentially expensive deep copies
        and/or conversions of the `dict` values of
        ``params_to_be_mapped_to_core_attrs``, as it is guaranteed that no
        copies or conversions are made in this case.

    """
    def __init__(self,
                 reference_pt=\
                 _default_reference_pt,
                 rng_seed=\
                 _default_rng_seed,
                 sampling_grid_dims_in_pixels=\
                 _default_sampling_grid_dims_in_pixels,
                 least_squares_alg_params=\
                 _default_least_squares_alg_params,
                 device_name=\
                 _default_device_name,
                 skip_validation_and_conversion=\
                 _default_skip_validation_and_conversion):
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        module_alias = emicroml.modelling.cbed.distortion._common
        cls_alias = module_alias._DefaultDistortionModelGenerator
        cls_alias.__init__(self, **kwargs)

        return None


_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_num_pixels_across_each_cbed_pattern = \
    _module_alias._default_num_pixels_across_each_cbed_pattern
_default_max_num_disks_in_any_cbed_pattern = \
    _module_alias._default_max_num_disks_in_any_cbed_pattern



_module_alias = emicroml.modelling.cbed.distortion._common
_cls_alias = _module_alias._DefaultCBEDPatternGenerator
class DefaultCBEDPatternGenerator(_cls_alias):
    r"""The default class of random "fake" CBED pattern generators.

    The current class is a subclass of
    :class:`fancytypes.PreSerializableAndUpdatable`.

    The current class represents the default random "fake" CBED pattern
    generators used to generate random fake CBED patterns, where each fake CBED
    pattern is represented by an instance of the class
    :class:`fakecbed.discretized.CBEDPattern`. See the documentation for the
    class :class:`fakecbed.discretized.CBEDPattern` for a discussion on how
    fake CBED patterns are modelled/parameterized.

    A random number generator is used to fake CBED patterns. Upon construction
    of an instance of the current class, or core attribute update via the method
    :meth:`~fancytypes.Updatable.update`, a random numpy generator ``rng`` is
    constructed via ``import numpy; rng=numpy.random.default_rng(rng_seed)``,
    where ``rng_seed`` is the construction parameter or core attribute that
    specifies the seed used in the random number generator. See the
    documentation for the class :attr:`~fancytypes.Checkable.core_attrs` for a
    discussion on core attributes.

    Instances of the current class use instances of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.DefaultDistortionModelGenerator`
    to generate random distortion models, which are subsequently used to
    generate random fake CBED patterns.

    The randomization scheme employed by the current class to generate random
    fake CBED patterns is somewhat convoluted, and will not be documented here
    in detail. For those who are interested, you can parse through the source
    code of the current class for more details on the scheme.

    Parameters
    ----------
    num_pixels_across_each_cbed_pattern : `int`, optional
        The number of pixels across each fake CBED pattern to be generated. Note
        that the number of pixels from top to bottom is also equal to
        ``num_pixels_across_each_cbed_pattern`` for each fake CBED pattern to be
        generated. Moreover, the parameter
        ``num_pixels_across_each_cbed_pattern`` is expected to be a positive
        integer that is divisible by ``2**5``.
    max_num_disks_in_any_cbed_pattern : `int`, optional
        The maximum number of CBED disks to appear in the image of any fake CBED
        pattern to be generated.
    rng_seed : `int` | `None`, optional
        ``rng_seed`` specifies the seed used in the random number generator.
    sampling_grid_dims_in_pixels : `array_like` (`int`, shape=(2,)), optional
        The dimensions of the sampling grid, in units of pixels, used for
        all distortion models.
    least_squares_alg_params : :class:`distoptica.LeastSquaresAlgParams` | `None`, optional
        ``least_squares_alg_params`` specifies the parameters of the
        least-squares algorithm to be used to calculate the mappings of
        fractional Cartesian coordinates of distorted images to those of the
        corresponding undistorted images. ``least_squares_alg_params`` is used
        to calculate the interim distortion models mentioned above in the
        summary documentation. If ``least_squares_alg_params`` is set to
        ``None``, then the parameter will be reassigned to the value
        ``distoptica.LeastSquaresAlgParams()``. See the documentation for the
        class :class:`distoptica.LeastSquaresAlgParams` for details on the
        parameters of the least-squares algorithm.
    device_name : `str` | `None`, optional
        This parameter specifies the device to be used to perform
        computationally intensive calls to PyTorch functions and where to store
        attributes of the type :class:`torch.Tensor` for each fake CBED pattern
        represented by the class :class:`fakecbed.discretized.CBEDPattern`. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.
    skip_validation_and_conversion : `bool`, optional
        Let ``validation_and_conversion_funcs`` and ``core_attrs`` denote the
        attributes :attr:`~fancytypes.Checkable.validation_and_conversion_funcs`
        and :attr:`~fancytypes.Checkable.core_attrs` respectively, both of which
        being `dict` objects.

        Let ``params_to_be_mapped_to_core_attrs`` denote the `dict`
        representation of the constructor parameters excluding the parameter
        ``skip_validation_and_conversion``, where each `dict` key ``key`` is a
        different constructor parameter name, excluding the name
        ``"skip_validation_and_conversion"``, and
        ``params_to_be_mapped_to_core_attrs[key]`` would yield the value of the
        constructor parameter with the name given by ``key``.

        If ``skip_validation_and_conversion`` is set to ``False``, then for each
        key ``key`` in ``params_to_be_mapped_to_core_attrs``,
        ``core_attrs[key]`` is set to ``validation_and_conversion_funcs[key]
        (params_to_be_mapped_to_core_attrs)``.

        Otherwise, if ``skip_validation_and_conversion`` is set to ``True``,
        then ``core_attrs`` is set to
        ``params_to_be_mapped_to_core_attrs.copy()``. This option is desired
        primarily when the user wants to avoid potentially expensive deep copies
        and/or conversions of the `dict` values of
        ``params_to_be_mapped_to_core_attrs``, as it is guaranteed that no
        copies or conversions are made in this case.

    """
    def __init__(self,
                 num_pixels_across_each_cbed_pattern=\
                 _default_num_pixels_across_each_cbed_pattern,
                 max_num_disks_in_any_cbed_pattern=\
                 _default_max_num_disks_in_any_cbed_pattern,
                 rng_seed=\
                 _default_rng_seed,
                 sampling_grid_dims_in_pixels=\
                 _default_sampling_grid_dims_in_pixels,
                 least_squares_alg_params=\
                 _default_least_squares_alg_params,
                 device_name=\
                 _default_device_name,
                 skip_validation_and_conversion=\
                 _default_skip_validation_and_conversion):
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        module_alias = emicroml.modelling.cbed.distortion._common
        cls_alias = module_alias._DefaultCBEDPatternGenerator
        cls_alias.__init__(self, **kwargs)

        return None



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_num_cbed_patterns = \
    _module_alias._default_num_cbed_patterns
_default_cbed_pattern_generator = \
    _module_alias._default_cbed_pattern_generator
_default_output_filename = \
    _module_alias._default_output_filename
_default_max_num_ml_data_instances_per_file_update = \
    _module_alias._default_max_num_ml_data_instances_per_file_update



def generate_and_save_ml_dataset(
        num_cbed_patterns=\
        _default_num_cbed_patterns,
        max_num_disks_in_any_cbed_pattern=\
        _default_max_num_disks_in_any_cbed_pattern,
        cbed_pattern_generator=\
        _default_cbed_pattern_generator,
        output_filename=\
        _default_output_filename,
        max_num_ml_data_instances_per_file_update=\
        _default_max_num_ml_data_instances_per_file_update):
    r"""Generate a machine learning dataset.

    According to the parameters described below, the current function generates
    a file storing a machine learning (ML) dataset that can be used to train
    and/or evaluate ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`.

    The number of ML data instances to be generated is specified by the
    parameter ``num_cbed_patterns``. Each ML data instance is derived from a
    "fake" CBED pattern, with each fake CBED pattern being generated from a fake
    CBED pattern generator that is specified by the parameter
    ``cbed_pattern_generator``. The maximum number of (fake) CBED disks that can
    appear in any generated fake CBED pattern is specified by the parameter
    ``max_num_disks_in_any_cbed_pattern``.

    ``cbed_pattern_generator`` can be set to either ``None``, or any object that
    satisfies the following:

    1. ``cbed_pattern_generator`` must have a method called ``generate`` which
    returns an instance of the class :class:`fakecbed.discretized.CBEDPattern`
    upon calling said method via ``cbed_pattern_generator.generate()``.

    2. For each object ``fake_cbed_pattern`` returned by
    ``cbed_pattern_generator.generate()``,
    ``fake_cbed_pattern.core_attrs["num_pixels_across_pattern"]`` must yield the
    same integer value ``num_pixels_across_each_pattern``.

    3. For each object ``fake_cbed_pattern`` returned by
    ``cbed_pattern_generator.generate()``,
    ``fake_cbed_pattern.core_attrs["undistorted_disks"]`` must be a nonempty
    sequence ``undistorted_disks`` where for each element ``undistorted_disk``
    of the sequence, ``undistorted_disk.core_attrs["support"]`` must yield an
    instance ``undistorted_disk_support`` of the class
    :class:`fakecbed.shapes.Circle`, with
    ``undistorted_disk_support.core_attrs["radius"]`` yielding a positive number
    ``common_undistorted_disk_radius``. The number
    ``common_undistorted_disk_radius`` has the same value for all elements of
    the sequence ``undistorted_disks`` of the same object
    ``fake_cbed_pattern``. From one object ``fake_cbed_pattern`` returned by
    ``cbed_pattern_generator.generate()`` to another, the value of
    ``common_undistorted_disk_radius`` can change. Further below we refer to
    ``common_undistorted_disk_radius`` as the common undistorted disk radius.

    4. For each object ``fake_cbed_pattern`` returned by
    ``cbed_pattern_generator.generate()``,
    ``fake_cbed_pattern.core_attrs["distortion_model"].is_standard`` must yield
    ``True``.

    5. For each object ``fake_cbed_pattern`` returned by
    ``cbed_pattern_generator.generate()``,
    ``(~fake_cbed_pattern.disk_absence_registry).sum().item()`` must yield an
    integer less than or equal to ``max_num_disks_in_any_cbed_pattern``.

    If ``cbed_pattern_generator`` is set to ``None``, then the parameter will be
    reassigned to the value of
    ``emicroml.modelling.cbed.distortion.estimation.DefaultCBEDPatternGenerator()``,
    which satisfies the same conditions described above.

    As alluded to above, each valid object ``fake_cbed_pattern`` returned by
    ``cbed_pattern_generator.generate()`` stores an instance
    ``distortion_model`` of the class :class:`distoptica.DistortionModel`,
    accessed by
    ``fake_cbed_pattern.core_attrs["distortion_model"]``. ``distortion_model``
    is the distortion model that determines that distortion field applied to the
    fake CBED pattern represented by ``fake_cbed_pattern``. See the
    documentation for :class:`distoptica.DistortionModel` for additional
    context. As implied above, ``distortion_model`` is a "standard" distortion
    model, meaning that the corresponding coordinate transformation
    :math:`T_{⌑;x}\left(u_{x},u_{y}\right)` that describes the optical
    distortions can be specified equivalently by an instance
    ``standard_coord_transform_params`` of
    :class:`distoptica.StandardCoordTransformParams`.
    ``standard_coord_transform_params`` is the standard coordinate
    transformation parameter set of the fake CBED pattern. As discussed in the
    documentation for the class
    :class:`distoptica.StandardCoordTransformParams`, each instance of said
    class specifies a distortion center :math:`\left(x_{c;D},y_{c;D}\right)`, a
    quadratic radial distortion amplitude :math:`A_{r;0,2}`, an elliptical
    distortion vector :math:`\left(A_{r;2,0},B_{r;1,0}\right)`, a spiral
    distortion amplitude :math:`A_{t;0,2}`, and a parabolic distortion vector
    :math:`\left(A_{r;1,1},B_{r;0,1}\right)`.

    As alluded to above, each valid object ``fake_cbed_pattern`` returned by
    ``cbed_pattern_generator.generate()`` stores a nonempty sequence
    ``undistorted_disks``, accessed by
    ``fake_cbed_pattern.core_attrs["undistorted_disks"]``. For every nonnegative
    integer ``k`` less than ``fake_cbed_pattern.num_disks``,
    ``undistorted_disks[k]`` specifies the intensity pattern of the ``k`` th
    undistorted fake CBED disk of the fake CBED pattern represented by
    ``fake_cbed_pattern``. The center of the ``k`` th undistorted fake CBED disk
    can be accessed by
    ``undistorted_disks[k].core_attrs["support"].core_attrs["center"]``. During
    the process of deriving an ML data instance from ``fake_cbed_pattern``, the
    intra-disk averages of the distorted fake CBED disks of the fake CBED
    pattern are calculated, where the ``k`` th distorted fake CBED disk
    corresponds to the ``k`` th undistorted fake CBED disk, i.e. the former is
    obtained by distorting the latter. The intra-disk average
    ``kth_intra_disk_avg`` of the ``k`` th distorted fake CBED disk is
    calculated by

    .. code-block:: python

        kth_intra_disk_sum = (fake_cbed_pattern.image
                              * fake_cbed_pattern.disk_supports[k]).sum().item()

        kth_disk_area = (fake_cbed_pattern.disk_supports[k].sum().item()
                         / (fake_cbed_pattern.image.shape[0]**2))

        if (kth_disk_area > 0):
            kth_intra_disk_avg = kth_intra_disk_sum/kth_disk_area
        else:
            kth_intra_disk_avg = 0

    We reference intra-disk averages again further below.

    The ML data instances generated by the current function are stored in an
    HDF5 file, which has the following file structure:

    - cbed_pattern_images: <HDF5 3D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "row"
        + dim_2: "col"

    - disk_overlap_maps: <HDF5 3D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "row"
        + dim_2: "col"

    - disk_objectness_sets: <HDF5 2D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "disk idx"

    - disk_clipping_registries: <HDF5 2D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "disk idx"

    - undistorted_disk_center_sets: <HDF5 3D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "disk idx"
        + dim_2: "vector cmpnt idx [0->x, 1->y]"
        + normalization_weight: <float>
        + normalization_bias: <float>

    - common_undistorted_disk_radii: <HDF5 1D dataset>
    
        + dim_0: "cbed pattern idx"
        + normalization_weight: <float>
        + normalization_bias: <float>

    - distortion_centers: <HDF5 2D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "vector cmpnt idx [0->x, 1->y]"
        + normalization_weight: <float>
        + normalization_bias: <float>

    - quadratic_radial_distortion_amplitudes: <HDF5 1D dataset>
    
        + dim_0: "cbed pattern idx"
        + normalization_weight: <float>
        + normalization_bias: <float>

    - spiral_distortion_amplitudes: <HDF5 1D dataset>
    
        + dim_0: "cbed pattern idx"
        + normalization_weight: <float>
        + normalization_bias: <float>

    - elliptical_distortion_vectors: <HDF5 2D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "vector cmpnt idx [0->x, 1->y]"
        + normalization_weight: <float>
        + normalization_bias: <float>

    - parabolic_distortion_vectors: <HDF5 2D dataset>
    
        + dim_0: "cbed pattern idx"
        + dim_1: "vector cmpnt idx [0->x, 1->y]"
        + normalization_weight: <float>
        + normalization_bias: <float>

    Note that the sub-bullet points listed immediately below a given HDF5
    dataset display the HDF5 attributes associated with said HDF5 dataset. Each
    HDF5 dataset has a set of attributes with names of the form
    ``"dim_{}".format(i)`` with ``i`` being an integer ranging from 0 to the
    rank of said HDF5 dataset minus 1. Attribute ``"dim_{}".format(i)`` of a
    given HDF5 dataset labels the ``i`` th dimension of the underlying array of
    the dataset. The ``"cbed pattern idx"`` dimension is of the size
    ``num_cbed_patterns``, the ``"row"`` dimension is of the size
    ``num_pixels_across_each_pattern``, the ``"col"`` dimension is of the size
    ``num_pixels_across_each_pattern``, the ``"disk idx"`` dimension is of the
    size ``max_num_disks_in_any_cbed_pattern``, and the ``"vector cmpnt idx
    [0->x, 1->y]"`` is of the size ``2``.

    The HDF5 datasets that have attributes named ``"normalization_weight"`` and
    ``"normalization_bias"`` are min-max normalized, and are referred to as
    "normalizable". Let ``hdf5_dataset`` be the numerical data of such an HDF5
    dataset. Furthermore, let ``normalization_weight`` and
    ``normalization_bias`` be the values stored in the attributes
    ``"normalization_weight"`` and ``"normalization_bias"`` of said HDF5 dataset
    respectively. ``hdf5_dataset`` in this scenario is already min-max
    normalized. To reverse the normalization, i.e. to unnormalize the data,
    simply calculate ``(hdf5_dataset-normalization_bias) /
    normalization_weight``.

    We describe below how the data of the HDF5 datasets are calculated
    effectively.

    1. Set ``N`` to ``num_pixels_across_each_pattern``.

    2. Set ``cbed_pattern_images`` to ``np.zeros((num_cbed_patterns, N, N))``,
    where ``np`` is an alias for the NumPy library ``numpy``.

    3. Set ``disk_overlap_maps`` to ``np.zeros((num_cbed_patterns, N, N),
    dtype="int")``.

    4. Set ``disk_objectness_sets`` to ``np.zeros((num_cbed_patterns,
    max_num_disks_in_any_cbed_pattern))``.

    5. Set ``disk_clipping_registries`` to ``np.zeros((num_cbed_patterns,
    max_num_disks_in_any_cbed_pattern), dtype="bool")``.

    6. Set ``undistorted_disk_center_sets`` to ``np.zeros((num_cbed_patterns,
    max_num_disks_in_any_cbed_pattern, 2))``.

    7. Set ``common_undistorted_disk_radii`` to
    ``np.zeros((num_cbed_patterns,))``.

    8. Set ``distortion_centers`` to ``np.zeros((num_cbed_patterns, 2))``.

    9. Set ``quadratic_radial_distortion_amplitudes`` to
    ``np.zeros((num_cbed_patterns,))``.

    10. Set ``spiral_distortion_amplitudes`` to
    ``np.zeros((num_cbed_patterns,))``.

    11. Set ``elliptical_distortion_vectors`` to ``np.zeros((num_cbed_patterns,
    2))``.

    12. Set ``parabolic_distortion_vectors`` to ``np.zeros((num_cbed_patterns,
    2))``.

    13. Set ``cbed_pattern_idx`` to ``-1``.

    14. Set ``cbed_pattern_idx`` to ``cbed_pattern_idx+1``.

    15. Set ``fake_cbed_pattern`` to ``cbed_pattern_generator.generate()``.

    16. Store ``fake_cbed_pattern.image.numpy(force=True)`` in
    ``cbed_pattern_images[cbed_pattern_idx]``.

    17. Store ``fake_cbed_pattern.disk_overlap_map.numpy(force=True)`` in
    ``disk_overlap_maps[cbed_pattern_idx]``.

    18. Set ``intra_disk_avgs`` to ``np.zeros((fake_cbed_pattern.num_disks,))``.

    19. Set ``num_elems_to_pad`` to ``max_num_disks_in_any_cbed_pattern -
    fake_cbed_pattern.num_disks``.

    20. Set ``single_dim_slice`` to ``slice(0,
    max_num_disks_in_any_cbed_pattern)``.

    21. For every nonnegative integer ``k`` less than
    ``fake_cbed_pattern.num_disks``, store the intra-disk average of the ``k``
    th distorted fake CBED disk of `fake_cbed_pattern`` in
    ``intra_disk_avgs[k]``.

    22. Set ``new_disk_order`` to ``np.argsort(intra_disk_avgs)[::-1]``.

    23. Set ``disk_objectness_set`` to ``(intra_disk_avgs >
    0).astype("float")``.

    24. Set ``disk_objectness_set`` to ``disk_objectness_set[new_disk_order]``.

    25. Pad ``num_elems_to_pad`` times ``0`` to the end of the zeroth axis of
    ``disk_objectness_set``.

    26. Set ``disk_objectness_set`` to
    ``disk_objectness_set[single_dim_slice]``.

    27. Store ``disk_objectness_set`` in
    ``disk_objectness_sets[cbed_pattern_idx]``.

    28. Set ``disk_clipping_registry`` to
    ``fake_cbed_pattern.disk_clipping_registry.numpy(force=True)``.

    29. Pad ``num_elems_to_pad`` times ``0`` to the end of the zeroth axis of
    ``disk_clipping_registry``.

    30. Set ``disk_clipping_registry`` to
    ``disk_clipping_registry[single_dim_slice]``.

    31. Store ``disk_clipping_registry`` in
    ``disk_clipping_registries[cbed_pattern_idx]``.

    32. Set ``undistorted_disk_center_set`` to
    ``np.ones((fake_cbed_pattern.num_disks, 2))/2``.

    33. For every nonnegative integer ``k`` less than
    ``fake_cbed_pattern.num_disks``, if ``intra_disk_avgs[k]>0`` then store the
    center of the ``k`` th undistorted fake CBED disk of ``fake_cbed_pattern``
    in ``undistorted_disk_center_set[k]``.

    34. Pad ``num_elems_to_pad`` times ``0.5`` to the end of the zeroth axis of
    ``undistorted_disk_center_set``.

    35. Set ``undistorted_disk_center_set`` to
    ``undistorted_disk_center_set[single_dim_slice]``.

    36. Store ``undistorted_disk_center_set`` in
    ``undistorted_disk_center_sets[cbed_pattern_idx]``.

    37. Store the common undistorted disk radius of ``fake_cbed_pattern`` in
    ``common_undistorted_disk_radii[cbed_pattern_idx]``.

    38. Store the distortion center of ``fake_cbed_pattern`` in
    ``distortion_centers[cbed_pattern_idx]``.

    39. Store the quadratic radial distortion amplitude of ``fake_cbed_pattern``
    in ``quadratic_radial_distortion_amplitudes[cbed_pattern_idx]``.

    40. Store the spiral distortion amplitude of ``fake_cbed_pattern`` in
    ``spiral_distortion_amplitudes[cbed_pattern_idx]``.

    41. Store the elliptical distortion vector of ``fake_cbed_pattern`` in
    ``elliptical_distortion_vectors[cbed_pattern_idx]``.

    42. Store the parabolic distortion vector of ``fake_cbed_pattern`` in
    ``parabolic_distortion_vectors[cbed_pattern_idx]``.

    43. If ``cbed_pattern_idx < num_cbed_patterns-1``, then go to instruction
    14. Otherwise, go to instruction 44.

    44. Min-max normalized all normalizable HDF5 datasets.

    45. Stop.

    Parameters
    ----------
    num_cbed_patterns : `int`, optional
        The number of images of fake CBED patterns to generate and store in the
        machine learning (ML) dataset.
    max_num_disks_in_any_cbed_pattern : `int`, optional
        The maximum number of CBED disks to appear in the image of any fake CBED
        pattern to be generated.
    cbed_pattern_generator : `any_fake_cbed_pattern_generator` | `None`, optional
        ``cbed_pattern_generator`` specifies the fake CBED pattern generator to 
        be used.
    output_filename : `str`, optional
        The relative or absolute filename of the HDF5 file to which to store the
        ML dataset to be generated.
    max_num_ml_data_instances_per_file_update : `int`, optional
        The number of ML data instances to write to file per file update. The
        larger the value, the larger the memory requirements.

    """
    params = locals()
    params["start_time"] = time.time()

    global_symbol_table = globals()

    func_name = "_check_and_convert_generate_and_save_ml_dataset_params"
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    func_alias(**kwargs)

    return None



def _check_and_convert_generate_and_save_ml_dataset_params(params):
    module_alias = \
        emicroml.modelling.cbed.distortion._common
    func_alias = \
        module_alias._check_and_convert_generate_and_save_ml_dataset_params
    params = \
        func_alias(params)

    return params



def _generate_and_save_ml_dataset(max_num_disks_in_any_cbed_pattern,
                                  cbed_pattern_generator,
                                  max_num_ml_data_instances_per_file_update,
                                  num_cbed_patterns,
                                  output_filename,
                                  start_time):
    kwargs = locals()
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._generate_and_save_ml_dataset
    func_alias(**kwargs)

    return None



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_output_ml_dataset_filename = \
    _module_alias._default_output_ml_dataset_filename
_default_rm_input_ml_dataset_files = \
    _module_alias._default_rm_input_ml_dataset_files



def combine_ml_dataset_files(
        input_ml_dataset_filenames,
        output_ml_dataset_filename=\
        _default_output_ml_dataset_filename,
        rm_input_ml_dataset_files=\
        _default_rm_input_ml_dataset_files,
        max_num_ml_data_instances_per_file_update=\
        _default_max_num_ml_data_instances_per_file_update):
    r"""Combine files storing machine learning datasets.

    The current function copies the machine learning (ML) data instances stored
    in a set of input HDF5 files, and stores all those copies into a single new
    output HDF5 file. 

    The input HDF5 files and the output HDF5 file are assumed to have the same
    file structure as an HDF5 file generated by the function
    :func:`emicroml.modelling.cbed.distortion.estimation.generate_and_save_ml_dataset`.
    See the documentation of said function for a description of the file
    structure. Moreover, the input HDF5 files are assumed to have been created
    in a manner that is consistent with the way HDF5 files are generated by the
    function
    :func:`emicroml.modelling.cbed.distortion.estimation.generate_and_save_ml_dataset`.

    As discussed in the aforementioned documentation, some of the HDF5 datasets
    are normalizable. Prior to combining all the copies of the input ML data
    instances into a single new output HDF5 file, the copies of the normalizable
    input HDF5 datasets are unnormalized. After combining the input ML data
    instances into the new output HDF5 file, the normalizable HDF5 datasets
    therein are min-max normalized, only this time with respect to all ML data
    instances.

    Parameters
    ----------
    input_ml_dataset_filenames : `array_like` (`str`, ndim=1), optional
        The relative or absolute filenames of the input HDF5 files storing the
        ML datasets of interest.
    output_ml_dataset_filename : `str`, optional
        The relative or absolute filename of the output HDF5 file.
    rm_input_ml_dataset_files : `bool`, optional
        If ``rm_input_ml_dataset_files`` is set to ``True``, then the input HDF5
        files are deleted after all the ML data instances stored in those input
        HDF5 files are copied into the output HDF5 file.
    max_num_ml_data_instances_per_file_update : `int`, optional
        The number of ML data instances to write to the output file per file
        update. The larger the value, the larger the memory requirements.

    """
    params = locals()
    params["start_time"] = time.time()

    global_symbol_table = globals()

    func_name = "_check_and_convert_combine_ml_dataset_files_params"
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    func_alias(**kwargs)

    return None



def _check_and_convert_combine_ml_dataset_files_params(params):
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._check_and_convert_combine_ml_dataset_files_params
    params = func_alias(params)

    return params



def _combine_ml_dataset_files(max_num_ml_data_instances_per_file_update,
                              input_ml_dataset_filenames,
                              output_ml_dataset_filename,
                              rm_input_ml_dataset_files,
                              start_time):
    kwargs = locals()
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._combine_ml_dataset_files
    func_alias(**kwargs)

    return None



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_output_ml_dataset_filename_1 = \
    _module_alias._default_output_ml_dataset_filename_1
_default_output_ml_dataset_filename_2 = \
    _module_alias._default_output_ml_dataset_filename_2
_default_output_ml_dataset_filename_3 = \
    _module_alias._default_output_ml_dataset_filename_3
_default_enable_shuffling = \
    _module_alias._default_enable_shuffling
_default_split_ratio = \
    _module_alias._default_split_ratio
_default_rm_input_ml_dataset_file = \
    _module_alias._default_rm_input_ml_dataset_file



def split_ml_dataset_file(
        input_ml_dataset_filename,
        output_ml_dataset_filename_1=\
        _default_output_ml_dataset_filename_1,
        output_ml_dataset_filename_2=\
        _default_output_ml_dataset_filename_2,
        output_ml_dataset_filename_3=\
        _default_output_ml_dataset_filename_3,
        split_ratio=\
        _default_split_ratio,
        enable_shuffling=\
        _default_enable_shuffling,
        rng_seed=\
        _default_rng_seed,
        rm_input_ml_dataset_file=\
        _default_rm_input_ml_dataset_file,
        max_num_ml_data_instances_per_file_update=\
        _default_max_num_ml_data_instances_per_file_update):
    r"""Split file storing a machine learning dataset.

    The current function copies the machine learning (ML) data instances stored
    in an input HDF5 file, and distributes those copies among at most three new
    output HDF5 files.

    The input HDF5 file and the output HDF5 files are assumed to have the same
    file structure as an HDF5 file generated by the function
    :func:`emicroml.modelling.cbed.distortion.estimation.generate_and_save_ml_dataset`.
    See the documentation of said function for a description of the file
    structure. Moreover, the input HDF5 file is assumed to have been created in
    a manner that is consistent with the way HDF5 files are generated by the
    function
    :func:`emicroml.modelling.cbed.distortion.estimation.generate_and_save_ml_dataset`.

    Unlike the combining of ML datasets, as implemented in
    :func:`emicroml.modelling.cbed.distortion.estimation.combine_ml_dataset_files`, no
    renormalization is performed in the process of splitting a machine learning
    dataset.

    The actual number of output HDF5 files is determined by the parameter
    ``split_ratio``. The distribution of the copies of the input ML data
    instances is determined by the total number of input ML data instances
    ``num_input_ml_data_instances``, and the parameters ``split_ratio``,
    ``enable_shuffling``, and ``rng_seed``.

    From ``split_ratio`` and ``num_input_ml_data_instances``, the current
    function calculates an adjusted split ratio ``adjusted_split_ratio`` and
    uses this adjusted split ratio to distribute the copies of the input ML data
    instances.  ``adjusted_split_ratio`` is calculated by:

    .. code-block:: python

        import np as numpy

        adjusted_split_ratio = (num_input_ml_data_instances
                                * np.array(split_ratio)
                                / np.sum(split_ratio))
        adjusted_split_ratio = np.round(split_ratio).astype(int)

        for idx, _ in enumerate(adjusted_split_ratio):
            discrepancy = (num_input_ml_data_instances 
                           - np.sum(adjusted_split_ratio))
            if discrepancy*adjusted_split_ratio[idx] != 0:
                adjustment_candidate = (adjusted_split_ratio[idx]
                                        + np.sign(discrepancy))
                if adjustment_candidate >= 0:
                    adjusted_split_ratio[idx] = adjustment_candidate

    We describe below how the copies of the input ML data instances are
    distributed effectively.

    1. Copy the input ML data instances.

    2. If ``enable_shuffling`` is set to ``True``, then reorder the copy of the
    input ML data instances using a random number generator with the seed
    specified by ``rng_seed``. Otherwise, if ``enable_shuffling`` is set to
    ``False``, then no reordering is performed.

    3. If ``adjusted_split_ratio[0] > 0`` go to instruction 4. Otherwise, go to
    instruction 7.

    4. Set ``i`` to ``0``.

    5. Set ``j`` to ``i+adjusted_split_ratio[0]-1``.

    6. Store the copies of the input ML data instances indexed from ``i`` to
    ``j`` (i.e. including the ``j`` th instance) after reordering into in a new
    output HDF5 file at a file location specified by the parameter
    ``output_ml_dataset_filename_1``.

    7. If ``adjusted_split_ratio[1] > 0`` go to instruction 8. Otherwise, go to
    instruction 11.

    8. Set ``i`` to ``j+1``.

    9. Set ``j`` to ``i+adjusted_split_ratio[1]-1``.

    10. Store the copies of the input ML data instances indexed from ``i`` to
    ``j`` (i.e. including the ``j`` th instance) after reordering into in a new
    output HDF5 file at a file location specified by the parameter
    ``output_ml_dataset_filename_2``.

    11. If ``adjusted_split_ratio[2] > 0`` go to instruction 8. Otherwise, stop.

    12. Set ``i`` to ``j+1``.

    13. Set ``j`` to ``i+adjusted_split_ratio[2]-1``.

    14. Store the copies of the input ML data instances indexed from ``i`` to
    ``j`` (i.e. including the ``j`` th instance) after reordering into in a new
    output HDF5 file at a file location specified by the parameter
    ``output_ml_dataset_filename_3``.

    Parameters
    ----------
    input_ml_dataset_filename : `str`, optional
        The relative or absolute filename of the input HDF5 file.
    output_ml_dataset_filename_1 : `str`, optional
        The relative or absolute filename of the first potential output HDF5
        file.
    output_ml_dataset_filename_2 : `str`, optional
        The relative or absolute filename of the second potential output HDF5
        file.
    output_ml_dataset_filename_3 : `str`, optional
        The relative or absolute filename of the third potential output HDF5
        file.
    split_ratio : `array_like` (`float`, ndim=1), optional
        The split ratio. Must be a triplet of nonnegative numbers that add up to
        a positive number.
    enable_shuffling : `bool`, optional
        If ``enable_shuffling`` is set to ``True``, then the copy of the input
        ML data instances is reordered using a random number generator prior to
        splitting. Otherwise, if ``enable_shuffling`` is set to ``False``, then
        no reordering is performed.
    rng_seed : `int` | `None`, optional
        ``rng_seed`` specifies the seed used in the random number generator, 
        which specifies the distribution of the ML data instances. 
    rm_input_ml_dataset_file : `bool`, optional
        If ``rm_input_ml_dataset_file`` is set to ``True``, then the input HDF5
        file is deleted after all the ML data instances stored in that input
        HDF5 file are copied into the output HDF5 files.
    max_num_ml_data_instances_per_file_update : `int`, optional
        The number of input ML data instances to distribute per file update. The
        larger the value, the larger the memory requirements.

    """
    params = locals()
    params["start_time"] = time.time()

    global_symbol_table = globals()

    func_name = "_check_and_convert_split_ml_dataset_file_params"
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    func_alias(**kwargs)

    return None



def _check_and_convert_split_ml_dataset_file_params(params):
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._check_and_convert_split_ml_dataset_file_params
    params = func_alias(params)

    return params



def _split_ml_dataset_file(output_ml_dataset_filename_1,
                           output_ml_dataset_filename_2,
                           output_ml_dataset_filename_3,
                           max_num_ml_data_instances_per_file_update,
                           input_ml_dataset_filename,
                           split_ratio,
                           enable_shuffling,
                           rng_seed,
                           rm_input_ml_dataset_file,
                           start_time):
    kwargs = locals()
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._split_ml_dataset_file
    func_alias(**kwargs)

    return None



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_max_num_ml_data_instances_per_chunk = \
    _module_alias._default_max_num_ml_data_instances_per_chunk
_default_entire_ml_dataset_is_to_be_cached = \
    _module_alias._default_entire_ml_dataset_is_to_be_cached
_default_ml_data_values_are_to_be_checked = \
    _module_alias._default_ml_data_values_are_to_be_checked



_module_alias = emicroml.modelling.cbed.distortion._common
_cls_alias = _module_alias._MLDataset
class MLDataset(_cls_alias):
    r"""A wrapper to the PyTorch dataset class 
    :class:`torch.utils.data.Dataset`.

    The current class is a subclass of
    :class:`fancytypes.PreSerializableAndUpdatable`.

    The current class represents machine learning (ML) datasets that can be used
    to train and/or evaluate ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`.

    Parameters
    ----------
    path_to_ml_dataset : `str`, optional
        The relative or absolute filename of the HDF5 file in which the ML
        dataset is stored. The input HDF5 file is assumed to have the same file
        structure as an HDF5 file generated by the function
        :func:`emicroml.modelling.cbed.distortion.estimation.generate_and_save_ml_dataset`.
        See the documentation of said function for a description of the file
        structure. Moreover, the input HDF5 file is assumed to have been created
        in a manner that is consistent with the way HDF5 files are generated by
        the function
        :func:`emicroml.modelling.cbed.distortion.estimation.generate_and_save_ml_dataset`.
    entire_ml_dataset_is_to_be_cached : `bool`, optional
        If ``entire_ml_dataset_is_to_be_cached`` is set to ``True``, then as
        long as there is sufficient memory, the entire ML dataset is read from
        the HDF5 file and cached in the instance of the current class, upon
        construction of said instance. In this case, method calls that access ML
        data instances do so via accessing the cached ML dataset. Otherwise, the
        entire ML dataset is not read and cached upon construction of the
        instance of the current class. In this case, method calls that access ML
        data instances do so via reading from the HDF5 file. The first scenario
        yields slower instance construction times, larger memory requirements,
        and faster ML dataset access post instance construction, compared to the
        second scenario. If the parameter ``ml_data_values_are_to_be_checked``
        is set to ``True``, then the construction times in the two
        aforementioned scenarios are comparable.
    ml_data_values_are_to_be_checked : `bool`, optional
        If ``ml_data_values_are_to_be_checked`` is set to ``True``, then the
        data values of the relevant HDF5 datasets stored in the HDF5 file are
        checked, raising an exception if any data values are invalid. Otherwise,
        the data values are not checked. 
    max_num_ml_data_instances_per_chunk : `int` | ``float("inf")``, optional
        If ``ml_data_values_are_to_be_checked`` is set to ``False``, then
        ``max_num_ml_data_instances_per_chunk`` is effectively
        ignored. Otherwise, ``max_num_ml_data_instances_per_chunk`` specifies
        the maximum number of ML data instances to read from the HDF5 file at a
        time when validating the data values stored threrein.
    skip_validation_and_conversion : `bool`, optional
        Let ``validation_and_conversion_funcs`` and ``core_attrs`` denote the
        attributes :attr:`~fancytypes.Checkable.validation_and_conversion_funcs`
        and :attr:`~fancytypes.Checkable.core_attrs` respectively, both of which
        being `dict` objects.

        Let ``params_to_be_mapped_to_core_attrs`` denote the `dict`
        representation of the constructor parameters excluding the parameter
        ``skip_validation_and_conversion``, where each `dict` key ``key`` is a
        different constructor parameter name, excluding the name
        ``"skip_validation_and_conversion"``, and
        ``params_to_be_mapped_to_core_attrs[key]`` would yield the value of the
        constructor parameter with the name given by ``key``.

        If ``skip_validation_and_conversion`` is set to ``False``, then for each
        key ``key`` in ``params_to_be_mapped_to_core_attrs``,
        ``core_attrs[key]`` is set to ``validation_and_conversion_funcs[key]
        (params_to_be_mapped_to_core_attrs)``.

        Otherwise, if ``skip_validation_and_conversion`` is set to ``True``,
        then ``core_attrs`` is set to
        ``params_to_be_mapped_to_core_attrs.copy()``. This option is desired
        primarily when the user wants to avoid potentially expensive deep copies
        and/or conversions of the `dict` values of
        ``params_to_be_mapped_to_core_attrs``, as it is guaranteed that no
        copies or conversions are made in this case.

    """
    def __init__(self,
                 path_to_ml_dataset,
                 entire_ml_dataset_is_to_be_cached=\
                 _default_entire_ml_dataset_is_to_be_cached,
                 ml_data_values_are_to_be_checked=\
                 _default_ml_data_values_are_to_be_checked,
                 max_num_ml_data_instances_per_chunk=\
                 _default_max_num_ml_data_instances_per_chunk,
                 skip_validation_and_conversion=\
                 _default_skip_validation_and_conversion):
        ctor_params = {key: val
                       for key, val in locals().items()
                       if (key not in ("self", "__class__"))}
        
        module_alias = emicroml.modelling.cbed.distortion._common
        cls_alias = module_alias._MLDataset
        kwargs = ctor_params
        cls_alias.__init__(self, **kwargs)

        return None



def ml_data_dict_to_distortion_models(ml_data_dict,
                                      sampling_grid_dims_in_pixels=\
                                      _default_sampling_grid_dims_in_pixels,
                                      device_name=\
                                      _default_device_name,
                                      least_squares_alg_params=\
                                      _default_least_squares_alg_params):
    r"""Convert a dictionary representation of ML data instances to a sequence 
    of distortion models.

    The current function converts a dictionary representation ``ml_data_dict``
    of complete or incomplete machine learning (ML) data instances to a sequence
    of distortion models. A complete dictionary representation is identical in
    structure to a dictionary returned by the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLDataset.get_ml_data_instances`
    of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`.  See the
    documentation for said method for more details. An incomplete dictionary
    representation is identical in structure to a complete dictionary
    representation, except that at least one `dict` item is missing. If
    incomplete, the dictionary representation must have the following keys:
    ``"distortion_centers"``, ``"quadratic_radial_distortion_amplitudes"``,
    ``"spiral_distortion_amplitudes"``, ``"elliptical_distortion_vectors"`, and
    ``"parabolic_distortion_vectors"``. The `dict` items corresponding to the
    aforementioned `dict` keys are the only `dict` items used to construct the
    distortion models.

    For each ML data instance, an instance ``distortion_model`` of the class
    :class:`distoptica.DistortionModel` is constructed according to the ML data
    instance's features. See the documentation for the class
    :class:`distoptica.DistortionModel` for a discussion on distortion models.

    Parameters
    ----------
    ml_data_dict : `dict`
        The dictionary representation of the ML data instances to be converted.
        The current function assumes that all normalizable features of the ML
        data instances are unnormalized. If the normalizable features of the ML
        data instances are not normalized, they can be unnormalized using the
        function
        :func:`emicroml.modelling.cbed.distortion.estimation.unnormalize_normalizable_elems_in_ml_data_dict`
        prior to using the current function.
    sampling_grid_dims_in_pixels : `array_like` (`int`, shape=(2,)), optional
        The dimensions of the sampling grid, in units of pixels, used for
        all distortion models.
    least_squares_alg_params : :class:`distoptica.LeastSquaresAlgParams` | `None`, optional
        ``least_squares_alg_params`` specifies the parameters of the
        least-squares algorithm to be used to calculate the mappings of
        fractional Cartesian coordinates of distorted images to those of the
        corresponding undistorted images. ``least_squares_alg_params`` is used
        to calculate the distortion models mentioned above in the summary
        documentation. If ``least_squares_alg_params`` is set to ``None``, then
        the parameter will be reassigned to the value
        ``distoptica.LeastSquaresAlgParams()``. See the documentation for the
        class :class:`distoptica.LeastSquaresAlgParams` for details on the
        parameters of the least-squares algorithm.
    device_name : `str` | `None`, optional
        This parameter specifies the device to be used to perform
        computationally intensive calls to PyTorch functions and where to store
        attributes of the type :class:`torch.Tensor` for each distortion model
        represented by the class :class:`distoptica.DistortionModel`. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.

    Returns
    -------
    distortion_models : `array_like` (:class:`distoptica.DistortionModel`, ndim=1)
        The distortion models.

    """
    params = locals()

    global_symbol_table = globals()

    func_name = "_check_and_convert_ml_data_dict_to_distortion_models_params"
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    distortion_models = func_alias(**kwargs)

    return distortion_models



def _check_and_convert_ml_data_dict_to_distortion_models_params(params):
    module_alias = \
        emicroml.modelling.cbed.distortion._common
    func_alias = \
        module_alias._check_and_convert_ml_data_dict_to_distortion_models_params
    params = \
        func_alias(params)

    return params



def _ml_data_dict_to_distortion_models(ml_data_dict,
                                       sampling_grid_dims_in_pixels,
                                       device_name,
                                       least_squares_alg_params):
    kwargs = locals()
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._ml_data_dict_to_distortion_models
    distortion_models = func_alias(**kwargs)
    
    return distortion_models



def ml_data_dict_to_signals(ml_data_dict,
                            sampling_grid_dims_in_pixels=\
                            _default_sampling_grid_dims_in_pixels,
                            device_name=\
                            _default_device_name,
                            least_squares_alg_params=\
                            _default_least_squares_alg_params):
    r"""Convert a dictionary representation of ML data instances to a sequence 
    of Hyperspy signals.

    See the documentation for the classes
    :class:`fakecbed.discretized.CBEDPattern`,
    :class:`distoptica.DistortionModel`, and
    :class:`hyperspy._signals.signal2d.Signal2D` for discussions on "fake" CBED
    patterns, distortion models, and Hyperspy signals respectively.

    The current function converts a dictionary representation ``ml_data_dict``
    of complete or incomplete machine learning (ML) data instances to a sequence
    of Hyperspy signals. If incomplete, the ML data instances can, at the very
    least be used to evaluate ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`, and if
    complete, the ML data instances can be used to train such ML models as well.
    
    Each `dict` key in ``ml_data_dict`` is the name of a feature of the ML data
    instances. The only required `dict` key is ``"cbed_pattern_images"``. If
    additional valid `dict` items are present in ``ml_data_dict``, then more
    data and metadata can be stored potentially in the Hyperspy representations
    of the ML data instances. A complete dictionary representation is identical
    in structure to a dictionary returned by the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLDataset.get_ml_data_instances`
    of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`.  See the
    documentation for said method for more details. An incomplete dictionary
    representation is identical in structure to a complete dictionary
    representation, except that at least one `dict` item is missing.

    For each ML data instance, an instance ``distortion_model`` of the class
    :class:`distoptica.DistortionModel` is constructed according to the ML data
    instance's features. The object ``distortion_model`` is a distortion model
    that describes the distortion field of the imaged CBED pattern of the ML
    data instance. If no distortion information is present in ``ml_data_dict``,
    then ``distortion_model.is_trivial`` yields ``True``, i.e. the distortion
    model is assumed to be trivial. After constructing ``distortion_model``, an
    instance of the class :class:`fakecbed.discretized.CBEDPattern` is
    constructed according to the ML data instance's features and
    ``distortion_model``. ``fake_cbed_pattern`` is a fake CBED pattern
    representation of the CBED pattern of the ML data instance. Next, a Hyperspy
    signal ``fake_cbed_pattern_signal`` is obtained from
    ``fake_cbed_pattern.signal``. The Hyperspy signal representation of the ML
    data instance is obtained by modifying in place
    ``fake_cbed_pattern_signal.data[1:3]`` according to the ML data instance's
    features. Note that the illumination support of the fake CBED pattern
    representation of the CBED pattern of the ML data instance is inferred from
    the features of the ML data instance, and is stored in
    ``fake_cbed_pattern_signal.data[1]``. Moreover, the illumination suport
    implied by the signal's metadata should be ignored.

    Parameters
    ----------
    ml_data_dict : `dict`
        The dictionary representation of the ML data instances to be converted.
        The current function assumes that all normalizable features of the ML
        data instances are unnormalized. If the normalizable features of the ML
        data instances are not normalized, they can be unnormalized using the
        function
        :func:`emicroml.modelling.cbed.distortion.estimation.unnormalize_normalizable_elems_in_ml_data_dict`
        prior to using the current function.
    sampling_grid_dims_in_pixels : `array_like` (`int`, shape=(2,)), optional
        The dimensions of the sampling grid, in units of pixels, used for
        all distortion models.
    least_squares_alg_params : :class:`distoptica.LeastSquaresAlgParams` | `None`, optional
        ``least_squares_alg_params`` specifies the parameters of the
        least-squares algorithm to be used to calculate the mappings of
        fractional Cartesian coordinates of distorted images to those of the
        corresponding undistorted images. ``least_squares_alg_params`` is used
        to calculate the interim distortion models mentioned above in the
        summary documentation. If ``least_squares_alg_params`` is set to
        ``None``, then the parameter will be reassigned to the value
        ``distoptica.LeastSquaresAlgParams()``. See the documentation for the
        class :class:`distoptica.LeastSquaresAlgParams` for details on the
        parameters of the least-squares algorithm.
    device_name : `str` | `None`, optional
        This parameter specifies the device to be used to perform
        computationally intensive calls to PyTorch functions and to store
        intermediate arrays of the type :class:`torch.Tensor`. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.

    Returns
    -------
    signals : `array_like` (:class:`hyperspy._signals.signal2d.Signal2D`, ndim=1)
        The ML data instances, represented as a sequence of Hyperspy signals,
        where each Hyperspy signal represents a ML data instance.

    """
    params = locals()

    global_symbol_table = globals()

    func_name = "_check_and_convert_ml_data_dict_to_signals_params"
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    signals = func_alias(**kwargs)

    return signals



def _check_and_convert_ml_data_dict_to_signals_params(params):
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._check_and_convert_ml_data_dict_to_signals_params
    params = func_alias(params)

    return params



def _ml_data_dict_to_signals(ml_data_dict,
                             sampling_grid_dims_in_pixels,
                             device_name,
                             least_squares_alg_params):
    kwargs = locals()
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._ml_data_dict_to_signals
    signals = func_alias(**kwargs)
    
    return signals



def _check_and_convert_ml_training_dataset(params):
    key = "accepted_nontrivial_cls_of_obj_alias_of_ml_dataset"
    params[key] = MLDataset

    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._check_and_convert_ml_training_dataset
    ml_training_dataset = func_alias(params)

    return ml_training_dataset



def _pre_serialize_ml_training_dataset(ml_training_dataset):
    obj_to_pre_serialize = ml_training_dataset
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._pre_serialize_ml_training_dataset
    serializable_rep = func_alias(obj_to_pre_serialize)
    
    return serializable_rep



def _de_pre_serialize_ml_training_dataset(serializable_rep):
    ml_training_dataset = (serializable_rep
                           if (serializable_rep is None)
                           else MLDataset.de_pre_serialize(serializable_rep))

    return ml_training_dataset



def _check_and_convert_ml_validation_dataset(params):
    key = "accepted_nontrivial_cls_of_obj_alias_of_ml_dataset"
    params[key] = MLDataset
    
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._check_and_convert_ml_validation_dataset
    ml_validation_dataset = func_alias(params)

    return ml_validation_dataset



def _pre_serialize_ml_validation_dataset(ml_validation_dataset):
    obj_to_pre_serialize = ml_validation_dataset
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._pre_serialize_ml_validation_dataset
    serializable_rep = func_alias(obj_to_pre_serialize)
    
    return serializable_rep



def _de_pre_serialize_ml_validation_dataset(serializable_rep):
    if serializable_rep is None:
        ml_validation_dataset = serializable_rep
    else:
        ml_validation_dataset = MLDataset.de_pre_serialize(serializable_rep)

    return ml_validation_dataset



def _check_and_convert_ml_testing_dataset(params):
    key = "accepted_nontrivial_cls_of_obj_alias_of_ml_dataset"
    params[key] = MLDataset
    
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._check_and_convert_ml_testing_dataset
    ml_testing_dataset = func_alias(params)

    return ml_testing_dataset



def _pre_serialize_ml_testing_dataset(ml_testing_dataset):
    obj_to_pre_serialize = ml_testing_dataset
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._pre_serialize_ml_testing_dataset
    serializable_rep = func_alias(obj_to_pre_serialize)
    
    return serializable_rep



def _de_pre_serialize_ml_testing_dataset(serializable_rep):
    if serializable_rep is None:
        ml_testing_dataset = serializable_rep
    else:
        ml_testing_dataset = MLDataset.de_pre_serialize(serializable_rep)

    return ml_testing_dataset



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_ml_training_dataset = \
    _module_alias._default_ml_training_dataset
_default_ml_validation_dataset = \
    _module_alias._default_ml_validation_dataset
_default_ml_testing_dataset = \
    _module_alias._default_ml_testing_dataset
_default_mini_batch_size = \
    _module_alias._default_mini_batch_size
_default_num_data_loader_workers = \
    _module_alias._default_num_data_loader_workers



_module_alias = emicroml.modelling.cbed.distortion._common
_cls_alias = _module_alias._MLDatasetManager
class MLDatasetManager(_cls_alias):
    r"""A machine learning dataset manager.

    The current class is a subclass of
    :class:`fancytypes.PreSerializableAndUpdatable`.

    The current class represents machine learning (ML) dataset manager that can
    be used to train and/or evaluate ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`.

    Parameters
    ----------
    ml_training_dataset : :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset` | `None`, optional
        This parameter specifies the ML training dataset to be used, if any at
        all. If ``ml_training_dataset`` is an instance of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`, then a
        ML training dataset is to be used, and is represented by the object
        ``ml_training_dataset``. Otherwise, no ML training dataset is to be
        used.
    ml_validation_dataset : :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset` | `None`, optional
        This parameter specifies the ML validation dataset to be used, if any at
        all. If ``ml_validation_dataset`` is an instance of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`, then a
        ML validation dataset is to be used, and is represented by the object
        ``ml_validation_dataset``. Otherwise, no ML validation dataset is to be
        used.
    ml_testing_dataset : :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset` | `None`, optional
        This parameter specifies the ML testing dataset to be used, if any at
        all. If ``ml_testing_dataset`` is an instance of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`, then a
        ML testing dataset is to be used, and is represented by the object
        ``ml_testing_dataset``. Otherwise, no ML testing dataset is to be used.
    mini_batch_size : `int`, optional
        The mini-batch size to be used in training and/or evaluating ML models.
    rng_seed : `int` | `None`, optional
        ``rng_seed`` specifies the seed used in the random number generator used
        to shuffle ML data instances in the PyTorch data loaders used during
        training and/or validation, or testing.
    num_data_loader_workers : `int`, optional
        The number of subprocesses to use for data loading. If set to zero, then
        the data will be loaded in the main process only.
    skip_validation_and_conversion : `bool`, optional
        Let ``validation_and_conversion_funcs`` and ``core_attrs`` denote the
        attributes :attr:`~fancytypes.Checkable.validation_and_conversion_funcs`
        and :attr:`~fancytypes.Checkable.core_attrs` respectively, both of which
        being `dict` objects.

        Let ``params_to_be_mapped_to_core_attrs`` denote the `dict`
        representation of the constructor parameters excluding the parameter
        ``skip_validation_and_conversion``, where each `dict` key ``key`` is a
        different constructor parameter name, excluding the name
        ``"skip_validation_and_conversion"``, and
        ``params_to_be_mapped_to_core_attrs[key]`` would yield the value of the
        constructor parameter with the name given by ``key``.

        If ``skip_validation_and_conversion`` is set to ``False``, then for each
        key ``key`` in ``params_to_be_mapped_to_core_attrs``,
        ``core_attrs[key]`` is set to ``validation_and_conversion_funcs[key]
        (params_to_be_mapped_to_core_attrs)``.

        Otherwise, if ``skip_validation_and_conversion`` is set to ``True``,
        then ``core_attrs`` is set to
        ``params_to_be_mapped_to_core_attrs.copy()``. This option is desired
        primarily when the user wants to avoid potentially expensive deep copies
        and/or conversions of the `dict` values of
        ``params_to_be_mapped_to_core_attrs``, as it is guaranteed that no
        copies or conversions are made in this case.

    """
    _validation_and_conversion_funcs_ = \
        {**_cls_alias._validation_and_conversion_funcs_,
         "ml_training_dataset": _check_and_convert_ml_training_dataset,
         "ml_validation_dataset": _check_and_convert_ml_validation_dataset,
         "ml_testing_dataset": _check_and_convert_ml_testing_dataset}

    _pre_serialization_funcs_ = \
        {**_cls_alias._pre_serialization_funcs_,
         "ml_training_dataset": _pre_serialize_ml_training_dataset,
         "ml_validation_dataset": _pre_serialize_ml_validation_dataset,
         "ml_testing_dataset": _pre_serialize_ml_testing_dataset}

    _de_pre_serialization_funcs_ = \
        {**_cls_alias._de_pre_serialization_funcs_,
         "ml_training_dataset": _de_pre_serialize_ml_training_dataset,
         "ml_validation_dataset": _de_pre_serialize_ml_validation_dataset,
         "ml_testing_dataset": _de_pre_serialize_ml_testing_dataset}

    
    
    def __init__(self,
                 ml_training_dataset=\
                 _default_ml_training_dataset,
                 ml_validation_dataset=\
                 _default_ml_validation_dataset,
                 ml_testing_dataset=\
                 _default_ml_testing_dataset,
                 mini_batch_size=\
                 _default_mini_batch_size,
                 rng_seed=\
                 _default_rng_seed,
                 num_data_loader_workers=\
                 _default_num_data_loader_workers,
                 skip_validation_and_conversion=\
                 _default_skip_validation_and_conversion):
        ctor_params = {key: val
                       for key, val in locals().items()
                       if (key not in ("self", "__class__"))}
        
        module_alias = emicroml.modelling.cbed.distortion._common
        cls_alias = module_alias._MLDatasetManager
        kwargs = ctor_params
        cls_alias.__init__(self, **kwargs)

        return None



def _check_and_convert_ml_dataset_manager(params):
    key = "ml_dataset_manager_cls"
    params[key] = MLDatasetManager
    
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._check_and_convert_ml_dataset_manager
    ml_dataset_manager = func_alias(params)

    return ml_dataset_manager



def _pre_serialize_ml_dataset_manager(ml_dataset_manager):
    obj_to_pre_serialize = ml_dataset_manager
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = module_alias._pre_serialize_ml_dataset_manager
    serializable_rep = func_alias(obj_to_pre_serialize)
    
    return serializable_rep



def _de_pre_serialize_ml_dataset_manager(serializable_rep):
    ml_dataset_manager = MLDatasetManager.de_pre_serialize(serializable_rep)

    return ml_dataset_manager



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_num_pixels_across_each_cbed_pattern = \
    _module_alias._default_num_pixels_across_each_cbed_pattern
_default_architecture = \
    _module_alias._default_architecture
_default_mini_batch_norm_eps = \
    _module_alias._default_mini_batch_norm_eps
_default_normalization_weights = \
    _module_alias._default_normalization_weights
_default_normalization_biases = \
    _module_alias._default_normalization_biases
_default_unnormalize_normalizable_elems_of_ml_predictions = \
    _module_alias._default_unnormalize_normalizable_elems_of_ml_predictions



_module_alias = emicroml.modelling.cbed.distortion._common
_cls_alias = _module_alias._MLModel
class _MLModel(_cls_alias):
    def __init__(self,
                 num_pixels_across_each_cbed_pattern,
                 max_num_disks_in_any_cbed_pattern,
                 architecture,
                 mini_batch_norm_eps,
                 normalization_weights,
                 normalization_biases):
        ctor_params = {key: val
                       for key, val in locals().items()
                       if (key not in ("self", "__class__"))}
        ctor_params = self._check_and_convert_ctor_params(ctor_params)
        
        module_alias = emicroml.modelling.cbed.distortion._common
        cls_alias = module_alias._MLModel
        kwargs = ctor_params
        cls_alias.__init__(self, **kwargs)

        return None



class MLModel(_MLModel):
    r"""A machine learning model for distortion estimation in CBED.

    The current class is a subclass of :class:`torch.nn.Module`.

    A given machine learning (ML) model represented by the current class takes
    as input a mini-batch of images, where each image is assumed to depict a
    distorted CBED pattern, and as output, the ML model predicts sets of
    coordinate transformation parameters that specify the coordinate
    transformations that describe the distortions of the input images. The
    coordinate transformation used to describe the distortions of an image is
    defined in the documentation for the class
    :class:`distoptica.StandardCoordTransformParams`. The parameter set
    parameterizing said coordinate transformation is referred to as the
    "standard" coordinate transformation parameter set, and is represented by
    the class :class:`distoptica.StandardCoordTransformParams`.

    ML models are trained using the
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer`. 

    After a ML model has been trained, users should use the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLModel.make_predictions`
    of the current class to make predictions.

    Parameters
    ----------
    num_pixels_across_each_cbed_pattern : `int`, optional
        The number of pixels across each imaged CBED pattern stored in the ML
        dataset used or to be used to train the ML model. This parameter is
        expected to be equal to the instance attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.num_pixels_across_each_cbed_pattern`
        of the instance of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`
        representing the aforementioned ML dataset. Moreover, the parameter is
        expected to be a positive integer that is divisible by ``2**5``.
    max_num_disks_in_any_cbed_pattern : `int`, optional
        The maximum possible number of CBED disks in any imaged CBED pattern
        stored in the ML dataset used or to be used to train the ML model. This
        parameter is expected to be equal to the instance attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.max_num_disks_in_any_cbed_pattern`
        of the instance of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`
        representing the aforementioned ML dataset.
    architecture : `str`, optional
        This parameter specifies the network architecture of the ML model. At
        the moment, only one network architecture is available for this ML
        model, and it is specified by setting ``architecture`` to
        ``"distoptica_net"``, referring to the DistopticaNet architecture. Below
        we refer to this network architecture as the DistopticaNet architecture.

        In short, the DistopticaNet architecture is a custom residual network
        with 37 non-trivial layers, and downsampling operations being performed
        using strided convolutions rather than pooling. By a non-trivial layer,
        we mean either a fully connected layer or a 2D convolutional layer with
        kernel dimensions other than :math:`1 \times 1`.

        Before describing in more detail the DistopticaNet architecture, it is
        worth introducing several smaller networks used to construct the
        architecture.

        First, we introduce the residual network building block, defined
        graphically as:

        .. _modelling_cbed_distortion_estimation_resnet_building_block:
        .. figure:: ../_images/modelling/cbed/distortion/estimation/resnet_building_block.png

           The residual network building block, where :math:`C_1`, :math:`C_2`,
           :math:`K`, :math:`N_{\downarrow}`, and :math:`f` are the number of
           input channels, the number of output channels, the maximum kernel
           size, the number of downsamplings to perform in the first
           convolutional layer, and the final activation function respectively.

        The above image introduces several mathematical objects:
        :math:`\text{conv_2d}\left(C_{1},C_{2},K,S\right)` is a 2D convolutional
        layer with :math:`C_1` input channels, :math:`C_2` output channels, a
        kernel size of :math:`K`, a stride of :math:`S`, a zero-padding width of
        :math:`(K-1) // 2` on all sides, all biases fixed to zero, a dilation of
        unity, and all inputs are convolved to all outputs;
        :math:`\text{mini_batch_norm_2d}\left(C\right)` is a mini-batch
        normalization layer of 2D inputs with :math:`C` input channels;
        :math:`\text{relu}` is the ReLU activation function;
        :math:`\text{shortcut}\left(C_{1},C_{2},N_{\downarrow}\right)` is
        :math:`\text{conv_2d}\left(C_{1},C_{2},1,1+N_{\downarrow}\right)`
        followed by :math:`\text{mini_batch_norm_2d}\left(C_{2}\right)` if
        either :math:`C_{1} \neq C_{2}` or :math:`N_{\downarrow} > 0`, else it
        is an identity shortcut connection; :math:`+` is the addition operator;
        :math:`f` is an activation function.

        Next, we introduce the residual network stage, defined graphically as:

        .. _modelling_cbed_distortion_estimation_resnet_stage:
        .. figure:: ../_images/modelling/cbed/distortion/estimation/resnet_stage.png

           The residual network stage, where :math:`C`, :math:`K`, :math:`N_B`,
           and :math:`f` are the number of input channels, the maximum kernel
           size, the number of residual network building blocks in the stage,
           and the final activation function respectively.

        Next, we introduce the "enhance" operation, defined graphically as:

        .. _modelling_cbed_distortion_estimation_enhance:
        .. figure:: ../_images/modelling/cbed/distortion/estimation/enhance.png

           The enhance operation.

        The above image introduces a few mathematical objects:
        :math:`\text{min-max normalize}` is the min-max normalization operation,
        applied to each image stored in the input tensor of the entry flow;
        :math:`\text{pow}\left(\gamma\right)` is the gamma correction operation
        with the power-law exponent :math:`\gamma`; :math:`\text{equalize}` is
        the histogram equalization operation, applied to each feature map.

        Next, we introduce the DistopticaNet entry flow, defined graphically as:

        .. _modelling_cbed_distortion_estimation_distoptica_net_entry_flow:
        .. figure:: ../_images/modelling/cbed/distortion/estimation/distoptica_net_entry_flow.png

           The DistopticaNet entry flow, where :math:`C_1`, :math:`C_2`,
           :math:`K_1`, and :math:`K_2` are the number of input channels, the
           number of output channels, the kernel size of the first convolutional
           layer, and the maximum kernel size of the resnet building blocks
           respectively.

        Next, we introduce the DistopticaNet middle flow, defined graphically
        as:

        .. _modelling_cbed_distortion_estimation_distoptica_net_middle_flow:
        .. figure:: ../_images/modelling/cbed/distortion/estimation/distoptica_net_middle_flow.png

           The DistopticaNet middle flow, where :math:`C_1`, :math:`K`, and
           :math:`\mathbf{N}_{\mathbf{B}}` are the number of input channels, the
           maximum kernel size, and the building block counts in the residual
           network stages of the middle flow respectively.

        The above image introduces the block of the general form :math:`X
        \leftarrow Y`, which denotes the operation of setting the variable
        :math:`X` to the value of :math:`Y` while leaving the input tensor of
        said block unchanged.

        Next, we introduce the DistopticaNet exit flow, defined graphically as:

        .. _modelling_cbed_distortion_estimation_distoptica_net_exit_flow:
        .. figure:: ../_images/modelling/cbed/distortion/estimation/distoptica_net_exit_flow.png

           The no-pool DistopticaNet exit flow, where :math:`F_1`, :math:`F_2`,
           and :math:`F_3` are the number of nodes in the third last, the second
           last, and the last layers respectively.

        The above image introduces several mathematical objects:
        :math:`\text{flatten}` is the flatten operation applied to all but the
        ML data instance dimension;
        :math:`\text{fc}\left(F_{1},F_{2},\text{biases}\right)` is a
        fully-connected layer with :math:`F_1` input channels, :math:`F_2`
        output channels, and the biases fixed to zero if the boolean variable
        :math:`\text{biases}` is set to :math:`\text{False}`;
        :math:`\text{mini_batch_norm_1d}\left(C\right)` is a mini-batch
        normalization layer of 1D inputs with :math:`C` input channels.

        Finally, the DistopticaNet architecture is defined graphically as:

        .. _modelling_cbed_distortion_estimation_distoptica_net:
        .. figure:: ../_images/modelling/cbed/distortion/estimation/distoptica_net.png

           The ``"distoptica_net"`` architecture, where :math:`W` is the width
           of the input tensor in pixels.

        See the documentation for the method
        :meth:`emicroml.modelling.cbed.distortion.estimation.MLModel.forward`
        for a discussion on how the output tensor is parsed as a dictionary.

        The weights of all the convolutional and fully-connected (FC) layers,
        except for those of the last FC layer, are He-initialized using the
        function :func:`torch.nn.init.kaiming_normal_`, with the parameters
        ``a``, ``mode``, ``nonlinearity``, and ``generator`` set to ``0``,
        ``'fan_out'``, ``'relu'``, and ``None`` respectively. The weights of the
        last FC layer are Glorot-initialized using the function
        :func:`torch.nn.init.xavier_normal_`, with the parameters ``gain``, and
        ``generator`` set to ``5/3`` and ``None`` respectively.

        The biases of the last FC layer, and all of the mini-batch normalization
        layers are initialized to zero; the weights of all the mini-batch
        normalization layers except for those of the mini-batch normalization
        layers in :math:`\text{shortcut}\left(C_{1},C_{2},N_{\downarrow}\right)`
        objects are normalized to unity; and the weights of the mini-batch
        normalization layers in
        :math:`\text{shortcut}\left(C_{1},C_{2},N_{\downarrow}\right)` objects
        are normalized to zero.
    mini_batch_norm_eps : `float`, optional
        This parameter specifies the value to use for the construction parameter
        ``eps`` for every construction of an instance of the class
        :class:`torch.nn.BatchNorm1d` and every construction of an instance of
        the class :class:`torch.nn.BatchNorm2d`. Must be a positive number.
    normalization_weights : `dict`, optional
        The normalization weights of the ML dataset used or to be used to train
        the ML model. This parameter is expected to be equal to the instance
        attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.normalization_weights`
        of the instance of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`
        representing the aforementioned ML dataset. See the documentation for
        the function
        :func:`emicroml.modelling.cbed.distortion.estimation.normalize_normalizable_elems_in_ml_data_dict`
        for a discussion on normalizing features of ML data instances.
    normalization_biases : `dict`, optional
        The normalization biases of the ML dataset used or to be used to train
        the ML model. This parameter is expected to be equal to the instance
        attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.normalization_biases`
        of the instance of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`
        representing the aforementioned ML dataset.

    """
    def __init__(self,
                 num_pixels_across_each_cbed_pattern=\
                 _default_num_pixels_across_each_cbed_pattern,
                 max_num_disks_in_any_cbed_pattern=\
                 _default_max_num_disks_in_any_cbed_pattern,
                 architecture=\
                 _default_architecture,
                 mini_batch_norm_eps=\
                 _default_mini_batch_norm_eps,
                 normalization_weights=\
                 _default_normalization_weights,
                 normalization_biases=\
                 _default_normalization_biases):
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        _MLModel.__init__(self, **kwargs)

        return None



    def forward(self, ml_inputs):
        r"""Perform forward propagation.

        The current function forward propagates a dictionary representation
        ``ml_inputs`` of a mini-batch of machine learning (ML) inputs through
        the ML model.

        The ML model takes as input a mini-batch of images, where each image is
        assumed to depict a distorted CBED pattern, and as output, the ML model
        predicts sets of coordinate transformation parameters that specify the
        coordinate transformations that describe the distortions of the input
        images. The coordinate transformation used to describe the distortions
        of an image is defined in the documentation for the class
        :class:`distoptica.StandardCoordTransformParams`. The parameter set
        parameterizing said coordinate transformation is referred to as the
        "standard" coordinate transformation parameter set, and is represented
        by the class :class:`distoptica.StandardCoordTransformParams`. See the
        documentation for said class for a discussion on standard coordinate
        transformation parameter sets.

        The output tensor ``output_tensor`` of the neural network of the ML
        model is an 8-column PyTorch tensor, i.e. PyTorch matrix, of the data
        type ``torch.float32``. Let ``mini_batch_size`` be the number of rows in
        the ``output_tensor``. For each nonnegative integer ``n`` less than
        ``mini_batch_size``, ``output_tensor[n]`` stores the predicted
        normalized parameters of the standard coordinate transformation that are
        suppose to describe the distortions of the ``n`` th input image of the
        mini-batch. The parameters are normalized according to the normalization
        weights and biases of the ML dataset used or to be used to train the ML
        model. See the documentation for the function
        :func:`emicroml.modelling.cbed.distortion.estimation.normalize_normalizable_elems_in_ml_data_dict`
        for a discussion on normalizing features of ML data instances, e.g. the
        standard coordinate transformation parameters. The normalization weights
        and biases are stored in ``core_attrs["normalization_weights"]`` and
        ``core_attrs["normalization_biases"]`` respectively, where
        ``core_attrs`` is the instance attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLModel.core_attrs`.

        ``output_tensor[:, 0]`` stores the normalized quadratic radial
        distortion amplitudes, ``output_tensor[:, 1]`` stores the normalized
        spiral distortion amplitudes, ``output_tensor[:, 2:4]`` stores the
        normalized elliptical distortion vectors, ``output_tensor[:, 4:6]``
        stores the normalized parabolic distortion vectors, and
        ``output_tensor[:, 6:8]`` stores the normalized distortion centers.

        Parameters
        ----------
        ml_inputs : `dict`
            The dictionary representation of the mini-batch of ML inputs. 

            ``ml_inputs`` must have only one `dict` key, the value of which
            being
            ``"cbed_pattern_images"``. ``ml_inputs["cbed_pattern_images"]`` must
            be a 3D PyTorch tensor of the data type ``torch.float32`` storing
            the mini-batch of images assumed to depict distorted CBED patterns.
            For each nonnegative integer ``n`` less than ``mini_batch_size``,
            ``ml_inputs["cbed_pattern_images"][n]`` stores the ``n`` th input
            image of the mini-batch. ``mini_batch_size`` must be positive and
            ``ml_inputs["cbed_pattern_images"].shape[1:]`` must be equal to
            ``2*(num_pixels_across_each_cbed_pattern,)``, where
            ``num_pixels_across_each_cbed_pattern`` is
            ``core_attrs["num_pixels_across_each_cbed_pattern"]``, i.e. the
            number of pixels across each input image.

        Returns
        -------
        ml_predictions : `dict`
            The dictionary representation of the mini-batch of ML outputs. 

            Using the output tensor ``output_tensor`` discussed above,
            ``ml_predictions`` is constructed essentially by

            .. code-block:: python

                ml_predictions = {"quadratic_radial_distortion_amplitudes": \
                                  output_tensor[:, 0],
                                  "spiral_distortion_amplitudes": \
                                  output_tensor[:, 1],
                                  "elliptical_distortion_vectors": \
                                  output_tensor[:, 2:4],
                                  "parabolic_distortion_vectors": \
                                  output_tensor[:, 4:6],
                                  "distortion_centers": \
                                  output_tensor[:, 6:8]}

            Users can use the function
            :func:`emicroml.modelling.cbed.distortion.estimation.ml_data_dict_to_distortion_models`
            to convert ``ml_predictions`` to a sequence of distortion models,
            with each distortion model being represented by the class
            :class:`distoptica.DistortionModel`.

        """
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        ml_predictions = super().forward(**kwargs)

        return ml_predictions



    def make_predictions(
            self,
            ml_inputs,
            unnormalize_normalizable_elems_of_ml_predictions=\
            _default_unnormalize_normalizable_elems_of_ml_predictions):
        r"""Make predictions according to machine learning inputs.

        The machine learning (ML) model takes as input a mini-batch of images,
        where each image is assumed to depict a distorted CBED pattern, and as
        output, the ML model predicts sets of coordinate transformation
        parameters that specify the coordinate transformations that describe the
        distortions of the input images. The coordinate transformation used to
        describe the distortions of an image is defined in the documentation for
        the class :class:`distoptica.StandardCoordTransformParams`. The
        parameter set parameterizing said coordinate transformation is referred
        to as the "standard" coordinate transformation parameter set, and is
        represented by the class
        :class:`distoptica.StandardCoordTransformParams`. See the documentation
        for said class for a discussion on standard coordinate transformation
        parameter sets.

        Parameters
        ----------
        ml_inputs : `dict`
            The dictionary representation of the mini-batch of ML inputs.
            ``ml_inputs`` must have the `dict` key
            ``"cbed_pattern_images"``. ``ml_inputs["cbed_pattern_images"]`` must
            be a 3D PyTorch tensor of the data type ``torch.float32`` storing
            the mini-batch of images assumed to depict distorted CBED
            patterns. Let ``mini_batch_size`` be
            ``ml_inputs["cbed_pattern_images"].shape[0]``, and ``core_attrs`` be
            the instance attribute
            :attr:`emicroml.modelling.cbed.distortion.estimation.MLModel.core_attrs`.
            For each nonnegative integer ``n`` less than ``mini_batch_size``,
            ``ml_inputs["cbed_pattern_images"][n]`` stores the ``n`` th input
            image of the mini-batch. ``mini_batch_size`` must be positive and
            ``ml_inputs["cbed_pattern_images"].shape[1:]`` must be equal to
            ``2*(num_pixels_across_each_cbed_pattern,)``, where
            ``num_pixels_across_each_cbed_pattern`` is
            ``core_attrs["num_pixels_across_each_cbed_pattern"]``, i.e. the
            number of pixels across each input image.
        unnormalize_normalizable_elems_of_ml_predictions : `bool`
            If ``unnormalize_normalizable_elems_of_ml_predictions`` is set to
            ``False``, then the predicted parameters of the standard coordinate
            transformations are returned normalized. Otherwise, said parameters
            are returned unnormalized. See the description below of
            ``ml_predictions`` for more details on how this is implemented
            effectively.

        Returns
        -------
        ml_predictions : `dict`
            The dictionary representation of the mini-batch of ML outputs.

            Let ``ml_model`` be an instance of the current class. Then
            ``ml_predictions`` is calculated effectively by:

            .. code-block:: python

                import emicroml.modelling.cbed.distortion.estimation

                module_alias = \
                    emicroml.modelling.cbed.distortion.estimation
                func_alias = \
                    module_alias.unnormalize_normalizable_elems_in_ml_data_dict

                ml_predictions = ml_model.forward(ml_inputs)

                if unnormalize_normalizable_elems_of_ml_predictions:
                    kwargs = {"ml_data_dict": \
                              ml_predictions,
                              "normalization_weights": \
                              ml_model.core_attrs["normalization_weights"],
                              "normalization_biases": \
                              ml_model.core_attrs["normalization_biases"]}
                    ml_predictions = func_alias(**kwargs)

            See the documentation for the method
            :meth:`emicroml.modelling.cbed.distortion.estimation.MLModel.forward`
            for details on the output returned by said method. See the
            documentation for the function
            :func:`emicroml.modelling.cbed.distortion.estimation.normalize_normalizable_elems_in_ml_data_dict`
            for a discussion on normalizing features of ML data instances,
            e.g. the standard coordinate transformation parameters.

            Users can use the function
            :func:`emicroml.modelling.cbed.distortion.estimation.ml_data_dict_to_distortion_models`
            to convert ``ml_predictions`` to a sequence of distortion models,
            with each distortion model being represented by the class
            :class:`distoptica.DistortionModel`.

        """
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        ml_predictions = super().make_predictions(**kwargs)

        return ml_predictions



    def predict_distortion_models(self,
                                  cbed_pattern_images,
                                  sampling_grid_dims_in_pixels=\
                                  _default_sampling_grid_dims_in_pixels,
                                  least_squares_alg_params=\
                                  _default_least_squares_alg_params):
        r"""Predict distortion models according to a mini-batch of images.

        The machine learning (ML) model takes as input a mini-batch of images,
        where each image is assumed to depict a distorted CBED pattern, and as
        output, the ML model predicts a set of distortion models that describe
        the distortions of the input images. The distortion model used to
        describe the distortion of an image is defined in the documentation for
        the class :class:`distoptica.DistortionModel`. See the documentation for
        the class :class:`distoptica.DistortionModel` for a discussion on
        distortion models.

        For each CBED pattern image, an instance ``distortion_model`` of the
        class :class:`distoptica.DistortionModel` is constructed according to
        the distortions predicted by the ML model. 

        Parameters
        ----------
        cbed_pattern_images : `array_like` (`float`, ndim=3)
            The mini-batch of images. Let ``mini_batch_size`` be
            ``cbed_pattern_images.shape[0]``, and ``core_attrs`` be the
            instance attribute
            :attr:`emicroml.modelling.cbed.distortion.estimation.MLModel.core_attrs`.
            For each nonnegative integer ``n`` less than ``mini_batch_size``,
            ``cbed_pattern_images[n]`` stores the ``n`` th input image of the
            mini-batch. ``mini_batch_size`` must be positive and
            ``cbed_pattern_images.shape[1:]`` must be equal to
            ``2*(num_pixels_across_each_cbed_pattern,)``, where
            ``num_pixels_across_each_cbed_pattern`` is
            ``core_attrs["num_pixels_across_each_cbed_pattern"]``, i.e. the
            number of pixels across each input image.
        sampling_grid_dims_in_pixels : `array_like` (`int`, shape=(2,)), optional
            The dimensions of the sampling grid, in units of pixels, used for
            all distortion models.
        least_squares_alg_params : :class:`distoptica.LeastSquaresAlgParams` | `None`, optional
            ``least_squares_alg_params`` specifies the parameters of the
            least-squares algorithm to be used to calculate the mappings of
            fractional Cartesian coordinates of distorted images to those of the
            corresponding undistorted images. ``least_squares_alg_params`` is
            used to calculate the distortion models mentioned above in the
            summary documentation. If ``least_squares_alg_params`` is set to
            ``None``, then the parameter will be reassigned to the value
            ``distoptica.LeastSquaresAlgParams()``. See the documentation for
            the class :class:`distoptica.LeastSquaresAlgParams` for details on
            the parameters of the least-squares algorithm.

        Returns
        -------
        distortion_models : `array_like` (:class:`distoptica.DistortionModel`, ndim=1)
            The distortion models. Note that each distortion model is stored on 
            the same device as that on which the ML model is stored.

        """
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        distortion_models = super().predict_distortion_models(**kwargs)

        return distortion_models



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_normalization_weights = \
    _module_alias._default_normalization_weights
_default_normalization_biases = \
    _module_alias._default_normalization_biases
_default_check_ml_data_dict_first = \
    _module_alias._default_check_ml_data_dict_first



def normalize_normalizable_elems_in_ml_data_dict(
        ml_data_dict,
        normalization_weights=_default_normalization_weights,
        normalization_biases=_default_normalization_biases,
        check_ml_data_dict_first=_default_check_ml_data_dict_first):
    r"""Normalize in-place normalizable features of a dictionary representation 
    of machine learning data instances.

    The current function normalizes in-place the normalizable features of a
    dictionary representation ``ml_data_dict`` of complete or incomplete machine
    learning (ML) data instances. If complete, the ML data instances can be used
    to train and/or evaluate ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`.
    
    Each `dict` key in ``ml_data_dict`` is the name of a feature of the ML data
    instances. A complete dictionary representation is identical in structure to
    a dictionary returned by the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLDataset.get_ml_data_instances`
    of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`.  See the
    documentation for said method for more details. An incomplete dictionary
    representation is identical in structure to a complete dictionary
    representation, except that at least one `dict` item is missing.

    The normalizable features are ``"undistorted_disk_center_sets"``,
    ``"common_undistorted_disk_radii"``, ``"distortion_centers"``,
    ``"quadratic_radial_distortion_amplitudes"``,
    ``"spiral_distortion_amplitudes"``, ``"elliptical_distortion_vectors"`, and
    ``"parabolic_distortion_vectors"``.

    Let ``unnormalized_values`` be the unnormalized values of a normalizable
    feature of the ML data instances. The normalization is performed by

    .. code-block:: python

        normalized_values = (unnormalized_values*normalization_weight
                             + normalization_bias)

    where ``normalized_values`` are the normalized values,
    ``normalization_weight`` is a valid normalization weight, and
    ``normalization_bias`` is a valid noramlization bias. Valid normalization
    weights and biases are those with values that yield normalized features with
    elements that lie within the closed interval :math:`[0, 1]`.

    Parameters
    ----------
    ml_data_dict : `dict`
        The dictionary representation of the ML data instances, for which to
        perform in-place normalization. Prior to normalization, all normalizable
        features are assumed to be unnormalized.
    normalization_weights : `dict`, optional
        The normalization weights. The `dict` keys must be the same as those as
        the instance attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.normalization_weights`
        of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`. The
        value of each `dict` item is expected to be a valid real number.
    normalization_biases : `dict`, optional
        The normalization biases. The `dict` keys must be the same as those as
        the instance attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.normalization_biases`
        of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`. The
        value of each `dict` item is expected to be a valid real number.
    check_ml_data_dict_first : `bool`, optional
        If ``check_ml_data_dict_first`` is set to ``True``, then
        ``ml_data_dict`` is checked, raising an exception if ``ml_data_dict`` is
        not a valid dictionary representation of ML data instances. Otherwise,
        ``ml_data_dict`` is not checked.

    """
    params = locals()

    global_symbol_table = globals()

    func_name = ("_check_and_convert"
                 "_normalize_normalizable_elems_in_ml_data_dict_params")
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    func_alias(**kwargs)

    return None



def _check_and_convert_normalize_normalizable_elems_in_ml_data_dict_params(
        params):
    current_func_name = ("_check_and_convert_normalize_normalizable_elems"
                         "_in_ml_data_dict_params")
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = getattr(module_alias, current_func_name)
    params = func_alias(params)

    return params



def _normalize_normalizable_elems_in_ml_data_dict(check_ml_data_dict_first,
                                                  normalization_weights,
                                                  normalization_biases,
                                                  ml_data_dict):
    params = locals()
    kwargs = params.copy()
    del kwargs["check_ml_data_dict_first"]
    try:
        current_func_name = "_normalize_normalizable_elems_in_ml_data_dict"
        module_alias = emicroml.modelling.cbed.distortion._common
        func_alias = getattr(module_alias, current_func_name)
        func_alias(**kwargs)
    except:
        func_name = ("_check_and_convert_normalize_normalizable_elems"
                     "_in_ml_data_dict_params")
        func_alias = globals()[func_name]
        func_alias(params)

    return None



def unnormalize_normalizable_elems_in_ml_data_dict(
        ml_data_dict,
        normalization_weights=_default_normalization_weights,
        normalization_biases=_default_normalization_biases,
        check_ml_data_dict_first=_default_check_ml_data_dict_first):
    r"""Unnormalize in-place normalizable features of a dictionary 
    representation of machine learning data instances.

    See the documentation for the function
    :func:`emicroml.modelling.cbed.distortion.estimation.normalize_normalizable_elems_in_ml_data_dict`
    for a discussion on normalized features of machine learning (ML) data
    instances.

    The current function unnormalizes in-place the normalizable features of a
    dictionary representation ``ml_data_dict`` of complete or incomplete ML data
    instances. If complete, the ML data instances can be used to train and/or
    evaluate ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`.
    
    Each `dict` key in ``ml_data_dict`` is the name of a feature of the ML data
    instances. A complete dictionary representation is identical in structure to
    a dictionary returned by the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLDataset.get_ml_data_instances`
    of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`.  See the
    documentation for said method for more details. An incomplete dictionary
    representation is identical in structure to a complete dictionary
    representation, except that at least one `dict` item is missing.

    The normalizable features are ``"undistorted_disk_center_sets"``,
    ``"common_undistorted_disk_radii"``, ``"distortion_centers"``,
    ``"quadratic_radial_distortion_amplitudes"``,
    ``"spiral_distortion_amplitudes"``, ``"elliptical_distortion_vectors"`, and
    ``"parabolic_distortion_vectors"``.

    Let ``normalized_values`` be the normalized values of a normalizable feature
    of the ML data instances. The reverse normalization is performed by

    .. code-block:: python

        unnormalized_values = ((normalized_values-normalization_bias) 
                               / normalization_weight)

    where ``unnormalized_values`` are the unnormalized values,
    ``normalization_weight`` is a valid normalization weight, and
    ``normalization_bias`` is a valid noramlization bias. Valid normalization
    weights and biases are those with values that yield unnormalized features
    with elements that lie within valid ranges of values.

    Parameters
    ----------
    ml_data_dict : `dict`
        The dictionary representation of the ML data instances, for which to
        perform in-place reverse normalization. Prior to reverse normalization, 
        all normalizable features are assumed to be normalized.
    normalization_weights : `dict`, optional
        The normalization weights. The `dict` keys must be the same as those as
        the instance attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.normalization_weights`
        of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`. The
        value of each `dict` item is expected to be a valid real number.
    normalization_biases : `dict`, optional
        The normalization biases. The `dict` keys must be the same as those as
        the instance attribute
        :attr:`emicroml.modelling.cbed.distortion.estimation.MLDataset.normalization_biases`
        of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLDataset`. The
        value of each `dict` item is expected to be a valid real number.
    check_ml_data_dict_first : `bool`, optional
        If ``check_ml_data_dict_first`` is set to ``True``, then
        ``ml_data_dict`` is checked, raising an exception if ``ml_data_dict`` is
        not a valid dictionary representation of ML data instances. Otherwise,
        ``ml_data_dict`` is not checked.

    """
    params = locals()

    global_symbol_table = globals()

    func_name = ("_check_and_convert_unnormalize_normalizable_elems"
                 "_in_ml_data_dict_params")
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    func_alias(**kwargs)

    return None



def _check_and_convert_unnormalize_normalizable_elems_in_ml_data_dict_params(
        params):
    original_param_names = tuple(params.keys())

    current_func_name = ("_check_and_convert_unnormalize_normalizable_elems"
                         "_in_ml_data_dict_params")
    module_alias = emicroml.modelling.cbed.distortion._common
    func_alias = getattr(module_alias, current_func_name)
    params = func_alias(params)

    return params



def _unnormalize_normalizable_elems_in_ml_data_dict(check_ml_data_dict_first,
                                                    normalization_weights,
                                                    normalization_biases,
                                                    ml_data_dict):
    params = locals()
    kwargs = params.copy()
    del kwargs["check_ml_data_dict_first"]
    try:
        current_func_name = "_unnormalize_normalizable_elems_in_ml_data_dict"
        module_alias = emicroml.modelling.cbed.distortion._common
        func_alias = getattr(module_alias, current_func_name)
        func_alias(**kwargs)
    except:
        func_name = ("_check_and_convert_unnormalize_normalizable_elems"
                     "_in_ml_data_dict_params")
        func_alias = globals()[func_name]
        func_alias(params)

    return None



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_lr_scheduler_manager = \
    _module_alias._default_lr_scheduler_manager
_default_checkpoints = \
    _module_alias._default_checkpoints
_default_output_dirname = \
    _module_alias._default_output_dirname
_default_misc_model_training_metadata = \
    _module_alias._default_misc_model_training_metadata



_module_alias = emicroml.modelling.cbed.distortion._common
_cls_alias = _module_alias._MLModelTrainer
class MLModelTrainer(_cls_alias):
    r"""A machine learning model trainer.

    The current class is a subclass of
    :class:`fancytypes.PreSerializableAndUpdatable`.

    The current class represents a machine learning (ML) model trainer that can
    be used to train via supervised learning ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`.

    As discussed in the documentation of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`, a given ML
    model represented by said class takes as input a mini-batch of images, where
    each image is assumed to depict a distorted CBED pattern, and as output, the
    ML model predicts sets of coordinate transformation parameters that specify
    the coordinate transformations that describe the distortions of the input
    images. The coordinate transformation used to describe the distortions of an
    image is defined in the documentation for the class
    :class:`distoptica.StandardCoordTransformParams`. The parameter set
    parameterizing said coordinate transformation is referred to as the
    "standard" coordinate transformation parameter set, and is represented by
    the class :class:`distoptica.StandardCoordTransformParams`. Similarly, we
    refer to distortions fields that are specified by standard coordinate
    transformation parameters sets as standard distortion fields.

    Consider an abstract undistorted CBED intensity pattern of non-overlapping
    CBED disks that share a common radius, and that outside the CBED disk
    supports the intensity is zero, and inside each CBED disk support the
    intensity is a common positive value. Next, consider an abstract distorted
    CBED intensity pattern obtained by distorting the abstract undistorted CBED
    intensity pattern according to a standard coordinate transformation. Under
    special circumstances, the abstract distorted CBED intensity pattern will
    not be perfectly correlated with the standard distortion field corresponding
    to the standard coordinate transformation. However, the abstract distorted
    CBED intensity pattern should be perfectly correlated with a vector field
    obtained by subtracting the original standard distortion field by its
    mean. We refer to the mean of the original standard distortion field as the
    "adjustment" vector.

    Mathematically, the loss associated with a given predicted mini-batch of
    standard coordinate transformation parameter sets, during either training or
    validation, is calculated effectively as follows:

    1. Let ``mini_batch_size`` be the mini-batch size.

    2. Index the predicted coordinate transformations from ``0`` to
    ``mini_batch_size-1``.

    3. Set ``losses_of_ml_data_instances`` to a floating-point array of shape
    ``(mini_batch_size,)``.

    4. Set ``n`` to ``-1``.

    5. Set ``n`` to ``n+1``.

    6. Set ``sampling_grid_dims_in_pixels`` to shape of ``n`` th input image.

    7. Use the ``n`` th predicted standard coordinate transformation parameter
    set to construct an instance ``predicted_standard_coord_transform_params``
    of the class :class:`distoptica.StandardCoordTransformParams`.

    8. Generate an instance ``predicted_distortion_model`` of the class
    :class:`distoptica.DistortionModel` by calculating:

    .. code-block:: python

        import distoptica

        kwargs = \
            {"standard_coord_transform_params": \
             predicted_standard_coord_transform_params,
             "sampling_grid_dims_in_pixels": \
             sampling_grid_dims_in_pixels}
        predicted_distortion_model = \
            distoptica.generate_standard_distortion_model(**kwargs)

    9. Sample the flow field of the ``n`` th predicted standard coordinate
    transformation on a uniform grid by calculating
    ``predicted_distortion_model.flow_field_of_coord_transform``, then store the
    result in ``predicted_flow_field``.

    10. Add to the sampled flow field from the previous step the adjustment
    vector, then store the result in ``predicted_adjusted_flow_field``. 

    11. Calculate the ground truth corresponding to the predicted sampled
    adjusted flow field from the previous step, then store the result in
    ``target_adjusted_flow_field``.

    12. Calculate the end-point error (EPE) of the predicted adjusted flow field
    stored in ``predicted_adjusted_flow_field``, using the ground truth stored
    in ``target_adjusted_flow_field``, then store the result in
    ``epe_of_distortion_field``.

    13. Store ``epe_of_distortion_field`` in ``losses_of_ml_data_instances[n]``.

    14. If ``n < mini_batch_size-1``, then go to instruction 5. Otherwise, go to
    instruction 15.

    15. Set ``mini_batch_loss`` to the average of the elements stored in
    ``losses_of_ml_data_instances``.

    16. Stop.

    The number ``mini_batch_loss`` is the loss associated with the given
    predicted mini-batch of standard coordinate transformation parameter sets.

    Note that the steps 6-12 describe the calculation of an EPE. Specifically,
    we refer to the resulting quantity as the EPE of an adjusted standard
    distortion field specified by a predicted standard coordinate
    transformation. We adopt this language elsewhere in the documentation of
    this module. 

    The above set of instructions made reference to a few objects from the
    :mod:`distoptica` library. See the documentation for said library for
    further details.

    Parameters
    ----------
    ml_dataset_manager : :class:`emicroml.modelling.cbed.distortion.estimation.MLDatasetManager`
        The ML dataset manager to use during ML model training. The ML dataset
        manager must specify at least a ML training dataset. 

        If a ML validation dataset is specified, then mini-batch losses are
        calculated during the validation phase in addition to the training
        phase. Otherwise, mini-batch losses are not calculated during the
        validation phase.

        Any ML testing dataset specified is ignored.
    device_name : `str` | `None`, optional
        This parameter specifies the device to be used to perform
        computationally intensive calls to PyTorch functions and to store
        intermediate arrays of the type :class:`torch.Tensor`. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.
    lr_scheduler_manager : :class:`emicroml.modelling.lr.LRSchedulerManager` | `None`, optional
        This parameter specifies the learning rate scheduler manager to use
        during ML model training. If ``lr_scheduler_manager`` is set to an
        instance of the class :class:`emicroml.modelling.lr.LRSchedulerManager`,
        then the learning rate scheduler manager is represented by the object
        ``lr_scheduler_manager``. Otherwise, if set to ``None``, then the
        parameter ``lr_scheduler_manager`` will be reassigned to the value of
        ``emicroml.modelling.lr.LRSchedulerManager()``. See the documentation
        for the class :class:`emicroml.modelling.lr.LRSchedulerManager` for a
        discussion on learning rate scheduler managers. Among other things,
        ``lr_scheduler_manager`` specifies whether the global learning rate
        multiplier of each learning rate scheduler is updated in the training or
        validation phase of each training-validation cycle, except the last.
    checkpoints : `array_like` (`int`, ndim=1) | `None`, optional
        This parameter specifies after which global optimization steps are the
        real-time dictionary representations of the ML model to be trained saved
        to output files. By global optimization step, we mean a single update
        applied to each fitting parameter of the ML model that is subject to
        updates during optimization. We refer to the moments when dictionary
        representations are saved as "checkpoints".

        If ``checkpoints`` is not set to ``None``, then it must be either an
        empty sequence, or a sequence of nonnegative integers.

        If ``checkpoints`` is set to ``None``, then only the final dictionary
        representation of the ML model is to be saved. Else if ``checkpoints``
        is set to an empty sequence, then no dictionary representations of the
        ML model are to be saved. Else if ``checkpoints`` is set to a nonempty
        sequence of nonnegative integers, and the global learning rate
        multipliers are updated during the training phase of each
        training-validation cycle except the last, then for every nonnegative
        integer ``n`` less than ``len(checkpoints)``, ``checkpoints[n]``
        specifies that the real-time dictionary representation of the ML model
        immediately after ``checkpoints[n]+1`` global optimization steps is to
        be saved as long as said number of global optimization steps are to be
        performed. Otherwise, if ``checkpoints`` is set to a nonempty sequence
        of nonnegative integers, and the global learning rate multipliers are
        updated during the validation phase of each training-validation cycle
        except the last, then for every nonnegative integer ``n`` less than
        ``len(checkpoints)``, ``checkpoints[n]`` specifies that the real-time
        dictionary representation of the ML model immediately after
        ``checkpoints[n]+1`` training epochs is to be saved as long as said
        number of training epochs are to occur.

        Note that if ``checkpoints`` is set to ``None``, then the parameter is
        reassigned to ``(lr_scheduler_manager.total_num_steps,)`` after all
        other parameter reassignments.
    output_dirname : `str`, optional
        The relative or absolute path to the directory in which all output files
        are saved.
    misc_model_training_metadata : `dict`, optional
        Miscellaneous ML model training metadata. Can be any `dict` object that
        is serializable, i.e. 
        ``import json; json.dumps(misc_model_training_metadata)`` must not raise
        an exception. Note that ``misc_model_training_metadata`` is not used to
        train ML models, but is serialized and saved as output. See the
        documentation for the method 
        :meth:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer.train_ml_model`,
        for details on how ``misc_model_training_metadata`` is saved as output.
    skip_validation_and_conversion : `bool`, optional
        Let ``validation_and_conversion_funcs`` and ``core_attrs`` denote the
        attributes :attr:`~fancytypes.Checkable.validation_and_conversion_funcs`
        and :attr:`~fancytypes.Checkable.core_attrs` respectively, both of which
        being `dict` objects.

        Let ``params_to_be_mapped_to_core_attrs`` denote the `dict`
        representation of the constructor parameters excluding the parameter
        ``skip_validation_and_conversion``, where each `dict` key ``key`` is a
        different constructor parameter name, excluding the name
        ``"skip_validation_and_conversion"``, and
        ``params_to_be_mapped_to_core_attrs[key]`` would yield the value of the
        constructor parameter with the name given by ``key``.

        If ``skip_validation_and_conversion`` is set to ``False``, then for each
        key ``key`` in ``params_to_be_mapped_to_core_attrs``,
        ``core_attrs[key]`` is set to ``validation_and_conversion_funcs[key]
        (params_to_be_mapped_to_core_attrs)``.

        Otherwise, if ``skip_validation_and_conversion`` is set to ``True``,
        then ``core_attrs`` is set to
        ``params_to_be_mapped_to_core_attrs.copy()``. This option is desired
        primarily when the user wants to avoid potentially expensive deep copies
        and/or conversions of the `dict` values of
        ``params_to_be_mapped_to_core_attrs``, as it is guaranteed that no
        copies or conversions are made in this case.

    """
    _validation_and_conversion_funcs_ = \
        {**_cls_alias._validation_and_conversion_funcs_,
         "ml_dataset_manager": _check_and_convert_ml_dataset_manager}

    _pre_serialization_funcs_ = \
        {**_cls_alias._pre_serialization_funcs_,
         "ml_dataset_manager": _pre_serialize_ml_dataset_manager}

    _de_pre_serialization_funcs_ = \
        {**_cls_alias._de_pre_serialization_funcs_,
         "ml_dataset_manager": _de_pre_serialize_ml_dataset_manager}


    
    def __init__(self,
                 ml_dataset_manager,
                 device_name=\
                 _default_device_name,
                 lr_scheduler_manager=\
                 _default_lr_scheduler_manager,
                 checkpoints=\
                 _default_checkpoints,
                 output_dirname=\
                 _default_output_dirname,
                 misc_model_training_metadata=\
                 _default_misc_model_training_metadata,
                 skip_validation_and_conversion=\
                 _default_skip_validation_and_conversion):
        ctor_params = {key: val
                       for key, val in locals().items()
                       if (key not in ("self", "__class__"))}
        
        module_alias = emicroml.modelling.cbed.distortion._common
        cls_alias = module_alias._MLModelTrainer
        kwargs = ctor_params
        cls_alias.__init__(self, **kwargs)

        return None



    def execute_post_core_attrs_update_actions(self):
        super().execute_post_core_attrs_update_actions()

        self._ml_model_cls = MLModel
                
        return None



    def train_ml_model(self, ml_model, ml_model_param_groups):
        r"""Train a machine learning model.

        See the summary documentation of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer`
        for additional context.

        Let ``core_attrs`` be the attribute
        :attr:`~fancytypes.Checkable.core_attrs`, ``lr_scheduler_manager`` be
        ``core_attrs["lr_scheduler_manager"]``, ``lr_schedulers`` be
        ``lr_scheduler_manager.core_attrs["lr_schedulers"]``,
        ``num_lr_schedulers`` be ``len(lr_schedulers)``, let ``checkpoints`` be
        ``core_attrs["checkpoints"]``, ``output_dirname`` be
        ``core_attrs["output_dirname"]``, and ``misc_model_training_metadata``
        be ``core_attrs["misc_model_training_metadata"]``.

        As discussed in the summary documentation of the current class, namely
        in the description of the core attribute ``checkpoints``, real-time
        dictionary representations of the machine learning (ML) model to be
        trained can be saved to output files at different moments during
        training called "checkpoints". For each nonnegative integer ``n`` less
        than ``len(checkpoints)``, the real-time dictionary representation of
        the ML model at the ``n`` th checkpoint is saved to a file at the file
        path
        ``output_dirname+"/ml_model_at_lr_step_{}.pth".format(checkpoints[n])``.
        To load/reconstruct a ML model from a dictionary representation stored
        in a file, users can use the function
        :func:`emicroml.modelling.cbed.distortion.estimation.load_ml_model_from_file`.

        The only other output file that is generated by the end of the ML model
        training is the ML model training summary output data file, which is an
        HDF5 file generated at the file path
        ``output_dirname+"/ml_model_training_summary_output_data.h5"``. The HDF5
        file is guaranteed to contain the following HDF5 objects:

        * ml_model_trainer_params: <HDF5 1D dataset>
    
        * num_training_mini_batches_per_epoch: <HDF5 0D dataset>

        * num_validation_mini_batches_per_epoch: <HDF5 0D dataset>

        - lr_schedules: <HDF5 group>

          * lr_schedule_0: <HDF5 1D dataset>

            + dim_0: "training mini batch instance idx" | "epoch"

        - ml_data_instance_metrics: <HDF5 group>

          - training: <HDF5 group>

            * epes_of_adjusted_distortion_fields <HDF5 1D dataset>

              + dim_0: "ml training data instance idx"

        - mini_batch_losses: <HDF5 group>

          - training: <HDF5 group>

            * total <HDF5 1D dataset>

              + dim_0: "training mini batch instance idx"

        Note that the sub-bullet points listed immediately below a given HDF5
        dataset display the HDF5 attributes associated with said HDF5
        dataset. Some HDF5 datasets have attributes with names of the form
        ``"dim_{}".format(i)`` with ``i`` being an integer. Attribute
        ``"dim_{}".format(i)`` of a given HDF5 dataset labels the ``i`` th
        dimension of the underlying array of the dataset.

        The HDF5 dataset at the HDF5 path ``"/ml_model_trainer_params"`` stores
        a serialized version of the attribute
        :attr:`~fancytypes.Checkable.core_attrs`, which is essentially the
        construction parameters used to construct an instance of the current
        class. From the output HDF5 file, users can reconstruct the instance of
        the current class that generated said output file by:

        .. code-block:: python

            import h5pywrappers
            import emicroml.modelling.cbed.distortion.estimation

            filename = (output_dirname 
                        +"/ml_model_training_summary_output_data.h5")

            kwargs = {"filename": filename,
                     "path_in_file": "ml_model_trainer_params"}
            json_document_id = h5pywrappers.obj.ID(**kwargs)

            serializable_rep = h5pywrappers.json.document.load(json_document_id)

            MLModelTrainer = \
                emicroml.modelling.cbed.distortion.estimation.MLModelTrainer
            ml_model_trainer = \
                MLModelTrainer.de_pre_serialize(serializable_rep)

        where ``ml_model_trainer`` is the reconstructed instance of the current
        class, and ``serializable_rep`` is a "pre-serialized" version of it. See
        the documentation for the class :class:`fancytypes.PreSerializable` for
        a discussion on pre-serialization.

        The zero-dimensional HDF5 dataset, i.e. scalar, at the HDF5 path
        ``"/num_training_mini_batches_per_epoch"`` stores the number of training
        mini-batches per epoch. 

        The zero-dimensional HDF5 dataset, i.e. scalar, at the HDF5 path
        ``"/num_validation_mini_batches_per_epoch"`` stores the number of
        validation mini-batches per epoch.

        The HDF5 dataset at the HDF5 path ``"/lr_schedules/lr_schedule_0"``
        stores the learning rate schedule according to which the ``0`` th subset
        of the ML model fitting parameters are optimized. Note that the subsets
        of ML model fitting parameters that are to be updated during training
        are specified by the parameter
        ``ml_model_param_groups``. 

        ``ml_model_param_groups`` must satisfy ``len(ml_model_param_groups) ==
        num_lr_schedulers``. Furthermore, for every nonnegative integer ``n``
        less than ``num_lr_schedulers``, ``import torch;
        torch.optim.AdamW(ml_model_param_groups[n])`` must not raise an
        exception.

        For every nonnegative integer ``n`` less than ``num_lr_schedulers``,
        ``ml_model_param_groups[n]`` is the subset of the ML model fitting
        parameters that are optimized according to the learning rate schedule
        specified by ``lr_schedulers[n]``.

        The HDF5 attribute ``"dim_0"`` of the HDF5 dataset at the HDF5 path
        ``"/lr_schedules/lr_schedule_0"`` is equal to ``"training mini batch
        instance idx"`` if the global learning rate multiplier of the ``0`` th
        learning rate scheduler is updated in the training phase of each
        training-validation cycle except the last. Otherwise, said HDF5
        attribute is equal to ``"epoch"``.

        More generally, for every nonnegative integer ``n`` less than
        ``num_lr_schedulers``, there is an HDF5 dataset at the HDF5 path
        ``"/lr_schedules/lr_schedule_{}".format(n)`` that stores the learning
        rate schedule according to which the ``n`` th subset of the ML model
        fitting parameters are optimized. Moreover, for every nonnegative
        integer ``m`` less than the number of elements in the HDF5 dataset at
        the HDF5 path ``"/lr_schedules/lr_schedule_{}".format(n)``, the ``m`` th
        data element of that HDF5 dataset is the value of the global learning
        rate multiplier of the ``n`` th learning schedule after ``m`` steps in
        said schedule. Note that all HDF5 datasets in the group at the HDF5 path
        ``"/lr_schedules"`` share the same HDF5 attribute in name and value,
        i.e. the attribute ``"dim_0"``.

        The HDF5 group at the HDF5 path ``"/ml_data_instance_metrics"`` stores
        the performance metrics that are tracked during training.

        The HDF5 dataset at the HDF5 path
        ``"/ml_data_instance_metrics/training/epes_of_adjusted_distortion_fields"``
        stores the end-point errors (EPEs) of the "adjusted" standard distortion
        fields specified by the predicted standard coordinate transformation
        parameter sets, during training. For every nonnegative integer ``m``
        less than the the total number of ML training data instances, the ``m``
        th element of the aforementioned HDF5 dataset is the EPE of the adjusted
        standard distortion field specified by the ``m`` th predicted standard
        standard coordinate transformation set, during training. See the summary
        documentation of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer`
        for a definition of an adjusted standard distortion field, and how the
        EPE is calculated exactly.

        If performance metrics are also calculated during validation, then the
        output HDF5 file will also include an additional HDF5 dataset, located
        at the HDF5 path
        ``"/ml_data_instance_metrics/validation/epes_of_adjusted_distortion_fields"``.
        One can simply replace every instance of the word "training" with
        "validation" in the previous paragraph to yield a description of the
        HDF5 dataset stored in HDF5 group at the HDF5 path
        ``"/ml_data_instance_metrics/validation"``.

        The HDF5 group at the HDF5 path ``"/mini_batch_losses"`` stores the
        mini-batch losses that are tracked during training, which are used to
        optimize the ML model.

        The HDF5 dataset at the HDF5 path
        ``"/mini_batch_losses/training/total"`` stores the mini-btach losses
        associated with the EPEs of the adjusted standard distortion fields
        specified by the predicted standard coordinate transformation parameter
        sets, during training. For every nonnegative integer ``m`` less than the
        total number of training mini-batches, the ``m`` th element of the
        aforementioned HDF5 dataset is the mean of the EPEs of the adjusted
        standard distortion fields specified by the ``m`` th predicted
        mini-batch of standard coordinate transformation sets, during training.

        If mini-batch losses are also calculated during validation, then the
        output HDF5 file will also include an additional HDF5 dataset, located
        at the HDF5 path ``"/mini_batch_losses/validation/total"``.  One can
        simply replace every instance of the word "training" with "validation"
        in the previous paragraph to yield a description of the HDF5 dataset
        stored in HDF5 group at the HDF5 path
        ``"/mini_batch_losses/validation"``.

        For further discussion on how losses are calculated, see the summary
        documentation of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer`.

        Parameters
        ----------
        ml_model : :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`
            The ML model to train.
        ml_model_param_groups : `array_like`
            The ML model fitting parameter groups.

        """
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        super().train_ml_model(**kwargs)

        return None



_module_alias = \
    emicroml.modelling.cbed.distortion._common
_default_misc_model_testing_metadata = \
    _module_alias._default_misc_model_testing_metadata



_module_alias = emicroml.modelling.cbed.distortion._common
_cls_alias = _module_alias._MLModelTester
class MLModelTester(_cls_alias):
    r"""A machine learning model tester.

    The current class is a subclass of
    :class:`fancytypes.PreSerializableAndUpdatable`.

    The current class represents a machine learning (ML) model tester that can
    be used to test ML models represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`.

    See the documentation for the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLModelTester.test_ml_model`
    for a discussion on how performance metrics are calculated and tracked
    during ML model testing.

    Parameters
    ----------
    ml_dataset_manager : :class:`emicroml.modelling.cbed.distortion.estimation.MLDatasetManager`
        The ML dataset manager to use during ML model testing. The ML dataset
        manager must specify at least a ML testing dataset. Any ML training and
        validation datasets specified are ignored.

        Note that ``ml_dataset_manager`` stores an integer
        ``ml_dataset_manager.core_attrs["mini_batch_size"]`` which specifies the
        mini-batch size to be used in evaluating ML models. This is different
        from the mini-batch size used for calculating mini-batch losses during
        testing, which is always equal to unity. Hence, each mini-batch loss is
        equivalent to the loss of a single ML data instance. Generally speaking,
        the higher the value of
        ``ml_dataset_manager.core_attrs["mini_batch_size"]``, the faster the
        testing of ML models since more parallelization is being used.
    device_name : `str` | `None`, optional
        This parameter specifies the device to be used to perform
        computationally intensive calls to PyTorch functions and to store
        intermediate arrays of the type :class:`torch.Tensor`. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.
    output_dirname : `str`, optional
        The relative or absolute path to the directory in which all output files
        are saved.
    misc_model_testing_metadata : `dict`, optional
        Miscellaneous ML model testing metadata. Can be any `dict` object that
        is serializable, i.e.  ``import json;
        json.dumps(misc_model_testing_metadata)`` must not raise an
        exception. Note that ``misc_model_testing_metadata`` is not used to test
        ML models, but is serialized and saved as output. See the documentation
        for the method
        :meth:`emicroml.modelling.cbed.distortion.estimation.MLModelTester.test_ml_model`,
        for details on how ``misc_model_testing_metadata`` is saved as output.
    skip_validation_and_conversion : `bool`, optional
        Let ``validation_and_conversion_funcs`` and ``core_attrs`` denote the
        attributes :attr:`~fancytypes.Checkable.validation_and_conversion_funcs`
        and :attr:`~fancytypes.Checkable.core_attrs` respectively, both of which
        being `dict` objects.

        Let ``params_to_be_mapped_to_core_attrs`` denote the `dict`
        representation of the constructor parameters excluding the parameter
        ``skip_validation_and_conversion``, where each `dict` key ``key`` is a
        different constructor parameter name, excluding the name
        ``"skip_validation_and_conversion"``, and
        ``params_to_be_mapped_to_core_attrs[key]`` would yield the value of the
        constructor parameter with the name given by ``key``.

        If ``skip_validation_and_conversion`` is set to ``False``, then for each
        key ``key`` in ``params_to_be_mapped_to_core_attrs``,
        ``core_attrs[key]`` is set to ``validation_and_conversion_funcs[key]
        (params_to_be_mapped_to_core_attrs)``.

        Otherwise, if ``skip_validation_and_conversion`` is set to ``True``,
        then ``core_attrs`` is set to
        ``params_to_be_mapped_to_core_attrs.copy()``. This option is desired
        primarily when the user wants to avoid potentially expensive deep copies
        and/or conversions of the `dict` values of
        ``params_to_be_mapped_to_core_attrs``, as it is guaranteed that no
        copies or conversions are made in this case.

    """
    _validation_and_conversion_funcs_ = \
        {**_cls_alias._validation_and_conversion_funcs_,
         "ml_dataset_manager": _check_and_convert_ml_dataset_manager}

    _pre_serialization_funcs_ = \
        {**_cls_alias._pre_serialization_funcs_,
         "ml_dataset_manager": _pre_serialize_ml_dataset_manager}

    _de_pre_serialization_funcs_ = \
        {**_cls_alias._de_pre_serialization_funcs_,
         "ml_dataset_manager": _de_pre_serialize_ml_dataset_manager}

    
    
    def __init__(self,
                 ml_dataset_manager,
                 device_name=\
                 _default_device_name,
                 output_dirname=\
                 _default_output_dirname,
                 misc_model_testing_metadata=\
                 _default_misc_model_testing_metadata,
                 skip_validation_and_conversion=\
                 _default_skip_validation_and_conversion):
        ctor_params = {key: val
                       for key, val in locals().items()
                       if (key not in ("self", "__class__"))}
        
        module_alias = emicroml.modelling.cbed.distortion._common
        cls_alias = module_alias._MLModelTester
        kwargs = ctor_params
        cls_alias.__init__(self, **kwargs)

        return None



    def execute_post_core_attrs_update_actions(self):
        super().execute_post_core_attrs_update_actions()

        self._ml_model_cls = MLModel
                
        return None



    def test_ml_model(self, ml_model):
        r"""Test a machine learning model.

        See the summary documentation of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTester`
        for additional context.

        Let ``core_attrs`` be the attribute
        :attr:`~fancytypes.Checkable.core_attrs`, ``output_dirname`` be
        ``core_attrs["output_dirname"]``, and ``misc_model_testing_metadata`` be
        ``core_attrs["misc_model_testing_metadata"]``.

        The only output file that is generated by the end of the ML model
        testing is the ML model testing summary output data file, which is an
        HDF5 file generated at the file path
        ``output_dirname+"/ml_model_testing_summary_output_data.h5"``. The HDF5
        file is guaranteed to contain the following HDF5 objects:

        * ml_model_tester_params: <HDF5 1D dataset>
    
        * total_num_ml_testing_data_instances: <HDF5 0D dataset>

        - ml_data_instance_metrics: <HDF5 group>

          - testing: <HDF5 group>

            * epes_of_adjusted_distortion_fields <HDF5 1D dataset>

              + dim_0: "ml testing data instance idx"

        Note that the sub-bullet points listed immediately below a given HDF5
        dataset display the HDF5 attributes associated with said HDF5
        dataset. Some HDF5 datasets have attributes with names of the form
        ``"dim_{}".format(i)`` with ``i`` being an integer. Attribute
        ``"dim_{}".format(i)`` of a given HDF5 dataset labels the ``i`` th
        dimension of the underlying array of the dataset.

        The HDF5 dataset at the HDF5 path ``"/ml_model_tester_params"`` stores a
        serialized version of the attribute
        :attr:`~fancytypes.Checkable.core_attrs`, which is essentially the
        construction parameters used to construct an instance of the current
        class. From the output HDF5 file, users can reconstruct the instance of
        the current class that generated said output file by:

        .. code-block:: python

            import h5pywrappers
            import emicroml.modelling.cbed.distortion.estimation

            filename = (output_dirname 
                        +"/ml_model_testing_summary_output_data.h5")

            kwargs = {"filename": filename,
                     "path_in_file": "ml_model_tester_params"}
            json_document_id = h5pywrappers.obj.ID(**kwargs)

            serializable_rep = h5pywrappers.json.document.load(json_document_id)

            MLModelTester = \
                emicroml.modelling.cbed.distortion.estimation.MLModelTester
            ml_model_tester = \
                MLModelTester.de_pre_serialize(serializable_rep)

        where ``ml_model_tester`` is the reconstructed instance of the current
        class, and ``serializable_rep`` is a "pre-serialized" version of it. See
        the documentation for the class :class:`fancytypes.PreSerializable` for
        a discussion on pre-serialization.

        The zero-dimensional HDF5 dataset, i.e. scalar, at the HDF5 path
        ``"/total_num_ml_testing_data_instances"`` stores the total number of ML
        testing data instances.

        The HDF5 group at the HDF5 path ``"/ml_data_instance_metrics"`` stores
        the performance metrics that are tracked during testing.

        The HDF5 dataset at the HDF5 path
        ``"/ml_data_instance_metrics/testing/epes_of_adjusted_distortion_fields"``
        stores the end-point errors (EPEs) of the "adjusted" standard distortion
        fields specified by the predicted standard coordinate transformation
        parameter sets, during testing. For every nonnegative integer ``m`` less
        than the the total number of ML testing data instances, the ``m`` th
        element of the aforementioned HDF5 dataset is the EPE of the adjusted
        standard distortion field specified by the ``m`` th predicted standard
        standard coordinate transformation set, during testing. See the summary
        documentation of the class
        :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer`
        for a definition of an adjusted standard distortion field, and how the
        EPE is calculated exactly.

        Parameters
        ----------
        ml_model : :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`
            The ML model to test.

        """
        kwargs = {key: val
                  for key, val in locals().items()
                  if (key not in ("self", "__class__"))}
        super().test_ml_model(**kwargs)

        return None



def load_ml_model_from_file(ml_model_state_dict_filename,
                            device_name=_default_device_name):
    r"""Load a machine learning model from a file.

    The current function loads/reconstructs machine learning (ML) models,
    represented by the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`, from files
    storing dictionary representations of said ML models.

    Dictionary representations of ML models can be generated via the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLModel.state_dict`.
    Subsequently, these dictionaries can be saved to files via the function
    :func:`torch.save`. For further details see the documentation for the
    function :func:`torch.load`.

    Moreover, dictionary representations of ML models can be generated then
    saved to files via the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer.train_ml_model`
    of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer`. For
    further details see the documentation for said method.

    Parameters
    ----------
    ml_model_state_dict_filename : `str`
        The relative or absolute path to the file storing the dictionary
        representation of the ML model to load/reconstruct.
    device_name : `str` | `None`, optional
        This parameter specifies the device in which to store the ML model. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.

    Returns
    -------
    ml_model : :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`
        The ML model represented by the dictionary stored in the file at the 
        file path ``ml_model_state_dict_filename``.

    """
    params = locals()
    params["ml_model_cls"] = MLModel

    global_symbol_table = globals()

    func_name = "_check_and_convert_load_ml_model_from_file_params"
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    ml_model = func_alias(**kwargs)

    return ml_model



def _check_and_convert_load_ml_model_from_file_params(params):
    module_alias = emicroml.modelling._common
    func_alias = module_alias._check_and_convert_load_ml_model_from_file_params
    params = func_alias(params)

    return params



def _load_ml_model_from_file(ml_model_state_dict_filename,
                             device_name,
                             ml_model_cls):
    kwargs = locals()
    module_alias = emicroml.modelling._common
    func_alias = module_alias._load_ml_model_from_file
    ml_model = func_alias(**kwargs)

    return ml_model



def load_ml_model_from_state_dict(ml_model_state_dict,
                                  device_name=_default_device_name):
    r"""Load a machine learning model from a dictionary

    The current function loads machine learning (ML) models, represented by the
    class :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`, from
    dictionary representations of said ML models.

    Dictionary representations of ML models can be generated via the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLModel.state_dict`.

    Moreover, dictionary representations of ML models can be generated then
    saved to files via the method
    :meth:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer.train_ml_model`
    of the class
    :class:`emicroml.modelling.cbed.distortion.estimation.MLModelTrainer`. For
    further details see the documentation for said method.

    Let ``ml_model_state_dict_filename`` be the relative or absolute path to a
    file storing a dictionary representation of an ML model of interest. One can
    load said dictionary representation via the function :func:`torch.load`,
    where the function parameter ``f`` should be set to
    ``ml_model_state_dict_filename``. In this case, the function
    :func:`torch.load` should return the dictionary representation. For further
    details see the documentation for the function :func:`torch.load`.

    Parameters
    ----------
    ml_model_state_dict : `dict`
        The dictionary representation of the ML model to load.
    device_name : `str` | `None`, optional
        This parameter specifies the device in which to store the ML model. If
        ``device_name`` is a string, then it is the name of the device to be
        used, e.g. ``”cuda”`` or ``”cpu”``. If ``device_name`` is set to
        ``None`` and a GPU device is available, then a GPU device is to be
        used. Otherwise, the CPU is used.

    Returns
    -------
    ml_model : :class:`emicroml.modelling.cbed.distortion.estimation.MLModel`
        The ML model represented by the dictionary ``ml_model_state_dict``.

    """
    params = locals()
    params["ml_model_cls"] = MLModel

    global_symbol_table = globals()

    func_name = "_check_and_convert_load_ml_model_from_state_dict_params"
    func_alias = global_symbol_table[func_name]
    params = func_alias(params)

    func_name = func_name[18:-7]
    func_alias = global_symbol_table[func_name]
    kwargs = params
    ml_model = func_alias(**kwargs)

    return ml_model



def _check_and_convert_load_ml_model_from_state_dict_params(params):
    module_alias = \
        emicroml.modelling._common
    func_alias = \
        module_alias._check_and_convert_load_ml_model_from_state_dict_params
    params = \
        func_alias(params)

    return params



def _load_ml_model_from_state_dict(ml_model_state_dict,
                                   device_name,
                                   ml_model_cls):
    kwargs = locals()
    module_alias = emicroml.modelling._common
    func_alias = module_alias._load_ml_model_from_state_dict
    ml_model = func_alias(**kwargs)

    return ml_model



###########################
## Define error messages ##
###########################
