# This file is part of pipe_base.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This software is dual licensed under the GNU General Public License and also
# under a 3-clause BSD license. Recipients may choose which of these licenses
# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
# respectively.  If you choose the GPL option then the following text applies
# (but note that there is still no warranty even if you opt for BSD instead):
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import annotations

__all__ = ("PipelineGraph", "log_config_mismatch", "compare_packages")

import gzip
import itertools
import json
import logging
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set
from typing import TYPE_CHECKING, Any, BinaryIO, Literal, TypeVar, cast

import networkx
import networkx.algorithms.bipartite
import networkx.algorithms.dag
from lsst.daf.butler import (
    Butler,
    DataCoordinate,
    DataId,
    DatasetRef,
    DatasetType,
    DimensionGroup,
    DimensionUniverse,
    MissingDatasetTypeError,
)
from lsst.daf.butler.registry import ConflictingDefinitionError, Registry
from lsst.resources import ResourcePath, ResourcePathExpression
from lsst.utils.packages import Packages

from .._dataset_handle import InMemoryDatasetHandle
from ..automatic_connection_constants import PACKAGES_INIT_OUTPUT_NAME, PACKAGES_INIT_OUTPUT_STORAGE_CLASS
from ._dataset_types import DatasetTypeNode
from ._edges import Edge, ReadEdge, WriteEdge
from ._exceptions import (
    DuplicateOutputError,
    EdgesChangedError,
    PipelineDataCycleError,
    PipelineGraphError,
    PipelineGraphExceptionSafetyError,
    UnresolvedGraphError,
)
from ._mapping_views import DatasetTypeMappingView, TaskMappingView
from ._nodes import NodeKey, NodeType
from ._task_subsets import TaskSubset
from ._tasks import TaskImportMode, TaskInitNode, TaskNode, _TaskNodeImportedData

if TYPE_CHECKING:
    from ..config import PipelineTaskConfig
    from ..connections import PipelineTaskConnections
    from ..pipeline import TaskDef
    from ..pipelineTask import PipelineTask


_G = TypeVar("_G", bound=networkx.DiGraph | networkx.MultiDiGraph)

_LOG = logging.getLogger("lsst.pipe.base.pipeline_graph")


class PipelineGraph:
    """A graph representation of fully-configured pipeline.

    `PipelineGraph` instances are typically constructed by calling
    `.Pipeline.to_graph`, but in rare cases constructing and then populating an
    empty one may be preferable.

    Parameters
    ----------
    description : `str`, optional
        String description for this pipeline.
    universe : `lsst.daf.butler.DimensionUniverse`, optional
        Definitions for all butler dimensions.  If not provided, some
        attributes will not be available until `resolve` is called.
    data_id : `lsst.daf.butler.DataCoordinate` or other data ID, optional
        Data ID that represents a constraint on all quanta generated by this
        pipeline.  This typically just holds the instrument constraint included
        in the pipeline definition, if there was one.
    """

    ###########################################################################
    #
    # Simple Pipeline Graph Inspection Interface:
    #
    # - for inspecting graph structure, not modifying it (except to sort and]
    #   resolve);
    #
    # - no NodeKey objects, just string dataset type name and task label keys;
    #
    # - graph structure is represented as a pair of mappings, with methods to
    #   find neighbors and edges of nodes.
    #
    ###########################################################################

    def __init__(
        self,
        *,
        description: str = "",
        universe: DimensionUniverse | None = None,
        data_id: DataId | None = None,
    ) -> None:
        self._init_from_args(
            xgraph=None,
            sorted_keys=None,
            task_subsets=None,
            description=description,
            universe=universe,
            data_id=data_id,
        )

    def __repr__(self) -> str:
        return f"{type(self).__name__}({self.description!r}, tasks={self.tasks!s})"

    @property
    def description(self) -> str:
        """String description for this pipeline."""
        return self._description

    @description.setter
    def description(self, value: str) -> None:
        # Docstring in setter.
        self._description = value

    @property
    def universe(self) -> DimensionUniverse | None:
        """Definitions for all butler dimensions."""
        return self._universe

    @property
    def data_id(self) -> DataCoordinate:
        """Data ID that represents a constraint on all quanta generated from
        this pipeline.

        This is may not be available unless `universe` is not `None`.
        """
        return DataCoordinate.standardize(self._raw_data_id, universe=self.universe)

    @property
    def tasks(self) -> TaskMappingView:
        """A mapping view of the tasks in the graph.

        This mapping has `str` task label keys and `TaskNode` values. Iteration
        is topologically and deterministically ordered if and only if `sort`
        has been called since the last modification to the graph.
        """
        return self._tasks

    @property
    def dataset_types(self) -> DatasetTypeMappingView:
        """A mapping view of the dataset types in the graph.

        This mapping has `str` parent dataset type name keys, but only provides
        access to its `DatasetTypeNode` values if `resolve` has been called
        since the last modification involving a task that uses a dataset type.
        See `DatasetTypeMappingView` for details.
        """
        return self._dataset_types

    @property
    def task_subsets(self) -> Mapping[str, TaskSubset]:
        """A mapping of all labeled subsets of tasks.

        Keys are subset labels, values are sets of task labels.  See
        `TaskSubset` for more information.

        Use `add_task_subset` to add a new subset.  The subsets themselves may
        be modified in-place.
        """
        return self._task_subsets

    @property
    def is_fully_resolved(self) -> bool:
        """Whether all of this graph's nodes are resolved."""
        return self._universe is not None and all(
            self.dataset_types.is_resolved(k) for k in self.dataset_types
        )

    @property
    def is_sorted(self) -> bool:
        """Whether this graph's tasks and dataset types are topologically
        sorted with the exact same deterministic tiebreakers that `sort` would
        apply.

        This may perform (and then discard) a full sort if `has_been_sorted` is
        `False`.  If the goal is to obtain a sorted graph, it is better to just
        call `sort` without guarding that with an ``if not graph.is_sorted``
        check.
        """
        if self._sorted_keys is not None:
            return True
        return all(
            sorted == unsorted
            for sorted, unsorted in zip(
                networkx.lexicographical_topological_sort(self._xgraph), self._xgraph, strict=True
            )
        )

    @property
    def has_been_sorted(self) -> bool:
        """Whether this graph's tasks and dataset types have been
        topologically sorted (with unspecified but deterministic tiebreakers)
        since the last modification to the graph.

        This may return `False` if the graph *happens* to be sorted but `sort`
        was never called, but it is potentially much faster than `is_sorted`,
        which may attempt (and then discard) a full sort if `has_been_sorted`
        is `False`.
        """
        return self._sorted_keys is not None

    def sort(self) -> None:
        """Sort this graph's nodes topologically with deterministic (but
        unspecified) tiebreakers.

        This does nothing if the graph is already known to be sorted.
        """
        if self._sorted_keys is None:
            try:
                sorted_keys: Sequence[NodeKey] = list(networkx.lexicographical_topological_sort(self._xgraph))
            except networkx.NetworkXUnfeasible as err:  # pragma: no cover
                # Should't be possible to get here, because we check for cycles
                # when adding tasks, but we guard against it anyway.
                cycle = networkx.find_cycle(self._xgraph)
                raise PipelineDataCycleError(
                    f"Cycle detected while attempting to sort graph: {cycle}."
                ) from err
            self._reorder(sorted_keys)

    def copy(self) -> PipelineGraph:
        """Return a copy of this graph that copies all mutable state."""
        xgraph = self._xgraph.copy()
        result = PipelineGraph.__new__(PipelineGraph)
        result._init_from_args(
            xgraph,
            self._sorted_keys,
            task_subsets={
                k: TaskSubset(xgraph, v.label, set(v._members), v.description)
                for k, v in self._task_subsets.items()
            },
            description=self._description,
            universe=self.universe,
            data_id=self._raw_data_id,
        )
        return result

    def __copy__(self) -> PipelineGraph:
        # Fully shallow copies are dangerous; we don't want shared mutable
        # state to lead to broken class invariants.
        return self.copy()

    def __deepcopy__(self, memo: dict) -> PipelineGraph:
        # Genuine deep copies are unnecessary, since we should only ever care
        # that mutable state is copied.
        return self.copy()

    def diff_tasks(self, other: PipelineGraph) -> list[str]:
        """Compare two pipeline graphs.

        This only compares graph structure and task classes (including their
        edges).  It does *not* compare full configuration (which is subject to
        spurious differences due to import-cache state), dataset type
        resolutions, or sort state.

        Parameters
        ----------
        other : `PipelineGraph`
            Graph to compare to.

        Returns
        -------
        differences : `list` [ `str` ]
            List of string messages describing differences between the
            pipelines.  If empty, the graphs have the same tasks and
            connections.
        """
        messages: list[str] = []
        common_labels: Set[str]
        if self.tasks.keys() != other.tasks.keys():
            common_labels = self.tasks.keys() & other.tasks.keys()
            messages.append(
                f"Pipelines have different tasks: A & ~B = {list(self.tasks.keys() - common_labels)}, "
                f"B & ~A = {list(other.tasks.keys() - common_labels)}."
            )
        else:
            common_labels = self.tasks.keys()
        for label in common_labels:
            a = self.tasks[label]
            b = other.tasks[label]
            if a.task_class != b.task_class:
                messages.append(
                    f"Task {label!r} has class {a.task_class_name} in A, " f"but {b.task_class_name} in B."
                )
            messages.extend(a.diff_edges(b))
        return messages

    def producing_edge_of(self, dataset_type_name: str) -> WriteEdge | None:
        """Return the `WriteEdge` that links the producing task to the named
        dataset type.

        Parameters
        ----------
        dataset_type_name : `str`
            Dataset type name.  Must not be a component.

        Returns
        -------
        edge : `WriteEdge` or `None`
            Producing edge or `None` if there isn't one in this graph.

        Raises
        ------
        DuplicateOutputError
            Raised if there are multiple tasks defined to produce this dataset
            type. This is only possible if the graph's dataset types are not
            resolved.

        Notes
        -----
        On resolved graphs, it may be slightly more efficient to use::

            graph.dataset_types[dataset_type_name].producing_edge

        but this method works on graphs with unresolved dataset types as well.
        """
        producer: str | None = None
        producing_edge: WriteEdge | None = None
        for _, _, producing_edge in self._xgraph.in_edges(
            NodeKey(NodeType.DATASET_TYPE, dataset_type_name), data="instance"
        ):
            assert producing_edge is not None, "Should only be None if we never loop."
            if producer is not None:
                raise DuplicateOutputError(
                    f"Dataset type {dataset_type_name!r} is produced by both {producing_edge.task_label!r} "
                    f"and {producer!r}."
                )
        return producing_edge

    def consuming_edges_of(self, dataset_type_name: str) -> list[ReadEdge]:
        """Return the `ReadEdge` objects that link the named dataset type to
        the tasks that consume it.

        Parameters
        ----------
        dataset_type_name : `str`
            Dataset type name.  Must not be a component.

        Returns
        -------
        edges : `list` [ `ReadEdge` ]
            Edges that connect this dataset type to the tasks that consume it.

        Notes
        -----
        On resolved graphs, it may be slightly more efficient to use::

            graph.dataset_types[dataset_type_name].producing_edges

        but this method works on graphs with unresolved dataset types as well.
        """
        return [
            edge
            for _, _, edge in self._xgraph.out_edges(
                NodeKey(NodeType.DATASET_TYPE, dataset_type_name), data="instance"
            )
        ]

    def producer_of(self, dataset_type_name: str) -> TaskNode | TaskInitNode | None:
        """Return the `TaskNode` or `TaskInitNode` that writes the given
        dataset type.

        Parameters
        ----------
        dataset_type_name : `str`
            Dataset type name.  Must not be a component.

        Returns
        -------
        edge : `TaskNode`, `TaskInitNode`, or `None`
            Producing node or `None` if there isn't one in this graph.

        Raises
        ------
        DuplicateOutputError
            Raised if there are multiple tasks defined to produce this dataset
            type. This is only possible if the graph's dataset types are not
            resolved.
        """
        if (producing_edge := self.producing_edge_of(dataset_type_name)) is not None:
            return self._xgraph.nodes[producing_edge.task_key]["instance"]
        return None

    def consumers_of(self, dataset_type_name: str) -> list[TaskNode | TaskInitNode]:
        """Return the  `TaskNode` and/or `TaskInitNode` objects that read
        the given dataset type.

        Parameters
        ----------
        dataset_type_name : `str`
            Dataset type name.  Must not be a component.

        Returns
        -------
        edges : `list` [ `ReadEdge` ]
            Edges that connect this dataset type to the tasks that consume it.

        Notes
        -----
        On resolved graphs, it may be slightly more efficient to use::

            graph.dataset_types[dataset_type_name].producing_edges

        but this method works on graphs with unresolved dataset types as well.
        """
        return [
            self._xgraph.nodes[consuming_edge.task_key]["instance"]
            for consuming_edge in self.consuming_edges_of(dataset_type_name)
        ]

    def inputs_of(self, task_label: str, init: bool = False) -> dict[str, DatasetTypeNode | None]:
        """Return the dataset types that are inputs to a task.

        Parameters
        ----------
        task_label : `str`
            Label for the task in the pipeline.
        init : `bool`, optional
            If `True`, return init-input dataset types instead of runtime
            (including prerequisite) inputs.

        Returns
        -------
        inputs : `dict` [ `str`, `DatasetTypeNode` or `None` ]
            Dictionary parent dataset type name keys and either
            `DatasetTypeNode` values (if the dataset type has been resolved)
            or `None` values.

        Notes
        -----
        To get the input edges of a task or task init node (which provide
        information about storage class overrides nd components) use::

            graph.tasks[task_label].iter_all_inputs()

        or

            graph.tasks[task_label].init.iter_all_inputs()

        or the various mapping attributes of the `TaskNode` and `TaskInitNode`
        class.
        """
        node: TaskNode | TaskInitNode = self.tasks[task_label] if not init else self.tasks[task_label].init
        return {
            edge.parent_dataset_type_name: self._xgraph.nodes[edge.dataset_type_key]["instance"]
            for edge in node.iter_all_inputs()
        }

    def outputs_of(
        self, task_label: str, init: bool = False, include_automatic_connections: bool = True
    ) -> dict[str, DatasetTypeNode | None]:
        """Return the dataset types that are outputs of a task.

        Parameters
        ----------
        task_label : `str`
            Label for the task in the pipeline.
        init : `bool`, optional
            If `True`, return init-output dataset types instead of runtime
            outputs.
        include_automatic_connections : `bool`, optional
            Whether to include automatic connections such as configs, metadata,
            and logs.

        Returns
        -------
        outputs : `dict` [ `str`, `DatasetTypeNode` or `None` ]
            Dictionary parent dataset type name keys and either
            `DatasetTypeNode` values (if the dataset type has been resolved)
            or `None` values.

        Notes
        -----
        To get the input edges of a task or task init node (which provide
        information about storage class overrides nd components) use::

            graph.tasks[task_label].iter_all_outputs()

        or

            graph.tasks[task_label].init.iter_all_outputs()

        or the various mapping attributes of the `TaskNode` and `TaskInitNode`
        class.
        """
        node: TaskNode | TaskInitNode = self.tasks[task_label] if not init else self.tasks[task_label].init
        iterable = node.iter_all_outputs() if include_automatic_connections else node.outputs.values()
        return {
            edge.parent_dataset_type_name: self._xgraph.nodes[edge.dataset_type_key]["instance"]
            for edge in iterable
        }

    def resolve(
        self,
        registry: Registry | None = None,
        dimensions: DimensionUniverse | None = None,
        dataset_types: Mapping[str, DatasetType] | None = None,
        visualization_only: bool = False,
    ) -> None:
        """Resolve all dimensions and dataset types and check them for
        consistency.

        Resolving a graph also causes it to be sorted.

        Parameters
        ----------
        registry : `lsst.daf.butler.Registry`, optional
            Client for the data repository to resolve against.
        dimensions : `lsst.daf.butler.DimensionUniverse`, optional
            Definitions for all dimensions.  Takes precedence over
            ``registry.dimensions`` if both are provided.  If neither is
            provided, defaults to the default dimension universe
            (``lsst.daf.butler.DimensionUniverse()``).
        dataset_types : `~collection.abc.Mapping` [ `str`, \
                `~lsst.daf.butler.DatasetType` ], optional
            Mapping of dataset types to consider registered.  Takes precedence
            over ``registry.getDatasetType()`` if both are provided.
        visualization_only : `bool`, optional
            Resolve the graph as well as possible even when dimensions and
            storage classes cannot really be determined.  This can include
            using the ``universe.commonSkyPix`` as the assumed dimensions of
            connections that use the "skypix" placeholder and using "<UNKNOWN>"
            as a storage class name (which will fail if the storage class
            itself is ever actually loaded).

        Notes
        -----
        The `universe` attribute is set to ``dimensions`` and used to set all
        `TaskNode.dimensions` attributes.  Dataset type nodes are resolved by
        first looking for a registry definition, then using the producing
        task's definition, then looking for consistency between all consuming
        task definitions.

        Raises
        ------
        ConnectionTypeConsistencyError
            Raised if a prerequisite input for one task appears as a different
            kind of connection in any other task.
        DuplicateOutputError
            Raised if multiple tasks have the same dataset type as an output.
        IncompatibleDatasetTypeError
            Raised if different tasks have different definitions of a dataset
            type.  Different but compatible storage classes are permitted.
        MissingDatasetTypeError
            Raised if a dataset type definition is required to exist in the
            data repository but none was found.  This should only occur for
            dataset types that are not produced by a task in the pipeline and
            are consumed with different storage classes or as components by
            tasks in the pipeline.
        EdgesChangedError
            Raised if ``check_edges_unchanged=True`` and the edges of a task do
            change after import and reconfiguration.
        """
        get_registered: Callable[[str], DatasetType | None] | None = None
        if dataset_types is not None:
            # Ruff seems confused about whether this is used below; it is!
            get_registered = dataset_types.get
        elif registry is not None:

            def get_registered(name: str) -> DatasetType | None:
                try:
                    return registry.getDatasetType(name)
                except MissingDatasetTypeError:
                    return None

        else:

            def get_registered(name: str) -> None:
                return None

        if dimensions is None:
            if registry is not None:
                dimensions = registry.dimensions
            else:
                dimensions = DimensionUniverse()

        node_key: NodeKey
        updates: dict[NodeKey, TaskNode | DatasetTypeNode] = {}
        for node_key, node_state in self._xgraph.nodes.items():
            match node_key.node_type:
                case NodeType.TASK:
                    task_node: TaskNode = node_state["instance"]
                    new_task_node = task_node._resolved(dimensions)
                    if new_task_node is not task_node:
                        updates[node_key] = new_task_node
                case NodeType.DATASET_TYPE:
                    dataset_type_node: DatasetTypeNode | None = node_state["instance"]
                    new_dataset_type_node = DatasetTypeNode._from_edges(
                        node_key,
                        self._xgraph,
                        get_registered,
                        dimensions,
                        previous=dataset_type_node,
                        visualization_only=visualization_only,
                    )
                    # Usage of `is`` here is intentional; `_from_edges` returns
                    # `previous=dataset_type_node` if it can determine that it
                    # doesn't need to change.
                    if new_dataset_type_node is not dataset_type_node:
                        updates[node_key] = new_dataset_type_node
        try:
            for node_key, node_value in updates.items():
                self._xgraph.nodes[node_key]["instance"] = node_value
        except Exception as err:  # pragma: no cover
            # There's no known way to get here, but we want to make it
            # clear it's a big problem if we do.
            raise PipelineGraphExceptionSafetyError(
                "Error during dataset type resolution has left the graph in an inconsistent state."
            ) from err
        self.sort()
        self._universe = dimensions

    ###########################################################################
    #
    # Graph Modification Interface:
    #
    # - methods to add, remove, and replace tasks;
    #
    # - methods to add and remove task subsets.
    #
    # These are all things that are usually done in a Pipeline before making a
    # graph at all, but there may be cases where we want to modify the graph
    # instead.  (These are also the methods used to make a graph from a
    # Pipeline, or make a graph from another graph.)
    #
    ###########################################################################

    def add_task(
        self,
        label: str | None,
        task_class: type[PipelineTask],
        config: PipelineTaskConfig | None = None,
        connections: PipelineTaskConnections | None = None,
    ) -> TaskNode:
        """Add a new task to the graph.

        Parameters
        ----------
        label : `str` or `None`
            Label for the task in the pipeline.  If `None`, `Task._DefaultName`
            is used.
        task_class : `type` [ `PipelineTask` ]
            Class object for the task.
        config : `PipelineTaskConfig`, optional
            Configuration for the task.  If not provided, a default-constructed
            instance of ``task_class.ConfigClass`` is used.
        connections : `PipelineTaskConnections`, optional
            Object that describes the dataset types used by the task.  If not
            provided, one will be constructed from the given configuration.  If
            provided, it is assumed that ``config`` has already been validated
            and frozen.

        Returns
        -------
        node : `TaskNode`
            The new task node added to the graph.

        Raises
        ------
        ValueError
            Raised if configuration validation failed when constructing
            ``connections``.
        PipelineDataCycleError
            Raised if the graph is cyclic after this addition.
        RuntimeError
            Raised if an unexpected exception (which will be chained) occurred
            at a stage that may have left the graph in an inconsistent state.
            Other exceptions should leave the graph unchanged.

        Notes
        -----
        Checks for dataset type consistency and multiple producers do not occur
        until `resolve` is called, since the resolution depends on both the
        state of the data repository and all contributing tasks.

        Adding new tasks removes any existing resolutions of all dataset types
        it references and marks the graph as unsorted.  It is most effiecient
        to add all tasks up front and only then resolve and/or sort the graph.
        """
        if label is None:
            label = task_class._DefaultName
        if config is None:
            config = task_class.ConfigClass()
        _LOG.debug("Adding task %s %s to the pipeline graph", label, task_class)
        task_node = TaskNode._from_imported_data(
            key=NodeKey(NodeType.TASK, label),
            init_key=NodeKey(NodeType.TASK_INIT, label),
            data=_TaskNodeImportedData.configure(label, task_class, config, connections),
            universe=self.universe,
        )
        self.add_task_nodes([task_node])
        return task_node

    def add_task_nodes(self, nodes: Iterable[TaskNode], parent: PipelineGraph | None = None) -> None:
        """Add one or more existing task nodes to the graph.

        Parameters
        ----------
        nodes : `~collections.abc.Iterable` [ `TaskNode` ]
            Iterable of task nodes to add.  If any tasks have resolved
            dimensions, they must have the same dimension universe as the rest
            of the graph.
        parent : `PipelineGraph`, optional
            If provided, another `PipelineGraph` from which these nodes were
            obtained.  Any dataset type nodes already present in ``parent``
            that are referenced by the given tasks will be used in this graph
            if they are not already present, preserving any dataset type
            resolutions present in the parent graph.  Adding nodes from a
            parent graph after the graph has its own nodes (e.g. from
            `add_task`) or nodes from a third graph may result in invalid
            dataset type resolutions.  It is safest to only use this argument
            when populating an empty graph for the first time.

        Raises
        ------
        PipelineDataCycleError
            Raised if the graph is cyclic after this addition.

        Notes
        -----
        Checks for dataset type consistency and multiple producers do not occur
        until `resolve` is called, since the resolution depends on both the
        state of the data repository and all contributing tasks.

        Adding new tasks removes any existing resolutions of all dataset types
        it references (unless ``parent is not None`` and marks the graph as
        unsorted.  It is most efficient to add all tasks up front and only then
        resolve and/or sort the graph.
        """
        node_data: list[tuple[NodeKey, dict[str, Any]]] = []
        edge_data: list[tuple[NodeKey, NodeKey, str, dict[str, Any]]] = []
        for task_node in nodes:
            task_node = task_node._resolved(self._universe)
            node_data.append(
                (task_node.key, {"instance": task_node, "bipartite": task_node.key.node_type.bipartite})
            )
            node_data.append(
                (
                    task_node.init.key,
                    {"instance": task_node.init, "bipartite": task_node.init.key.node_type.bipartite},
                )
            )
            # Convert the edge objects attached to the task node to networkx.
            for read_edge in task_node.init.iter_all_inputs():
                self._append_graph_data_from_edge(node_data, edge_data, read_edge, parent=parent)
            for write_edge in task_node.init.iter_all_outputs():
                self._append_graph_data_from_edge(node_data, edge_data, write_edge, parent=parent)
            for read_edge in task_node.iter_all_inputs():
                self._append_graph_data_from_edge(node_data, edge_data, read_edge, parent=parent)
            for write_edge in task_node.iter_all_outputs():
                self._append_graph_data_from_edge(node_data, edge_data, write_edge, parent=parent)
            # Add a special edge (with no Edge instance) that connects the
            # TaskInitNode to the runtime TaskNode.
            edge_data.append((task_node.init.key, task_node.key, Edge.INIT_TO_TASK_NAME, {"instance": None}))
        if not node_data and not edge_data:
            return
        # Checks and preparation complete; time to start the actual
        # modification, during which it's hard to provide strong exception
        # safety.  Start by resetting the sort ordering, if there is one.
        self._reset()
        try:
            self._xgraph.add_nodes_from(node_data)
            self._xgraph.add_edges_from(edge_data)
            if not networkx.algorithms.dag.is_directed_acyclic_graph(self._xgraph):
                cycle = networkx.find_cycle(self._xgraph)
                raise PipelineDataCycleError(f"Cycle detected while adding tasks: {cycle}.")
        except Exception:
            # First try to roll back our changes.
            try:
                self._xgraph.remove_edges_from(edge_data)
                self._xgraph.remove_nodes_from(key for key, _ in node_data)
            except Exception as err:  # pragma: no cover
                # There's no known way to get here, but we want to make it
                # clear it's a big problem if we do.
                raise PipelineGraphExceptionSafetyError(
                    "Error while attempting to revert PipelineGraph modification has left the graph in "
                    "an inconsistent state."
                ) from err
            # Successfully rolled back; raise the original exception.
            raise

    def reconfigure_tasks(
        self,
        *args: tuple[str, PipelineTaskConfig],
        check_edges_unchanged: bool = False,
        assume_edges_unchanged: bool = False,
        **kwargs: PipelineTaskConfig,
    ) -> None:
        """Update the configuration for one or more tasks.

        Parameters
        ----------
        *args : `tuple` [ `str`, `.PipelineTaskConfig` ]
            Positional arguments are each a 2-tuple of task label and new
            config object.  Note that the same arguments may also be passed as
            ``**kwargs``, which is usually more readable, but task labels in
            ``*args`` are not required to be valid Python identifiers.
        check_edges_unchanged : `bool`, optional
            If `True`, require the edges (connections) of the modified tasks to
            remain unchanged after the configuration updates, and verify that
            this is the case.
        assume_edges_unchanged : `bool`, optional
            If `True`, the caller declares that the edges (connections) of the
            modified tasks will remain unchanged after the configuration
            updates, and that it is unnecessary to check this.
        **kwargs : `.PipelineTaskConfig`
            New config objects or overrides to apply to copies of the current
            config objects, with task labels as the keywords.

        Raises
        ------
        ValueError
            Raised if ``assume_edges_unchanged`` and ``check_edges_unchanged``
            are both `True`, or if the same task appears twice.
        EdgesChangedError
            Raised if ``check_edges_unchanged=True`` and the edges of a task do
            change.

        Notes
        -----
        If reconfiguring a task causes its edges to change, any dataset type
        nodes connected to that task (not just those whose edges have changed!)
        will be unresolved.
        """
        new_configs: dict[str, PipelineTaskConfig] = {}
        for task_label, config_update in itertools.chain(args, kwargs.items()):
            if new_configs.setdefault(task_label, config_update) is not config_update:
                raise ValueError(f"Config for {task_label!r} provided more than once.")
        updates = {
            task_label: self.tasks[task_label]._reconfigured(config, rebuild=not assume_edges_unchanged)
            for task_label, config in new_configs.items()
        }
        self._replace_task_nodes(
            updates,
            check_edges_unchanged=check_edges_unchanged,
            assume_edges_unchanged=assume_edges_unchanged,
            message_header=(
                "Unexpected change in edges for task {task_label!r} from original config (A) to "
                "new configs (B):"
            ),
        )

    def remove_tasks(
        self, labels: Iterable[str], drop_from_subsets: bool = True
    ) -> list[tuple[TaskNode, set[str]]]:
        """Remove one or more tasks from the graph.

        Parameters
        ----------
        labels : `~collections.abc.Iterable` [ `str` ]
            Iterable of the labels of the tasks to remove.
        drop_from_subsets : `bool`, optional
            If `True`, drop each removed task from any subset in which it
            currently appears.  If `False`, raise `PipelineGraphError` if any
            such subsets exist.

        Returns
        -------
        nodes_and_subsets : `list` [ `tuple` [ `TaskNode`, `set` [ `str` ] ] ]
            List of nodes removed and the labels of task subsets that
            referenced them.

        Raises
        ------
        PipelineGraphError
            Raised if ``drop_from_subsets`` is `False` and the task is still
            part of one or more subsets.

        Notes
        -----
        Removing a task will cause dataset nodes with no other referencing
        tasks to be removed.  Any other dataset type nodes referenced by a
        removed task will be reset to an "unresolved" state.
        """
        task_nodes_and_subsets = []
        dataset_types: set[NodeKey] = set()
        nodes_to_remove = set()
        for label in labels:
            task_node: TaskNode = self._xgraph.nodes[NodeKey(NodeType.TASK, label)]["instance"]
            # Find task subsets that reference this task.
            referencing_subsets = {
                subset_label
                for subset_label, task_subset in self.task_subsets.items()
                if label in task_subset
            }
            if not drop_from_subsets and referencing_subsets:
                raise PipelineGraphError(
                    f"Task {label!r} is still referenced by subset(s) {referencing_subsets}."
                )
            task_nodes_and_subsets.append((task_node, referencing_subsets))
            # Find dataset types referenced by this task.
            dataset_types.update(self._xgraph.predecessors(task_node.key))
            dataset_types.update(self._xgraph.successors(task_node.key))
            dataset_types.update(self._xgraph.predecessors(task_node.init.key))
            dataset_types.update(self._xgraph.successors(task_node.init.key))
            # Since there's an edge between the task and its init node, we'll
            # have added those two nodes here, too, and we don't want that.
            dataset_types.remove(task_node.init.key)
            dataset_types.remove(task_node.key)
            # Mark the task node and its init node for removal from the graph.
            nodes_to_remove.add(task_node.key)
            nodes_to_remove.add(task_node.init.key)
        # Process the referenced datasets to see which ones are orphaned and
        # need to be removed vs. just unresolved.
        nodes_to_unresolve = []
        for dataset_type_key in dataset_types:
            related_tasks = set()
            related_tasks.update(self._xgraph.predecessors(dataset_type_key))
            related_tasks.update(self._xgraph.successors(dataset_type_key))
            related_tasks.difference_update(nodes_to_remove)
            if not related_tasks:
                nodes_to_remove.add(dataset_type_key)
            else:
                nodes_to_unresolve.append(dataset_type_key)
        # Checks and preparation complete; time to start the actual
        # modification, during which it's hard to provide strong exception
        # safety.  Start by resetting the sort ordering.
        self._reset()
        try:
            for dataset_type_key in nodes_to_unresolve:
                self._xgraph.nodes[dataset_type_key]["instance"] = None
            for task_node, referencing_subsets in task_nodes_and_subsets:
                for subset_label in referencing_subsets:
                    self._task_subsets[subset_label].remove(task_node.label)
            self._xgraph.remove_nodes_from(nodes_to_remove)
        except Exception as err:  # pragma: no cover
            # There's no known way to get here, but we want to make it
            # clear it's a big problem if we do.
            raise PipelineGraphExceptionSafetyError(
                "Error during task removal has left the graph in an inconsistent state."
            ) from err
        return task_nodes_and_subsets

    def add_task_subset(self, subset_label: str, task_labels: Iterable[str], description: str = "") -> None:
        """Add a label for a set of tasks that are already in the pipeline.

        Parameters
        ----------
        subset_label : `str`
            Label for this set of tasks.
        task_labels : `~collections.abc.Iterable` [ `str` ]
            Labels of the tasks to include in the set.  All must already be
            included in the graph.
        description : `str`, optional
            String description to associate with this label.
        """
        subset = TaskSubset(self._xgraph, subset_label, set(task_labels), description)
        self._task_subsets[subset_label] = subset

    def remove_task_subset(self, subset_label: str) -> None:
        """Remove a labeled set of tasks.

        Parameters
        ----------
        subset_label : `str`
            Label for this set of tasks.
        """
        del self._task_subsets[subset_label]

    ###########################################################################
    #
    # NetworkX Export Interface:
    #
    # - methods to export the PipelineGraph's content (or various subsets
    #   thereof) as NetworkX objects.
    #
    # These are particularly useful when writing tools to visualize the graph,
    # while providing options for which aspects of the graph (tasks, dataset
    # types, or both) to include, since all exported graphs have similar
    # attributes regardless of their structure.
    #
    ###########################################################################

    def make_xgraph(self) -> networkx.MultiDiGraph:
        """Export a networkx representation of the full pipeline graph,
        including both init and runtime edges.

        Returns
        -------
        xgraph : `networkx.MultiDiGraph`
            Directed acyclic graph with parallel edges.

        Notes
        -----
        The returned graph uses `NodeKey` instances for nodes.  Parallel edges
        represent the same dataset type appearing in multiple connections for
        the same task, and are hence rare.  The connection name is used as the
        edge key to disambiguate those parallel edges.

        Almost all edges connect dataset type nodes to task or task init nodes
        or vice versa, but there is also a special edge that connects each task
        init node to its runtime node.  The existence of these edges makes the
        graph not quite bipartite, though its init-only and runtime-only
        subgraphs are bipartite.

        See `TaskNode`, `TaskInitNode`, `DatasetTypeNode`, `ReadEdge`, and
        `WriteEdge` for the descriptive node and edge attributes added.
        """
        return self._transform_xgraph_state(self._xgraph.copy(), skip_edges=False)

    def make_bipartite_xgraph(self, init: bool = False) -> networkx.MultiDiGraph:
        """Return a bipartite networkx representation of just the runtime or
        init-time pipeline graph.

        Parameters
        ----------
        init : `bool`, optional
            If `True` (`False` is default) return the graph of task
            initialization nodes and init input/output dataset types, instead
            of the graph of runtime task nodes and regular
            input/output/prerequisite dataset types.

        Returns
        -------
        xgraph : `networkx.MultiDiGraph`
            Directed acyclic graph with parallel edges.

        Notes
        -----
        The returned graph uses `NodeKey` instances for nodes.  Parallel edges
        represent the same dataset type appearing in multiple connections for
        the same task, and are hence rare.  The connection name is used as the
        edge key to disambiguate those parallel edges.

        This graph is bipartite because each dataset type node only has edges
        that connect it to a task [init] node, and vice versa.

        See `TaskNode`, `TaskInitNode`, `DatasetTypeNode`, `ReadEdge`, and
        `WriteEdge` for the descriptive node and edge attributes added.
        """
        return self._transform_xgraph_state(
            self._make_bipartite_xgraph_internal(init).copy(), skip_edges=False
        )

    def make_task_xgraph(self, init: bool = False) -> networkx.DiGraph:
        """Return a networkx representation of just the tasks in the pipeline.

        Parameters
        ----------
        init : `bool`, optional
            If `True` (`False` is default) return the graph of task
            initialization nodes, instead of the graph of runtime task nodes.

        Returns
        -------
        xgraph : `networkx.DiGraph`
            Directed acyclic graph with no parallel edges.

        Notes
        -----
        The returned graph uses `NodeKey` instances for nodes.  The dataset
        types that link these tasks are not represented at all; edges have no
        attributes, and there are no parallel edges.

        See `TaskNode` and `TaskInitNode` for the descriptive node and
        attributes added.
        """
        bipartite_xgraph = self._make_bipartite_xgraph_internal(init)
        task_keys = [
            key
            for key, bipartite in bipartite_xgraph.nodes(data="bipartite")
            if bipartite == NodeType.TASK.bipartite
        ]
        return self._transform_xgraph_state(
            networkx.algorithms.bipartite.projected_graph(networkx.DiGraph(bipartite_xgraph), task_keys),
            skip_edges=True,
        )

    def make_dataset_type_xgraph(self, init: bool = False) -> networkx.DiGraph:
        """Return a networkx representation of just the dataset types in the
        pipeline.

        Parameters
        ----------
        init : `bool`, optional
            If `True` (`False` is default) return the graph of init input and
            output dataset types, instead of the graph of runtime (input,
            output, prerequisite input) dataset types.

        Returns
        -------
        xgraph : `networkx.DiGraph`
            Directed acyclic graph with no parallel edges.

        Notes
        -----
        The returned graph uses `NodeKey` instances for nodes.  The tasks that
        link these tasks are not represented at all; edges have no attributes,
        and there are no parallel edges.

        See `DatasetTypeNode` for the descriptive node and attributes added.
        """
        bipartite_xgraph = self._make_bipartite_xgraph_internal(init)
        dataset_type_keys = [
            key
            for key, bipartite in bipartite_xgraph.nodes(data="bipartite")
            if bipartite == NodeType.DATASET_TYPE.bipartite
        ]
        return self._transform_xgraph_state(
            networkx.algorithms.bipartite.projected_graph(
                networkx.DiGraph(bipartite_xgraph), dataset_type_keys
            ),
            skip_edges=True,
        )

    ###########################################################################
    #
    # Serialization Interface.
    #
    # Serialization of PipelineGraphs is currently experimental and may not be
    # retained in the future.  All serialization methods are
    # underscore-prefixed to ensure nobody mistakes them for a stable interface
    # (let a lone a stable file format).
    #
    ###########################################################################

    @classmethod
    def _read_stream(
        cls, stream: BinaryIO, import_mode: TaskImportMode = TaskImportMode.REQUIRE_CONSISTENT_EDGES
    ) -> PipelineGraph:
        """Read a serialized `PipelineGraph` from a file-like object.

        Parameters
        ----------
        stream : `BinaryIO`
            File-like object opened for binary reading, containing
            gzip-compressed JSON.
        import_mode : `TaskImportMode`, optional
            Whether to import tasks, and how to reconcile any differences
            between the imported task's connections and the those that were
            persisted with the graph.  Default is to check that they are the
            same.

        Returns
        -------
        graph : `PipelineGraph`
            Deserialized pipeline graph.

        Raises
        ------
        PipelineGraphReadError
            Raised if the serialized `PipelineGraph` is not self-consistent.
        EdgesChangedError
            Raised if ``import_mode`` is
            `TaskImportMode.REQUIRED_CONSISTENT_EDGES` and the edges of a task
            did change after import and reconfiguration.

        Notes
        -----
        `PipelineGraph` serialization is currently experimental and may be
        removed or significantly changed in the future, with no deprecation
        period.
        """
        from .io import SerializedPipelineGraph

        with gzip.open(stream, "rb") as uncompressed_stream:
            data = json.load(uncompressed_stream)
            serialized_graph = SerializedPipelineGraph.model_validate(data)
            return serialized_graph.deserialize(import_mode)

    @classmethod
    def _read_uri(
        cls,
        uri: ResourcePathExpression,
        import_mode: TaskImportMode = TaskImportMode.REQUIRE_CONSISTENT_EDGES,
    ) -> PipelineGraph:
        """Read a serialized `PipelineGraph` from a file at a URI.

        Parameters
        ----------
        uri : convertible to `lsst.resources.ResourcePath`
            URI to a gzip-compressed JSON file containing a serialized pipeline
            graph.
        import_mode : `TaskImportMode`, optional
            Whether to import tasks, and how to reconcile any differences
            between the imported task's connections and the those that were
            persisted with the graph.  Default is to check that they are the
            same.

        Returns
        -------
        graph : `PipelineGraph`
            Deserialized pipeline graph.

        Raises
        ------
        PipelineGraphReadError
            Raised if the serialized `PipelineGraph` is not self-consistent.
        EdgesChangedError
            Raised if ``import_mode`` is
            `TaskImportMode.REQUIRED_CONSISTENT_EDGES` and the edges of a task
            did change after import and reconfiguration.

        Notes
        -----
        `PipelineGraph` serialization is currently experimental and may be
        removed or significantly changed in the future, with no deprecation
        period.
        """
        uri = ResourcePath(uri)
        with uri.open("rb") as stream:
            return cls._read_stream(cast(BinaryIO, stream), import_mode=import_mode)

    def _write_stream(self, stream: BinaryIO) -> None:
        """Write the pipeline to a file-like object.

        Parameters
        ----------
        stream
            File-like object opened for binary writing.

        Notes
        -----
        `PipelineGraph` serialization is currently experimental and may be
        removed or significantly changed in the future, with no deprecation
        period.

        The file format is gzipped JSON, and is intended to be human-readable,
        but it should not be considered a stable public interface for outside
        code, which should always use `PipelineGraph` methods (or at least the
        `io.SerializedPipelineGraph` class) to read these files.
        """
        from .io import SerializedPipelineGraph

        with gzip.open(stream, mode="wb") as compressed_stream:
            compressed_stream.write(
                SerializedPipelineGraph.serialize(self).model_dump_json(exclude_defaults=True).encode("utf-8")
            )

    def _write_uri(self, uri: ResourcePathExpression) -> None:
        """Write the pipeline to a file given a URI.

        Parameters
        ----------
        uri : convertible to `lsst.resources.ResourcePath`
            URI to write to .  May have ``.json.gz`` or no extension (which
            will cause a ``.json.gz`` extension to be added).

        Notes
        -----
        `PipelineGraph` serialization is currently experimental and may be
        removed or significantly changed in the future, with no deprecation
        period.

        The file format is gzipped JSON, and is intended to be human-readable,
        but it should not be considered a stable public interface for outside
        code, which should always use `PipelineGraph` methods (or at least the
        `io.SerializedPipelineGraph` class) to read these files.
        """
        uri = ResourcePath(uri)
        extension = uri.getExtension()
        if not extension:
            uri = uri.updatedExtension(".json.gz")
        elif extension != ".json.gz":
            raise ValueError("Expanded pipeline files should always have a .json.gz extension.")
        with uri.open(mode="wb") as stream:
            self._write_stream(cast(BinaryIO, stream))

    def _import_and_configure(
        self, import_mode: TaskImportMode = TaskImportMode.REQUIRE_CONSISTENT_EDGES
    ) -> None:
        """Import the `PipelineTask` classes referenced by all task nodes and
        update those nodes accordingly.

        Parameters
        ----------
        import_mode : `TaskImportMode`, optional
            Whether to import tasks, and how to reconcile any differences
            between the imported task's connections and the those that were
            persisted with the graph.  Default is to check that they are the
            same.  This method does nothing if this is
            `TaskImportMode.DO_NOT_IMPORT`.

        Raises
        ------
        EdgesChangedError
            Raised if ``import_mode`` is
            `TaskImportMode.REQUIRED_CONSISTENT_EDGES` and the edges of a task
            did change after import and reconfiguration.

        Notes
        -----
        This method shouldn't need to be called unless the graph was
        deserialized without importing and configuring immediately, which is
        not the default behavior (but it can greatly speed up deserialization).
        If all tasks have already been imported this does nothing.

        Importing and configuring a task can change its
        `~TaskNode.task_class_name` or `~TaskClass.get_config_str` output,
        usually because the software used to read a serialized graph is newer
        than the software used to write it (e.g. a new config option has been
        added, or the task was moved to a new module with a forwarding alias
        left behind).  These changes are allowed by
        `TaskImportMode.REQUIRE_CONSISTENT_EDGES`.

        If importing and configuring a task causes its edges to change, any
        dataset type nodes linked to those edges will be reset to the
        unresolved state.
        """
        if import_mode is TaskImportMode.DO_NOT_IMPORT:
            return
        rebuild = (
            import_mode is TaskImportMode.REQUIRE_CONSISTENT_EDGES
            or import_mode is TaskImportMode.OVERRIDE_EDGES
        )
        updates: dict[str, TaskNode] = {}
        node_key: NodeKey
        for node_key, node_state in self._xgraph.nodes.items():
            if node_key.node_type is NodeType.TASK:
                task_node: TaskNode = node_state["instance"]
                new_task_node = task_node._imported_and_configured(rebuild)
                if new_task_node is not task_node:
                    updates[task_node.label] = new_task_node
        self._replace_task_nodes(
            updates,
            check_edges_unchanged=(import_mode is TaskImportMode.REQUIRE_CONSISTENT_EDGES),
            assume_edges_unchanged=(import_mode is TaskImportMode.ASSUME_CONSISTENT_EDGES),
            message_header=(
                "In task with label {task_label!r}, persisted edges (A)"
                "differ from imported and configured edges (B):"
            ),
        )

    ###########################################################################
    #
    # Advanced PipelineGraph Inspection Interface:
    #
    # - methods to iterate over all nodes and edges, utilizing NodeKeys;
    #
    # - methods to find overall inputs and group nodes by their dimensions,
    #   which are important operations for QuantumGraph generation.
    #
    ###########################################################################

    def iter_edges(self, init: bool = False) -> Iterator[Edge]:
        """Iterate over edges in the graph.

        Parameters
        ----------
        init : `bool`, optional
            If `True` (`False` is default) iterate over the edges between task
            initialization node and init input/output dataset types, instead of
            the runtime task nodes and regular input/output/prerequisite
            dataset types.

        Returns
        -------
        edges : `~collections.abc.Iterator` [ `Edge` ]
            A lazy iterator over `Edge` (`WriteEdge` or `ReadEdge`) instances.

        Notes
        -----
        This method always returns _either_ init edges or runtime edges, never
        both.  The full (internal) graph that contains both also includes a
        special edge that connects each task init node to its runtime node;
        that is also never returned by this method, since it is never a part of
        the init-only or runtime-only subgraphs.
        """
        edge: Edge
        for _, _, edge in self._xgraph.edges(data="instance"):
            if edge is not None and edge.is_init == init:
                yield edge

    def iter_nodes(
        self,
    ) -> Iterator[
        tuple[Literal[NodeType.TASK_INIT], str, TaskInitNode]
        | tuple[Literal[NodeType.TASK], str, TaskInitNode]
        | tuple[Literal[NodeType.DATASET_TYPE], str, DatasetTypeNode | None]
    ]:
        """Iterate over nodes in the graph.

        Returns
        -------
        nodes : `~collections.abc.Iterator` [ `tuple` ]
            A lazy iterator over all of the nodes in the graph.  Each yielded
            element is a tuple of:

            - the node type enum value (`NodeType`);
            - the string name for the node (task label or parent dataset type
              name);
            - the node value (`TaskNode`, `TaskInitNode`, `DatasetTypeNode`,
              or `None` for dataset type nodes that have not been resolved).
        """
        key: NodeKey
        if self._sorted_keys is not None:
            for key in self._sorted_keys:
                yield key.node_type, key.name, self._xgraph.nodes[key]["instance"]  # type: ignore
        else:
            for key, node in self._xgraph.nodes(data="instance"):
                yield key.node_type, key.name, node  # type: ignore

    def iter_overall_inputs(self) -> Iterator[tuple[str, DatasetTypeNode | None]]:
        """Iterate over all of the dataset types that are consumed but not
        produced by the graph.

        Returns
        -------
        dataset_types : `~collections.abc.Iterator` [ `tuple` ]
            A lazy iterator over the overall-input dataset types (including
            overall init inputs and prerequisites).  Each yielded element is a
            tuple of:

            - the parent dataset type name;
            - the resolved `DatasetTypeNode`, or `None` if the dataset type has
            - not been resolved.
        """
        for generation in networkx.algorithms.dag.topological_generations(self._xgraph):
            key: NodeKey
            for key in generation:
                # While we expect all tasks to have at least one input and
                # hence never appear in the first topological generation, that
                # is not true of task init nodes.
                if key.node_type is NodeType.DATASET_TYPE:
                    yield key.name, self._xgraph.nodes[key]["instance"]
            return

    def group_by_dimensions(
        self, prerequisites: bool = False
    ) -> dict[DimensionGroup, tuple[dict[str, TaskNode], dict[str, DatasetTypeNode]]]:
        """Group this graph's tasks and dataset types by their dimensions.

        Parameters
        ----------
        prerequisites : `bool`, optional
            If `True`, include prerequisite dataset types as well as regular
            input and output datasets (including intermediates).

        Returns
        -------
        groups : `dict` [ `DimensionGroup`, `tuple` ]
            A dictionary of groups keyed by `DimensionGroup`, in which each
            value is a tuple of:

            - a `dict` of `TaskNode` instances, keyed by task label
            - a `dict` of `DatasetTypeNode` instances, keyed by
              dataset type name.

            that have those dimensions.

        Notes
        -----
        Init inputs and outputs are always included, but always have empty
        dimensions and are hence are all grouped together.
        """
        result: dict[DimensionGroup, tuple[dict[str, TaskNode], dict[str, DatasetTypeNode]]] = {}
        next_new_value: tuple[dict[str, TaskNode], dict[str, DatasetTypeNode]] = ({}, {})
        for task_label, task_node in self.tasks.items():
            if task_node.dimensions is None:
                raise UnresolvedGraphError(f"Task with label {task_label!r} has not been resolved.")
            if (group := result.setdefault(task_node.dimensions, next_new_value)) is next_new_value:
                next_new_value = ({}, {})  # make new lists for next time
            group[0][task_node.label] = task_node
        for dataset_type_name, dataset_type_node in self.dataset_types.items():
            if dataset_type_node is None:
                raise UnresolvedGraphError(f"Dataset type {dataset_type_name!r} has not been resolved.")
            if not dataset_type_node.is_prerequisite or prerequisites:
                if (
                    group := result.setdefault(dataset_type_node.dataset_type.dimensions, next_new_value)
                ) is next_new_value:
                    next_new_value = ({}, {})  # make new lists for next time
                group[1][dataset_type_node.name] = dataset_type_node
        return result

    def split_independent(self) -> Iterable[PipelineGraph]:
        """Iterate over independent subgraphs that together comprise this
        pipeline graph.

        Returns
        -------
        subgraphs : `Iterable` [ `PipelineGraph` ]
            An iterable over component subgraphs that could be run
            independently (they have only overall inputs in common).  May be a
            lazy iterator.

        Notes
        -----
        All resolved dataset type nodes will be preserved.

        If there is only one component, ``self`` may be returned as the only
        element in the iterable.

        If `has_been_sorted`, all subgraphs will be sorted as well.
        """
        # Having an overall input in common isn't enough to make subgraphs
        # dependent on each other, so we want to look for connected component
        # subgraphs of the task-only projected graph.
        bipartite_xgraph = self._make_bipartite_xgraph_internal(init=False)
        task_keys = {
            key
            for key, bipartite in bipartite_xgraph.nodes(data="bipartite")
            if bipartite == NodeType.TASK.bipartite
        }
        task_xgraph = networkx.algorithms.bipartite.projected_graph(
            networkx.DiGraph(bipartite_xgraph), task_keys
        )
        # "Weakly" connected means connected in only one direction, which is
        # the only kind of "connected" a DAG can ever be.
        for component_task_keys in networkx.algorithms.weakly_connected_components(task_xgraph):
            if component_task_keys == task_keys:
                yield self
                return
            else:
                component_subgraph = PipelineGraph(universe=self._universe)
                component_subgraph.add_task_nodes(
                    [self._xgraph.nodes[key]["instance"] for key in component_task_keys], parent=self
                )
                if self.has_been_sorted:
                    component_subgraph.sort()
                yield component_subgraph

    ###########################################################################
    #
    # Data repository/collection initialization
    #
    ###########################################################################

    @property
    def packages_dataset_type(self) -> DatasetType:
        """The special "packages" dataset type that records software versions.

        This is not associated with a task and hence is
        not considered part of the pipeline graph in other respects, but it
        does get written with other provenance datasets.
        """
        if self.universe is None:
            raise UnresolvedGraphError(
                "PipelineGraph must be resolved in order to get the packages dataset type."
            )
        return DatasetType(PACKAGES_INIT_OUTPUT_NAME, self.universe.empty, PACKAGES_INIT_OUTPUT_STORAGE_CLASS)

    def register_dataset_types(self, butler: Butler, include_packages: bool = True) -> None:
        """Register all dataset types in a data repository.

        Parameters
        ----------
        butler : `~lsst.daf.butler.Butler`
            Data repository client.
        include_packages : `bool`, optional
            Whether to include the special "packages" dataset type that records
            software versions (this is not associated with a task and hence is
            not considered part of the pipeline graph in other respects, but it
            does get written with other provenance datasets).
        """
        dataset_types = [node.dataset_type for node in self.dataset_types.values()]
        if include_packages:
            dataset_types.append(self.packages_dataset_type)
        for dataset_type in dataset_types:
            butler.registry.registerDatasetType(dataset_type)

    def check_dataset_type_registrations(self, butler: Butler, include_packages: bool = True) -> None:
        """Check that dataset type registrations in a data repository match
        the definitions in this pipeline graph.

        Parameters
        ----------
        butler : `~lsst.daf.butler.Butler`
            Data repository client.
        include_packages : `bool`, optional
            Whether to include the special "packages" dataset type that records
            software versions (this is not associated with a task and hence is
            not considered part of the pipeline graph in other respects, but it
            does get written with other provenance datasets).

        Raises
        ------
        lsst.daf.butler.MissingDatasetTypeError
            Raised if one or more non-optional-input or output dataset types in
            the pipeline is not registered at all.
        lsst.daf.butler.ConflictingDefinitionError
            Raised if the definition in the data repository is not identical
            to the definition in the pipeline graph.

        Notes
        -----
        Note that dataset type definitions that are storage-class-conversion
        compatible but not identical are not permitted by these checks, because
        the expectation is that these differences are handled by `resolve`,
        which makes the pipeline graph use the data repository definitions.
        This method is intended to check that none of those definitions have
        changed.
        """
        dataset_types = [node.dataset_type for node in self.dataset_types.values()]
        if include_packages:
            dataset_types.append(self.packages_dataset_type)
        missing_dataset_types: list[str] = []
        for dataset_type in dataset_types:
            try:
                expected = butler.registry.getDatasetType(dataset_type.name)
            except MissingDatasetTypeError:
                expected = None
            if expected is None:
                # The user probably forgot to register dataset types
                # at least once (which should be an error),
                # but we could also get here if this is an optional input for
                # which no datasets were found in this repo (not an error).
                if (
                    not (
                        self.producer_of(dataset_type.name) is None
                        and all(
                            self.tasks[input_edge.task_label].is_optional(input_edge.connection_name)
                            for input_edge in self.consuming_edges_of(dataset_type.name)
                        )
                    )
                    or dataset_type.name == PACKAGES_INIT_OUTPUT_NAME
                ):
                    missing_dataset_types.append(dataset_type.name)
            elif expected != dataset_type:
                raise ConflictingDefinitionError(
                    f"DatasetType definition in registry has changed since the pipeline graph was resolved: "
                    f"{dataset_type} (graph) != {expected} (registry)."
                )
        if missing_dataset_types:
            plural = "s" if len(missing_dataset_types) != 1 else ""
            raise MissingDatasetTypeError(
                f"Missing dataset type definition{plural}: {', '.join(missing_dataset_types)}. "
                "Dataset types have to be registered in advance (on the command-line, either via "
                "`butler register-dataset-type` or the `--register-dataset-types` option to `pipetask run`."
            )

    def instantiate_tasks(
        self,
        get_init_input: Callable[[DatasetType], Any] | None = None,
        init_outputs: list[tuple[Any, DatasetType]] | None = None,
    ) -> list[PipelineTask]:
        """Instantiate all tasks in the pipeline.

        Parameters
        ----------
        get_init_input : `~collections.abc.Callable`, optional
            Callable that accepts a single `~lsst.daf.butler.DatasetType`
            parameter and returns the init-input dataset associated with that
            dataset type.  Must respect the storage class embedded in the type.
            This is optional if the pipeline does not have any overall init
            inputs.  When a full butler is available,
            `lsst.daf.butler.Butler.get` can be used directly here.
        init_outputs : `list`, optional
            A list of ``(obj, dataset type)`` init-output dataset pairs, to be
            appended to in-place.  Both the object and the dataset type will
            correspond to the storage class of the output connection, which
            may not be the same as the storage class on the graph's dataset
            type node.

        Returns
        -------
        tasks : `list`
            Constructed `PipelineTask` instances.
        """
        if not self.is_fully_resolved:
            raise UnresolvedGraphError("Pipeline graph must be fully resolved before instantiating tasks.")
        empty_data_id = DataCoordinate.make_empty(cast(DimensionUniverse, self.universe))
        handles: dict[str, InMemoryDatasetHandle] = {}
        tasks: list[PipelineTask] = []
        for task_node in self.tasks.values():
            task_init_inputs: dict[str, Any] = {}
            for read_edge in task_node.init.inputs.values():
                if (handle := handles.get(read_edge.dataset_type_name)) is not None:
                    obj = handle.get(storageClass=read_edge.storage_class_name)
                elif (
                    read_edge.component is not None
                    and (parent_handle := handles.get(read_edge.parent_dataset_type_name)) is not None
                ):
                    obj = parent_handle.get(
                        storageClass=read_edge.storage_class_name, component=read_edge.component
                    )
                else:
                    dataset_type_node = self.dataset_types[read_edge.parent_dataset_type_name]
                    if get_init_input is None:
                        raise ValueError(
                            f"Task {task_node.label!r} requires init-input "
                            f"{read_edge.dataset_type_name} but no 'get_init_input' callback was provided."
                        )
                    obj = get_init_input(read_edge.adapt_dataset_type(dataset_type_node.dataset_type))
                    n_consumers = len(self.consumers_of(dataset_type_node.name))
                    if (
                        n_consumers > 1
                        and read_edge.component is None
                        and read_edge.storage_class_name == dataset_type_node.storage_class_name
                    ):
                        # Caching what we just got is safe in general only
                        # if there was no storage class conversion, since
                        # a->b and a->c does not imply b->c.
                        handles[read_edge.dataset_type_name] = InMemoryDatasetHandle(
                            obj,
                            storageClass=dataset_type_node.storage_class,
                            dataId=empty_data_id,
                            copy=True,
                        )
                task_init_inputs[read_edge.connection_name] = obj
            task = task_node.task_class(
                config=task_node.config, initInputs=task_init_inputs, name=task_node.label
            )
            tasks.append(task)
            for write_edge in task_node.init.outputs.values():
                dataset_type_node = self.dataset_types[write_edge.parent_dataset_type_name]
                obj = getattr(task, write_edge.connection_name)
                # We don't immediately coerce obj to the dataset_type_node
                # storage class (which should be the repo storage class, if
                # there is one) when appending to `init_outputs` because a
                # formatter might be able to do a better job of that later;
                # instead we pair it with a dataset type that's consistent with
                # the in-memory type. We do coerce when populating `handles`,
                # though, because going through the dataset_type_node storage
                # class is the conversion path we checked when we resolved the
                # pipeline graph.
                if init_outputs is not None:
                    init_outputs.append((obj, write_edge.adapt_dataset_type(dataset_type_node.dataset_type)))
                n_consumers = len(self.consumers_of(dataset_type_node.name))
                if n_consumers > 0:
                    handles[dataset_type_node.name] = InMemoryDatasetHandle(
                        dataset_type_node.storage_class.coerce_type(obj),
                        dataId=empty_data_id,
                        storageClass=dataset_type_node.storage_class,
                        copy=(n_consumers > 1),
                    )
        return tasks

    def write_init_outputs(self, butler: Butler) -> None:
        """Write the init-output datasets for all tasks in the pipeline graph.

        Parameters
        ----------
        butler : `lsst.daf.butler.Butler`
            A full butler data repository client with its default run set
            to the collection where datasets should be written.

        Notes
        -----
        Datasets that already exist in the butler's output run collection will
        not be written.

        This method writes outputs with new random dataset IDs and should
        hence only be used when writing init-outputs prior to building a
        `QuantumGraph`.  Use `QuantumGraph.write_init_outputs` if a quantum
        graph has already been built.
        """
        init_outputs: list[tuple[Any, DatasetType]] = []
        self.instantiate_tasks(butler.get, init_outputs)
        found_refs: dict[str, DatasetRef] = {}
        to_put: list[tuple[Any, DatasetType]] = []
        for obj, dataset_type in init_outputs:
            if (ref := butler.find_dataset(dataset_type, collections=butler.run)) is not None:
                found_refs[dataset_type.name] = ref
            else:
                to_put.append((obj, dataset_type))
        for ref, stored in butler.stored_many(found_refs.values()).items():
            if not stored:
                raise FileNotFoundError(
                    f"Init-output dataset {ref.datasetType.name!r} was found in RUN {ref.run!r} "
                    f"but had not actually been stored (or was stored and later deleted)."
                )
        for obj, dataset_type in to_put:
            butler.put(obj, dataset_type)

    def write_configs(self, butler: Butler) -> None:
        """Write the config datasets for all tasks in the pipeline graph.

        Parameters
        ----------
        butler : `lsst.daf.butler.Butler`
            A full butler data repository client with its default run set
            to the collection where datasets should be written.

        Notes
        -----
        Config datasets that already exist in the butler's output run
        collection will be checked for consistency.

        This method writes outputs with new random dataset IDs and should
        hence only be used when writing init-outputs prior to building a
        `QuantumGraph`.  Use `QuantumGraph.write_configs` if a quantum graph
        has already been built.

        Raises
        ------
        lsst.daf.butler.registry.ConflictingDefinitionError
            Raised if a config dataset already exists and is not consistent
            with the config in the pipeline graph.
        """
        to_put: list[tuple[PipelineTaskConfig, str]] = []
        for task_node in self.tasks.values():
            dataset_type_name = task_node.init.config_output.dataset_type_name
            if (ref := butler.find_dataset(dataset_type_name, collections=butler.run)) is not None:
                old_config = butler.get(ref)
                if not task_node.config.compare(old_config, shortcut=False, output=log_config_mismatch):
                    raise ConflictingDefinitionError(
                        f"Config does not match existing task config {dataset_type_name!r} in "
                        "butler; tasks configurations must be consistent within the same run collection"
                    )
            else:
                to_put.append((task_node.config, dataset_type_name))
        # We do writes at the end to minimize the mess we leave behind when we
        # raise an exception.
        for config, dataset_type_name in to_put:
            butler.put(config, dataset_type_name)

    def write_packages(self, butler: Butler) -> None:
        """Write the 'packages' dataset for the currently-active software
        versions.

        Parameters
        ----------
        butler : `lsst.daf.butler.Butler`
            A full butler data repository client with its default run set
            to the collection where datasets should be written.

        Notes
        -----
        If the packages dataset already exists, it will be compared to the
        versions in the current packages.  New packages that weren't present
        before are not considered an inconsistency.

        This method writes outputs with new random dataset IDs and should
        hence only be used when writing init-outputs prior to building a
        `QuantumGraph`.  Use `QuantumGraph.write_packages` if a quantum graph
        has already been built.

        Raises
        ------
        lsst.daf.butler.registry.ConflictingDefinitionError
            Raised if the packages dataset already exists and is not consistent
            with the current packages.
        """
        new_packages = Packages.fromSystem()
        if (ref := butler.find_dataset(self.packages_dataset_type)) is not None:
            packages = butler.get(ref)
            if compare_packages(packages, new_packages):
                # have to remove existing dataset first; butler has no
                # replace option.
                butler.pruneDatasets([ref], unstore=True, purge=True)
                butler.put(packages, ref)
        else:
            butler.put(new_packages, self.packages_dataset_type)

    def init_output_run(self, butler: Butler) -> None:
        """Initialize a new output RUN collection by writing init-output
        datasets (including configs and packages).

        Parameters
        ----------
        butler : `lsst.daf.butler.Butler`
            A full butler data repository client with its default run set
            to the collection where datasets should be written.
        """
        self.write_configs(butler)
        self.write_packages(butler)
        self.write_init_outputs(butler)

    ###########################################################################
    #
    # Class- and Package-Private Methods.
    #
    ###########################################################################

    def _iter_task_defs(self) -> Iterator[TaskDef]:
        """Iterate over this pipeline as a sequence of `TaskDef` instances.

        Notes
        -----
        This is a package-private method intended to aid in the transition to a
        codebase more fully integrated with the `PipelineGraph` class, in which
        both `TaskDef` and `PipelineDatasetTypes` are expected to go away, and
        much of the functionality on the `Pipeline` class will be moved to
        `PipelineGraph` as well.

        Raises
        ------
        TaskNotImportedError
            Raised if `TaskNode.is_imported` is `False` for any task.
        """
        from ..pipeline import TaskDef

        for node in self._tasks.values():
            yield TaskDef(
                config=node.config,
                taskClass=node.task_class,
                label=node.label,
                connections=node.get_connections(),
            )

    def _init_from_args(
        self,
        xgraph: networkx.MultiDiGraph | None,
        sorted_keys: Sequence[NodeKey] | None,
        task_subsets: dict[str, TaskSubset] | None,
        description: str,
        universe: DimensionUniverse | None,
        data_id: DataId | None,
    ) -> None:
        """Initialize the graph with possibly-nontrivial arguments.

        Parameters
        ----------
        xgraph : `networkx.MultiDiGraph` or `None`
            The backing networkx graph, or `None` to create an empty one.
            This graph has `NodeKey` instances for nodes and the same structure
            as the graph exported by `make_xgraph`, but its nodes and edges
            have a single ``instance`` attribute that holds a `TaskNode`,
            `TaskInitNode`, `DatasetTypeNode` (or `None`), `ReadEdge`, or
            `WriteEdge` instance.
        sorted_keys : `Sequence` [ `NodeKey` ] or `None`
            Topologically sorted sequence of node keys, or `None` if the graph
            is not sorted.
        task_subsets : `dict` [ `str`, `TaskSubset` ]
            Labeled subsets of tasks.  Values must be constructed with
            ``xgraph`` as their parent graph.
        description : `str`
            String description for this pipeline.
        universe : `lsst.daf.butler.DimensionUniverse` or `None`
            Definitions of all dimensions.
        data_id : `lsst.daf.butler.DataCoordinate` or other data ID mapping.
            Data ID that represents a constraint on all quanta generated from
            this pipeline.

        Notes
        -----
        Only empty `PipelineGraph` instances should be constructed directly by
        users, which sets the signature of ``__init__`` itself, but methods on
        `PipelineGraph` and its helper classes need to be able to create them
        with state.  Those methods can call this after calling ``__new__``
        manually, skipping ``__init__``.
        """
        self._xgraph = xgraph if xgraph is not None else networkx.MultiDiGraph()
        self._sorted_keys: Sequence[NodeKey] | None = None
        self._task_subsets = task_subsets if task_subsets is not None else {}
        self._description = description
        self._tasks = TaskMappingView(self._xgraph)
        self._dataset_types = DatasetTypeMappingView(self._xgraph)
        self._raw_data_id: dict[str, Any]
        if isinstance(data_id, DataCoordinate):
            if universe is None:
                universe = data_id.universe
            else:
                assert universe is data_id.universe, "data_id.universe and given universe differ"
            self._raw_data_id = dict(data_id.required)
        elif data_id is None:
            self._raw_data_id = {}
        else:
            self._raw_data_id = dict(data_id)
        self._universe = universe
        if sorted_keys is not None:
            self._reorder(sorted_keys)

    def _make_bipartite_xgraph_internal(self, init: bool) -> networkx.MultiDiGraph:
        """Make a bipartite init-only or runtime-only internal subgraph.

        See `make_bipartite_xgraph` for parameters and return values.

        Notes
        -----
        This method returns a view of the `PipelineGraph` object's internal
        backing graph, and hence should only be called in methods that copy the
        result either explicitly or by running a copying algorithm before
        returning it to the user.
        """
        return self._xgraph.edge_subgraph([edge.key for edge in self.iter_edges(init)])

    def _transform_xgraph_state(self, xgraph: _G, skip_edges: bool) -> _G:
        """Transform networkx graph attributes in-place from the internal
        "instance" attributes to the documented exported attributes.

        Parameters
        ----------
        xgraph : `networkx.DiGraph` or `networkx.MultiDiGraph`
            Graph whose state should be transformed.
        skip_edges : `bool`
            If `True`, do not transform edge state.

        Returns
        -------
        xgraph : `networkx.DiGraph` or `networkx.MultiDiGraph`
            The same object passed in, after modification.

        Notes
        -----
        This should be called after making a copy of the internal graph but
        before any projection down to just task or dataset type nodes, since
        it assumes stateful edges.
        """
        state: dict[str, Any]
        for state in xgraph.nodes.values():
            node_value: TaskInitNode | TaskNode | DatasetTypeNode | None = state.pop("instance")
            if node_value is not None:
                state.update(node_value._to_xgraph_state())
            else:
                # This is a dataset type node that is not resolved.
                state["bipartite"] = NodeType.DATASET_TYPE.bipartite
        if not skip_edges:
            for _, _, state in xgraph.edges(data=True):
                edge: Edge | None = state.pop("instance", None)
                if edge is not None:
                    state.update(edge._to_xgraph_state())
        return xgraph

    def _replace_task_nodes(
        self,
        updates: Mapping[str, TaskNode],
        check_edges_unchanged: bool,
        assume_edges_unchanged: bool,
        message_header: str,
    ) -> None:
        """Replace task nodes and update edges and dataset type nodes
        accordingly.

        Parameters
        ----------
        updates : `Mapping` [ `str`, `TaskNode` ]
            New task nodes with task label keys.  All keys must be task labels
            that are already present in the graph.
        check_edges_unchanged : `bool`, optional
            If `True`, require the edges (connections) of the modified tasks to
            remain unchanged after importing and configuring each task, and
            verify that this is the case.
        assume_edges_unchanged : `bool`, optional
            If `True`, the caller declares that the edges (connections) of the
            modified tasks will remain unchanged importing and configuring each
            task, and that it is unnecessary to check this.
        message_header : `str`
            Template for `str.format` with a single ``task_label`` placeholder
            to use as the first line in `EdgesChangedError` messages that show
            the differences between new task edges and old task edges.  Should
            include the fact that the rest of the message will refer to the old
            task as "A" and the new task as "B", and end with a colon.

        Raises
        ------
        ValueError
            Raised if ``assume_edges_unchanged`` and ``check_edges_unchanged``
            are both `True`, or if a full config is provided for a task after
            another full config or an override has already been provided.
        EdgesChangedError
            Raised if ``check_edges_unchanged=True`` and the edges of a task do
            change.
        """
        deep: dict[str, TaskNode] = {}
        shallow: dict[str, TaskNode] = {}
        if assume_edges_unchanged:
            if check_edges_unchanged:
                raise ValueError("Cannot simultaneously assume and check that edges have not changed.")
            shallow.update(updates)
        else:
            for task_label, new_task_node in updates.items():
                old_task_node = self.tasks[task_label]
                messages = old_task_node.diff_edges(new_task_node)
                if messages:
                    if check_edges_unchanged:
                        messages.insert(0, message_header.format(task_label=task_label))
                        raise EdgesChangedError("\n".join(messages))
                    else:
                        deep[task_label] = new_task_node
                else:
                    shallow[task_label] = new_task_node
        try:
            if deep:
                removed = self.remove_tasks(deep.keys(), drop_from_subsets=True)
                self.add_task_nodes(deep.values())
                for replaced_task_node, referencing_subsets in removed:
                    for subset_label in referencing_subsets:
                        self._task_subsets[subset_label].add(replaced_task_node.label)
            for task_node in shallow.values():
                self._xgraph.nodes[task_node.key]["instance"] = task_node
                self._xgraph.nodes[task_node.init.key]["instance"] = task_node.init
        except PipelineGraphExceptionSafetyError:  # pragma: no cover
            raise
        except Exception as err:  # pragma: no cover
            # There's no known way to get here, but we want to make it clear
            # it's a big problem if we do.
            raise PipelineGraphExceptionSafetyError(
                "Error while replacing tasks has left the graph in an inconsistent state."
            ) from err

    def _append_graph_data_from_edge(
        self,
        node_data: list[tuple[NodeKey, dict[str, Any]]],
        edge_data: list[tuple[NodeKey, NodeKey, str, dict[str, Any]]],
        edge: Edge,
        parent: PipelineGraph | None,
    ) -> None:
        """Append networkx state dictionaries for an edge and the corresponding
        dataset type node.

        Parameters
        ----------
        node_data : `list`
            List of node keys and state dictionaries.  A node is appended if
            one does not already exist for this dataset type.
        edge_data : `list`
            List of node key pairs, connection names, and state dictionaries
            for edges.
        edge : `Edge`
            New edge being processed.
        parent : `PipelineGraph` or `None`
            Another pipeline graph whose dataset type nodes should be used
            when present.
        """
        new_dataset_type_node = None
        if parent is not None:
            new_dataset_type_node = parent._xgraph.nodes[edge.dataset_type_key].get("instance")
        if (existing_dataset_type_state := self._xgraph.nodes.get(edge.dataset_type_key)) is not None:
            existing_dataset_type_state["instance"] = new_dataset_type_node
        else:
            node_data.append(
                (
                    edge.dataset_type_key,
                    {
                        "instance": new_dataset_type_node,
                        "bipartite": NodeType.DATASET_TYPE.bipartite,
                    },
                )
            )
        edge_data.append(
            edge.nodes
            + (
                edge.connection_name,
                {"instance": edge},
            )
        )

    def _reorder(self, sorted_keys: Sequence[NodeKey]) -> None:
        """Set the order of all views of this graph from the given sorted
        sequence of task labels and dataset type names.
        """
        self._sorted_keys = sorted_keys
        self._tasks._reorder(sorted_keys)
        self._dataset_types._reorder(sorted_keys)

    def _reset(self) -> None:
        """Reset the all views of this graph following a modification that
        might invalidate them.
        """
        self._sorted_keys = None
        self._tasks._reset()
        self._dataset_types._reset()

    _xgraph: networkx.MultiDiGraph
    _sorted_keys: Sequence[NodeKey] | None
    _task_subsets: dict[str, TaskSubset]
    _description: str
    _tasks: TaskMappingView
    _dataset_types: DatasetTypeMappingView
    _raw_data_id: dict[str, Any]
    _universe: DimensionUniverse | None


def log_config_mismatch(msg: str) -> None:
    """Log messages about configuration mismatch.

    Parameters
    ----------
    msg : `str`
        Log message to use.
    """
    _LOG.fatal("Comparing configuration: %s", msg)


def compare_packages(packages: Packages, new_packages: Packages) -> bool:
    """Compare two versions of Packages.

    Parameters
    ----------
    packages : `Packages`
        Previously recorded package versions.  Updated in place to include
        any new packages that weren't present before.
    new_packages : `Packages`
        New set of package versions.

    Returns
    -------
    updated : `bool`
        `True` if ``packages`` was updated, `False` if not.

    Raises
    ------
    ConflictingDefinitionError
        Raised if versions are inconsistent.
    """
    diff = new_packages.difference(packages)
    if diff:
        versions_str = "; ".join(f"{pkg}: {diff[pkg][1]} vs {diff[pkg][0]}" for pkg in diff)
        raise ConflictingDefinitionError(f"Package versions mismatch: ({versions_str})")
    else:
        _LOG.debug("new packages are consistent with old")
    # Update the old set of packages in case we have more packages
    # that haven't been persisted.
    extra = new_packages.extra(packages)
    if extra:
        _LOG.debug("extra packages: %s", extra)
        packages.update(new_packages)
        return True
    return False
