"""
`deriva_ml_base.py` is the core module for the Deriva ML project.  This module implements the DerivaML class, which is
the primary interface to the Deriva based catalogs.  The module also implements the Feature and Vocabulary functions
in the DerivaML.

DerivaML and its associated classes all depend on a catalog that implements a `deriva-ml` schema with tables and
relationships that follow a specific data model.

"""

from __future__ import annotations

import getpass
import logging
from datetime import datetime
from itertools import chain
from pathlib import Path
import requests

from typing import Optional, Any, Iterable, TYPE_CHECKING

from deriva.core import (
    get_credential,
    urlquote,
    format_exception,
    DEFAULT_SESSION_CONFIG,
)
import deriva.core.datapath as datapath
from deriva.core.datapath import DataPathException
from deriva.core.deriva_server import DerivaServer
from deriva.core.ermrest_catalog import ResolveRidResult
from deriva.core.ermrest_model import Key, Table
from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
from pydantic import validate_call, ConfigDict

from .execution_configuration import ExecutionConfiguration, Workflow
from .feature import Feature, FeatureRecord
from .dataset import Dataset
from .dataset_aux_classes import DatasetSpec
from .dataset_bag import DatasetBag
from .deriva_model import DerivaModel
from .upload import table_path, execution_rids, asset_file_path
from .deriva_definitions import ColumnDefinition
from .deriva_definitions import (
    RID,
    Status,
    DerivaMLException,
    ML_SCHEMA,
    VocabularyTerm,
    MLVocab,
    FileSpec,
    TableDefinition,
)

try:
    from icecream import ic
except ImportError:  # Graceful fallback if IceCream isn't installed.
    ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a)  # noqa


if TYPE_CHECKING:
    from .execution import Execution


class DerivaML(Dataset):
    """Base class for ML operations on a Deriva catalog.

    This class is intended to be used as a base class on which more domain specific interfaces are built.

    Attributes:
        host_name: Hostname of the Deriva server.
        catalog_id: Catalog ID. Either and identifier, or a catalog name.
        domain_schema: Schema name for domain specific tables and relationships.
        model: ERMRest model for the catalog
    """

    def __init__(
        self,
        hostname: str,
        catalog_id: str | int,
        domain_schema: Optional[str] = None,
        project_name: Optional[str] = None,
        cache_dir: Optional[str] = None,
        working_dir: Optional[str] = None,
        model_version: str = "1",
        ml_schema: str = ML_SCHEMA,
        logging_level=logging.INFO,
    ):
        """Create and initialize a DerivaML instance.

        This method will connect to a catalog, and initialize local configuration for the ML execution.
        This class is intended to be used as a base class on which domain-specific interfaces are built.

        Args:
            hostname: Hostname of the Deriva server.
            catalog_id: Catalog ID. Either and identifier, or a catalog name.
            domain_schema: Schema name for domain specific tables and relationships.
            cache_dir: Directory path for caching data downloaded from the Deriva server as bdbag.
            working_dir: Directory path for storing data used by or generated by any computations.
            model_version: A string that indicates the version model.  Typically passed in via
        """
        self.credential = get_credential(hostname)
        server = DerivaServer(
            "https",
            hostname,
            credentials=self.credential,
            session_config=self._get_session_config(),
        )
        self.catalog = server.connect_ermrest(catalog_id)
        self.model = DerivaModel(
            self.catalog.getCatalogModel(), domain_schema=domain_schema
        )

        default_workdir = self.__class__.__name__ + "_working"
        self.working_dir = (
            Path(working_dir) / getpass.getuser()
            if working_dir
            else Path.home() / "deriva-ml"
        ) / default_workdir

        self.working_dir.mkdir(parents=True, exist_ok=True)
        self.cache_dir = (
            Path(cache_dir) if cache_dir else Path.home() / "deriva-ml" / "cache"
        )

        self.cache_dir.mkdir(parents=True, exist_ok=True)

        # Initialize dataset class.
        super().__init__(self.model, self.cache_dir, self.working_dir)
        self._logger = logging.getLogger("deriva_ml")
        self._logger.setLevel(logging_level)

        self.host_name = hostname
        self.catalog_id = catalog_id
        self.ml_schema = ml_schema
        self.version = model_version
        self.configuration = None
        self._execution: Optional[Execution] = None
        self.domain_schema = self.model.domain_schema
        self.project_name = project_name or self.domain_schema
        self.start_time = datetime.now()
        self.status = Status.pending.value

        logging.basicConfig(
            level=logging_level,
            format="%(asctime)s - %(name)s.%(levelname)s - %(message)s",
        )

        # Set logging level for Deriva library
        deriva_logger = logging.getLogger("deriva")
        deriva_logger.setLevel(logging_level)

        if "dirty" in self.version:
            logging.info(
                f"Loading dirty model.  Consider commiting and tagging: {self.version}"
            )

    def __del__(self):
        try:
            if self._execution and self._execution.status != Status.completed:
                self._execution.update_status(Status.aborted, "Execution Aborted")
        except (AttributeError, requests.HTTPError):
            pass

    @staticmethod
    def _get_session_config():
        """ """
        session_config = DEFAULT_SESSION_CONFIG.copy()
        session_config.update(
            {
                # our PUT/POST to ermrest is idempotent
                "allow_retry_on_all_methods": True,
                # do more retries before aborting
                "retry_read": 8,
                "retry_connect": 5,
                # increase delay factor * 2**(n-1) for Nth retry
                "retry_backoff_factor": 5,
            }
        )
        return session_config

    # noinspection PyProtectedMember
    @property
    def pathBuilder(self) -> datapath._CatalogWrapper:
        """Get a new instance of a pathBuilder object."""
        return self.catalog.getPathBuilder()

    @property
    def domain_path(self):
        """Get a new instance of a pathBuilder object to the domain schema"""

        return self.pathBuilder.schemas[self.domain_schema]

    def table_path(self, table: str | Table) -> Path:
        """Return a local file path in which to place a CSV to add values to a table on upload.

        Args:
          table: str | Table:

        Returns:
            Path to a CSV file in which to add values to a table on upload.
        """
        return table_path(
            self.working_dir,
            schema=self.domain_schema,
            table=self.model.name_to_table(table).name,
        )

    def download_dir(self, cached: bool = False) -> Path:
        """Location where downloaded files are placed.

        Args:
          cached: bool:  (Default value = False)

        Returns:

        """
        return self.cache_dir if cached else self.working_dir

    @staticmethod
    def globus_login(host: str) -> None:
        """Log  into the specified host using Globus.

        Args:
            host:

        Returns:

        """
        gnl = GlobusNativeLogin(host=host)
        if gnl.is_logged_in([host]):
            print("You are already logged in.")
        else:
            gnl.login(
                [host],
                no_local_server=True,
                no_browser=True,
                refresh_tokens=True,
                update_bdbag_keychain=True,
            )
            print("Login Successful")

    def chaise_url(self, table: RID | Table) -> str:
        """Return a Chaise URL to the specified table.

        Args:
            table: Table or RID to be visited
            table: str | Table:

        Returns:
            URL to the table in Chaise format.
        """
        table_obj = self.model.name_to_table(table)
        try:
            uri = self.catalog.get_server_uri().replace(
                "ermrest/catalog/", "chaise/recordset/#"
            )
        except DerivaMLException:
            # Perhaps we have a RID....
            uri = self.cite(table)
        return f"{uri}/{urlquote(table_obj.schema.name)}:{urlquote(table_obj.name)}"

    def cite(self, entity: dict | str) -> str:
        """Return a citation URL for the provided entity.

        Args:
            entity: A dict that contains the column values for a specific entity or a RID.

        Returns:
            The URI for the provided entity.

        Raises:
            DerivaMLException: if provided RID does not exist.
        """
        if isinstance(entity, str) and entity.startswith(
            f"https://{self.host_name}/id/{self.catalog_id}/"
        ):
            # Already got a citation...
            return entity
        try:
            self.resolve_rid(
                rid := entity if isinstance(entity, str) else entity["RID"]
            )
            return f"https://{self.host_name}/id/{self.catalog_id}/{rid}@{self.catalog.latest_snapshot().snaptime}"
        except KeyError as e:
            raise DerivaMLException(f"Entity {e} does not have RID column")
        except DerivaMLException as _e:
            raise DerivaMLException("Entity RID does not exist")

    def user_list(self) -> list[dict[str, str]]:
        """List of users in the catalog

        Args:

        Returns:
          A list of dictionaries containing user information.

        """
        user_path = self.pathBuilder.public.ERMrest_Client.path
        return [
            {"ID": u["ID"], "Full_Name": u["Full_Name"]}
            for u in user_path.entities().fetch()
        ]

    def resolve_rid(self, rid: RID) -> ResolveRidResult:
        """Return a named tuple with information about the specified RID.

        Args:
            rid: RID of the object of interest

        Returns:
            ResolveRidResult which has information about the specified RID.

        Raises:
          DerivaMLException: if the RID doesn't exist.
        """
        try:
            return self.catalog.resolve_rid(rid, self.model.model)
        except KeyError as _e:
            raise DerivaMLException(f"Invalid RID {rid}")

    def retrieve_rid(self, rid: RID) -> dict[str, Any]:
        """Return a dictionary that represents the values of the specified RID.

        Args:
            rid: RID of the object of interest

        Returns:
          A dictionary that represents the values of the specified RID.

        Raises:
          DerivaMLException: if the RID doesn't exist.
        """
        return self.resolve_rid(rid).datapath.entities().fetch()[0]

    def add_page(self, title: str, content: str) -> None:
        """

        Args:
          title: str:
          content: str:

        Returns:

        """
        self.pathBuilder.www.tables[self.domain_schema].insert(
            [{"Title": title, "Content": content}]
        )

    def create_vocabulary(
        self, vocab_name: str, comment: str = "", schema: Optional[str] = None
    ) -> Table:
        """Create a controlled vocabulary table with the given vocab name.

        Args:
            vocab_name: Name of the controlled vocabulary table.
            comment: Description of the vocabulary table. (Default value = '')
            schema: Schema in which to create the controlled vocabulary table.  Defaults to domain_schema.
            vocab_name: str:

        Returns:
            An ERMRest table object for the newly created vocabulary table.
        """
        schema = schema or self.domain_schema
        return self.model.schemas[schema].create_table(
            Table.define_vocabulary(
                vocab_name, f"{self.project_name}:{{RID}}", comment=comment
            )
        )

    def create_table(self, table: TableDefinition) -> Table:
        """Create a table from a table definition."""
        return self.model.schemas[self.domain_schema].create_table(table.model_dump())

    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def create_asset(
        self,
        asset_name: str,
        column_defs: Optional[Iterable[ColumnDefinition]] = None,
        fkey_defs: Optional[Iterable[ColumnDefinition]] = None,
        referenced_tables: Optional[Iterable[Table]] = None,
        comment: str = "",
        schema: Optional[str] = None,
    ) -> Table:
        """Create an asset table with the given asset name.

        Args:
            asset_name: Name of the asset table.
            column_defs: Iterable of ColumnDefinition objects to provide additional metadata for asset.
            fkey_defs: Iterable of ForeignKeyDefinition objects to provide additional metadata for asset.
            referenced_tables: Iterable of Table objects to which asset should provide foreign-key references to.
            comment: Description of the asset table. (Default value = '')
            schema: Schema in which to create the asset table.  Defaults to domain_schema.
            asset_name: str:
            schema: str:  (Default value = None)

        Returns:
            Table object for the asset table.
        """
        column_defs = column_defs or []
        fkey_defs = fkey_defs or []
        referenced_tables = referenced_tables or []
        schema = schema or self.domain_schema

        self.add_term(
            MLVocab.asset_type, asset_name, description=f"A {asset_name} asset"
        )
        asset_table = self.model.schemas[schema].create_table(
            Table.define_asset(
                schema,
                asset_name,
                column_defs=[c.model_dump() for c in column_defs],
                fkey_defs=[fk.model_dump() for fk in fkey_defs],
                comment=comment,
            )
        )

        self.model.schemas[self.domain_schema].create_table(
            Table.define_association(
                [
                    (asset_table.name, asset_table),
                    ("Asset_Type", self.model.name_to_table("Asset_Type")),
                ]
            )
        )
        for t in referenced_tables:
            asset_table.create_reference(self.model.name_to_table(t))
        # Create a table to track execution that creates the asset
        atable = self.model.schemas[self.domain_schema].create_table(
            Table.define_association(
                [
                    (asset_name, asset_table),
                    (
                        "Execution",
                        self.model.schemas[self.ml_schema].tables["Execution"],
                    ),
                ]
            )
        )
        atable.create_reference(self.model.name_to_table("Asset_Role"))
        return asset_table

    # @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def list_assets(self, asset_table: Table | str):
        """Return the contents of an asset table"""

        if not self.model.is_asset(asset_table):
            raise DerivaMLException(f"Table {asset_table.name} is not an asset")
        asset_table = self.model.name_to_table(asset_table)
        pb = self._model.catalog.getPathBuilder()
        asset_path = pb.schemas[asset_table.schema.name].tables[asset_table.name]

        asset_type_table = self._model.find_association(asset_table, MLVocab.asset_type)
        type_path = pb.schemas[asset_type_table.schema.name].tables[
            asset_type_table.name
        ]

        # Get a list of all the asset_type values associated with this dataset_table.
        assets = []
        for asset in asset_path.entities().fetch():
            asset_types = (
                type_path.filter(type_path.columns[asset_table.name] == asset["RID"])
                .attributes(type_path.Asset_Type)
                .fetch()
            )
            assets.append(
                asset
                | {
                    MLVocab.asset_type.value: [
                        asset_type[MLVocab.asset_type.value]
                        for asset_type in asset_types
                    ]
                }
            )
        return assets

    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def create_feature(
        self,
        target_table: Table | str,
        feature_name: str,
        terms: Optional[list[Table | str]] = None,
        assets: Optional[list[Table | str]] = None,
        metadata: Optional[Iterable[ColumnDefinition | Table | Key | str]] = None,
        optional: Optional[list[str]] = None,
        comment: str = "",
    ) -> type[FeatureRecord]:
        """Create a new feature that can be associated with a table.

        The feature can associate a controlled vocabulary term, an asset, or any other values with a
        specific instance of an object and  execution.

        Args:
            feature_name: Name of the new feature to be defined
            target_table: table name or object on which the feature is to be associated
            terms: List of controlled vocabulary terms that will be part of the feature value
            assets: List of asset table names or objects that will be part of the feature value
            metadata: List of other value types that are associated with the feature
            optional: List of columns that are optional in the feature
            comment: return: A Feature class that can be used to create instances of the feature.

        Returns:
            A Feature class that can be used to create instances of the feature.

        Raises:
            DerivaException: If the feature cannot be created.
        """

        terms = terms or []
        assets = assets or []
        metadata = metadata or []
        optional = optional or []

        def normalize_metadata(m: Key | Table | ColumnDefinition | str):
            """

            Args:
              m: Key | Table | ColumnDefinition | str:

            Returns:

            """
            if isinstance(m, str):
                return self.model.name_to_table(m)
            elif isinstance(m, ColumnDefinition):
                return m.model_dump()
            else:
                return m

        # Make sure that the provided assets or terms are actually assets or terms.
        if not all(map(self.model.is_asset, assets)):
            raise DerivaMLException("Invalid create_feature asset table.")
        if not all(map(self.model.is_vocabulary, terms)):
            raise DerivaMLException("Invalid create_feature asset table.")

        # Get references to the necessary tables and make sure that the
        # provided feature name exists.
        target_table = self.model.name_to_table(target_table)
        execution = self.model.schemas[self.ml_schema].tables["Execution"]
        feature_name_table = self.model.schemas[self.ml_schema].tables["Feature_Name"]
        feature_name_term = self.add_term(
            "Feature_Name", feature_name, description=comment
        )
        atable_name = f"Execution_{target_table.name}_{feature_name_term.name}"

        # Now create the association table that implements the feature.
        atable = self.model.schemas[self.domain_schema].create_table(
            target_table.define_association(
                table_name=atable_name,
                associates=[execution, target_table, feature_name_table],
                metadata=[
                    normalize_metadata(m) for m in chain(assets, terms, metadata)
                ],
                comment=comment,
            )
        )
        # Now set optional terms.
        for c in optional:
            atable.columns[c].alter(nullok=True)
        atable.columns["Feature_Name"].alter(default=feature_name_term.name)
        return self.feature_record_class(target_table, feature_name)

    def feature_record_class(
        self, table: str | Table, feature_name: str
    ) -> type[FeatureRecord]:
        """Create a pydantic model for entries into the specified feature table.

        For information on how to
        See the pydantic documentation for more details about the pydantic model.

        Args:
            table: table name or object on which the feature is to be associated
            feature_name: name of the feature to be created
            table: str | Table:
            feature_name: str:

        Returns:
            A Feature class that can be used to create instances of the feature.
        """
        return self.lookup_feature(table, feature_name).feature_record_class()

    def delete_feature(self, table: Table | str, feature_name: str) -> bool:
        """

        Args:
          table: Table | str:
          feature_name: str:

        Returns:
        """
        table = self.model.name_to_table(table)
        try:
            feature = next(
                f for f in self.find_features(table) if f.feature_name == feature_name
            )
            feature.feature_table.drop()
            return True
        except StopIteration:
            return False

    def lookup_feature(self, table: str | Table, feature_name: str) -> Feature:
        """Lookup the named feature associated with the provided table.

        Args:
            table: param feature_name:
            table: str | Table:
            feature_name: str:

        Returns:
            A Feature class that represents the requested feature.

        Raises:
          DerivaMLException: If the feature cannot be found.
        """
        return self.model.lookup_feature(table, feature_name)

    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def find_features(self, table: Table | str) -> Iterable[Feature]:
        """List the names of the features in the specified table.

        Args:
            table: The table to find features for.
            table: Table | str:

        Returns:
            An iterable of FeatureResult instances that describe the current features in the table.
        """
        return self.model.find_features(table)

    # noinspection PyProtectedMember
    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def list_feature_values(
        self, table: Table | str, feature_name: str
    ) -> datapath._ResultSet:
        """Return a datapath ResultSet containing all values of a feature associated with a table.

        Args:
            table: param feature_name:
            table: Table | str:
            feature_name: str:

        Returns:

        """
        table = self.model.name_to_table(table)
        feature = self.lookup_feature(table, feature_name)
        pb = self.catalog.getPathBuilder()
        return (
            pb.schemas[feature.feature_table.schema.name]
            .tables[feature.feature_table.name]
            .entities()
            .fetch()
        )

    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def add_term(
        self,
        table: str | Table,
        term_name: str,
        description: str,
        synonyms: Optional[list[str]] = None,
        exists_ok: bool = True,
    ) -> VocabularyTerm:
        """Creates a new control vocabulary term in the control vocabulary table.

        Args:

        Args:
            table: The name of the control vocabulary table.
            term_name: The name of the new control vocabulary.
            description: The description of the new control vocabulary.
            synonyms: Optional list of synonyms for the new control vocabulary. Defaults to an empty list.
            exists_ok: Optional flag indicating whether to allow creation if the control vocabulary name
                already exists. Defaults to True.

        Returns:
          The RID of the newly created control vocabulary.

        Raises:
          DerivaException: If the control vocabulary name already exists and exist_ok is False.
        """
        synonyms = synonyms or []
        table = self.model.name_to_table(table)
        pb = self.catalog.getPathBuilder()
        if not (self.model.is_vocabulary(table)):
            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")

        schema_name = table.schema.name
        table_name = table.name
        try:
            term_id = VocabularyTerm.model_validate(
                pb.schemas[schema_name]
                .tables[table_name]
                .insert(
                    [
                        {
                            "Name": term_name,
                            "Description": description,
                            "Synonyms": synonyms,
                        }
                    ],
                    defaults={"ID", "URI"},
                )[0]
            )
        except DataPathException:
            term_id = self.lookup_term(table, term_name)
            if not exists_ok:
                raise DerivaMLException(f"{term_name} already exists")
            # Check vocabulary
        return term_id

    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def lookup_term(self, table: str | Table, term_name: str) -> VocabularyTerm:
        """Given a term name, return the vocabulary record.  Can provide either the term name
         or a synonym for the term.  Generate an exception if the term is not in the vocabulary.

        Args:
            table: The name of the controlled vocabulary table or a ERMRest table object.
            term_name: The name of the term to look up.

        Returns:
          The entry the associated term or synonym.

        Raises:
          DerivaException: If the schema or vocabulary table doesn't exist, or if the term is not
            found in the vocabulary.
        """
        vocab_table = self.model.name_to_table(table)
        if not self.model.is_vocabulary(vocab_table):
            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")
        schema_name, table_name = vocab_table.schema.name, vocab_table.name
        schema_path = self.catalog.getPathBuilder().schemas[schema_name]

        for term in schema_path.tables[table_name].entities().fetch():
            if term_name == term["Name"] or (
                term["Synonyms"] and term_name in term["Synonyms"]
            ):
                return VocabularyTerm.model_validate(term)
        raise DerivaMLException(f"Term {term_name} is not in vocabulary {table_name}")

    def list_vocabulary_terms(self, table: str | Table) -> list[VocabularyTerm]:
        """Return a list of terms that are in a vocabulary table.

        Args:
            table: The name of the controlled vocabulary table or a ERMRest table object.
            table: str | Table:

        Returns:
            The list of terms that are in a vocabulary table.

        Raises:
            DerivaMLException: If the schema or vocabulary table doesn't exist, or if the table is not
                a controlled vocabulary.
        """
        pb = self.catalog.getPathBuilder()
        table = self.model.name_to_table(table)
        if not (self.model.is_vocabulary(table)):
            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")

        return [
            VocabularyTerm(**v)
            for v in pb.schemas[table.schema.name].tables[table.name].entities().fetch()
        ]

    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def download_dataset_bag(
        self,
        dataset: DatasetSpec,
        execution_rid: Optional[RID] = None,
    ) -> DatasetBag:
        """Download a dataset onto the local file system.  Create a MINID for the dataset if one doesn't already exist.

        Args:
            dataset: Specification of the dataset to be downloaded.
            execution_rid: Execution RID for the dataset.

        Returns:
            Tuple consisting of the path to the dataset, the RID of the dataset that was downloaded and the MINID
            for the dataset.
        """
        return self._download_dataset_bag(
            dataset=dataset,
            execution_rid=execution_rid,
            snapshot_catalog=DerivaML(self.host_name, self._version_snapshot(dataset)),
        )

    def _update_status(
        self, new_status: Status, status_detail: str, execution_rid: RID
    ):
        """Update the status of an execution in the catalog.

        Args:
            new_status: New status.
            status_detail: Details of the status.
            execution_rid: Resource Identifier (RID) of the execution.
            new_status: Status:
            status_detail: str:
             execution_rid: RID:

        Returns:

        """
        self.status = new_status.value
        self.pathBuilder.schemas[self.ml_schema].Execution.update(
            [
                {
                    "RID": execution_rid,
                    "Status": self.status,
                    "Status_Detail": status_detail,
                }
            ]
        )

    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
    def add_files(
        self,
        files: Iterable[FileSpec],
        file_types: str | list[str],
        execution_rid: Optional[RID] = None,
    ) -> Iterable[RID]:
        """Add a new file to the File table in the catalog.

        Args:
            file_types: One or more file types.  Must be a term from the File_Type controlled vocabulary.
            files: A sequence of file specifications that describe the files to add.
            execution_rid: Resource Identifier (RID) of the execution to associate with the file.

        Returns:
            Iterable of the RIDs of the files that were added.
        """
        defined_types = self.list_vocabulary_terms(MLVocab.file_type)
        if execution_rid and self.resolve_rid(execution_rid).table.name != "Execution":
            raise DerivaMLException(
                f"RID {execution_rid} is not for an execution table."
            )

        def check_file_type(dtype: str) -> bool:
            """Make sure that the specified string is either the name or synonym for a file type term."""
            for term in defined_types:
                if dtype == term.name or (term.synonyms and file_type in term.synonyms):
                    return True
            return False

        # Create the entry for the new dataset_table and get its RID.
        file_types = [file_types] if isinstance(file_types, str) else file_types
        pb = self._model.catalog.getPathBuilder()
        for file_type in file_types:
            if not check_file_type(file_type):
                raise DerivaMLException("File type must be a vocabulary term.")
        file_table_path = pb.schemas[self.ml_schema].tables["File"]
        file_rids = [
            e["RID"] for e in file_table_path.insert([f.model_dump() for f in files])
        ]

        # Get the name of the association table between file_table and file_type.
        atable = next(
            self._model.schemas[self._ml_schema]
            .tables[MLVocab.file_type]
            .find_associations()
        ).name
        pb.schemas[self._ml_schema].tables[atable].insert(
            [
                {"File_Type": file_type, "File": file_rid}
                for file_rid in file_rids
                for file_type in file_types
            ]
        )

        if execution_rid:
            # Get the name of the association table between file_table and execution.
            exec_table = next(
                self._model.schemas[self._ml_schema]
                .tables["Execution"]
                .find_associations()
            ).name
            pb.schemas[self._ml_schema].tables[exec_table].insert(
                [
                    {"File": file_rid, "Execution": execution_rid}
                    for file_rid in file_rids
                ]
            )

        return file_rids

    def list_files(
        self, file_types: Optional[list[str]] = None
    ) -> list[dict[str, Any]]:
        """Return the contents of the file table.  Denormalized file types into the file record."""
        ml_path = self.pathBuilder.schemas[self._ml_schema]
        file_path = ml_path.File
        type_path = ml_path.File_File_Type

        # Get a list of all the dataset_type values associated with this dataset_table.
        path = file_path.link(type_path)
        path = path.attributes(
            path.File.RID,
            path.File.URL,
            path.File.MD5,
            path.File.Length,
            path.File.Description,
            path.File_File_Type.File_Type,
        )
        file_map = {}
        for f in path.fetch():
            file_map.setdefault(f["RID"], f | {"File_Types": []})["File_Types"].append(
                f["File_Type"]
            )

        # Now get rid of the File_Type key and return the result
        return [(f, f.pop("File_Type"))[0] for f in file_map.values()]

    def list_workflows(self) -> list[Workflow]:
        """Return a list of all the workflows in the catalog."""
        workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
        return [
            Workflow(
                name=w["Name"],
                url=w["URL"],
                workflow_type=w["Workflow_Type"],
                version=w["Version"],
                description=w["Description"],
                rid=w["RID"],
                checksum=w["Checksum"],
            )
            for w in workflow_path.entities().fetch()
        ]

    def add_workflow(self, workflow: Workflow) -> RID:
        """Add a workflow to the Workflow table.

        Args:
            workflow: An instance of a Workflow object.

        Returns:
          - str: Resource Identifier (RID) of the added workflow.

        """

        # Check to make sure that the workflow is not already in the table. If it's not, add it.

        if workflow_rid := self.lookup_workflow(workflow.url):
            return workflow_rid

        ml_schema_path = self.pathBuilder.schemas[self.ml_schema]
        try:
            # Record doesn't exist already
            workflow_record = {
                "URL": workflow.url,
                "Name": workflow.name,
                "Description": workflow.description,
                "Checksum": workflow.checksum,
                "Version": workflow.version,
                MLVocab.workflow_type: self.lookup_term(
                    MLVocab.workflow_type, workflow.workflow_type
                ).name,
            }
            workflow_rid = ml_schema_path.Workflow.insert([workflow_record])[0]["RID"]
        except Exception as e:
            error = format_exception(e)
            raise DerivaMLException(f"Failed to insert workflow. Error: {error}")
        return workflow_rid

    def lookup_workflow(self, url: str) -> Optional[RID]:
        """Given a URL, look in the workflow table to find a matching workflow."""
        workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
        try:
            url_column = workflow_path.URL
            return list(workflow_path.filter(url_column == url).entities())[0]["RID"]
        except IndexError:
            return None

    def create_workflow(
        self, name: str, workflow_type: str, description: str = ""
    ) -> Workflow:
        """Identify current executing program and return a workflow RID for it

        Determine the notebook or script that is currently being executed. Assume that  this is
        being executed from a cloned GitHub repository.  Determine the remote repository name for
        this object.  Then either retrieve an existing workflow for this executable or create
        a new one.

        Args:
            name: The name of the workflow.
            workflow_type: The type of the workflow.
            description: The description of the workflow.
        """
        # Make sure type is correct.
        self.lookup_term(MLVocab.workflow_type, workflow_type)

        return Workflow.create_workflow(name, workflow_type, description)

    # @validate_call
    def create_execution(
        self, configuration: ExecutionConfiguration, dry_run: bool = False
    ) -> "Execution":
        """Create an execution object

        Given an execution configuration, initialize the local compute environment to prepare for executing an
        ML or analytic routine.  This routine has a number of side effects.

        1. The datasets specified in the configuration are downloaded and placed in the cache-dir. If a version is
        not specified in the configuration, then a new minor version number is created for the dataset and downloaded.

        Args:
            configuration: ExecutionConfiguration:
            dry_run: Do not create an execution record or upload results.

        Returns:
            An execution object.
        """
        from .execution import Execution

        self._execution = Execution(configuration, self, dry_run=dry_run)
        return self._execution

    # @validate_call
    def restore_execution(self, execution_rid: Optional[RID] = None) -> "Execution":
        """Return an Execution object for a previously started execution with the specified RID."""

        from .execution import Execution

        # Find path to execution
        if not execution_rid:
            e_rids = execution_rids(self.working_dir)
            if len(e_rids) != 1:
                raise DerivaMLException(f"Multiple execution RIDs were found {e_rids}.")

            execution_rid = e_rids[0]
        cfile = asset_file_path(
            prefix=self.working_dir,
            exec_rid=execution_rid,
            file_name="configuration.json",
            asset_table=self.model.name_to_table("Execution_Metadata"),
            metadata={},
        )

        if cfile.exists():
            configuration = ExecutionConfiguration.load_configuration(cfile)
        else:
            execution = self.retrieve_rid(execution_rid)
            configuration = ExecutionConfiguration(
                workflow=execution["Workflow"],
                description=execution["Description"],
            )
        return Execution(configuration, self, reload=execution_rid)
