__all__ = [
    "compute_visits_sha256",
    "VisitSequenceArchiveMetadata",
    "VSARCHIVE_PGDATABASE",
    "VSARCHIVE_PGHOST",
    "VSARCHIVE_PGUSER",
    "VSARCHIVE_PGSCHEMA",
]

import hashlib
import json
import logging
import os
import warnings
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Dict, Mapping, Tuple
from uuid import UUID

import numpy as np
import pandas as pd
import psycopg2
import psycopg2.extras
import psycopg2.pool
import sqlalchemy
import yaml
from astropy.time import Time
from lsst.resources import ResourcePath
from psycopg2 import sql

from .util import compute_conda_env, dayobs_to_date, opsimdb_to_hdf5

VSARCHIVE_PGDATABASE = "opsim_log"
VSARCHIVE_PGHOST = "usdf-maf-visit-seq-archive-tx-ro.sdf.slac.stanford.edu"
VSARCHIVE_PGUSER = "reader"
VSARCHIVE_PGPORT = None
VSARCHIVE_PGSCHEMA = "vsmd"

JSON_DUMP_LIMIT = 4096

LOGGER = logging.getLogger(__name__)

psycopg2.extras.register_uuid()


def compute_visits_sha256(visits: pd.DataFrame) -> bytes:
    recs = visits.to_records()
    visitseq_hash = hashlib.sha256(str(recs.dtype).encode())
    visitseq_hash.update(np.ascontiguousarray(recs).data.tobytes())
    visitseq_sha256 = bytes.fromhex(visitseq_hash.hexdigest())
    return visitseq_sha256


def metadata_yaml_from_prototype(
    sim_date: str | date,
    sim_index: str | int,
    proto_sim_archive_uri: str = "s3://rubin:rubin-scheduler-prenight/opsim/",
) -> str:
    """Return a sim metadata YAML derived from the prototype sim archive.

    Parameters
    ----------
    sim_date : `str` or `datetime.date`
        The simulation date in ISO format or a ``date`` object.
    sim_index : `str` or `int`
        The simulation index identifying the specific simulation
        within the date directory.
    proto_sim_archive_uri : `str`, optional
        Base URI of the simulation archive.
        Defaults to the prototype location in the S3 bucket.

    Returns
    -------
    sim_metadata: `str`
        `yaml` representation of the simulation metadata.

    """
    if isinstance(sim_date, date):
        sim_date = sim_date.isoformat()
    assert isinstance(sim_date, str)

    sim_archive_rp = (
        ResourcePath(proto_sim_archive_uri)
        .join(sim_date, forceDirectory=True)
        .join(str(sim_index), forceDirectory=True)
    )
    sims_found = sim_archive_rp.exists() and sim_archive_rp.join("sim_metadata.yaml").exists()
    if not sim_archive_rp.exists():
        raise ValueError(f"No simulations found at {sim_archive_rp.geturl()}.")

    if not sim_archive_rp.join("sim_metadata.yaml").exists():
        raise ValueError(
            f"No simulation metadata found at {sim_archive_rp.join('sim_metadata.yaml').geturl()}"
        )

    if sims_found:
        metadata = yaml.safe_load(sim_archive_rp.join("sim_metadata.yaml").read().decode())
    else:
        raise ValueError("No simulation found")

    if "label" not in metadata:
        raise ValueError("Metadata yaml must include a label")

    label = metadata["label"]

    # If there is no creation time given, try extracting it from the label
    if "creation_time" not in metadata:
        # The prenight labels usually end in the creation time.
        if " ".join(label.split()[-3:-1]) == "run at":
            maybe_date_str = label.split()[-1]
            is_date_str = False
            try:
                datetime.fromisoformat(maybe_date_str)
                is_date_str = True
            except ValueError:
                is_date_str = False

            if is_date_str:
                metadata["creation_time"] = maybe_date_str

    # If we still do not have a creation time, invent one from the date
    # and index.
    if "creation_time" not in metadata:
        creation_time = datetime.fromisoformat(sim_date + "T12:00:00Z") + timedelta(seconds=int(sim_index))
        metadata["creation_time"] = creation_time.isoformat()

    if "files" in metadata:
        for file_type in metadata["files"]:
            if "url" not in metadata["files"][file_type]:
                fname = metadata["files"][file_type]["name"]
                metadata["files"][file_type]["url"] = sim_archive_rp.join(fname).geturl()

    metadata_yaml = yaml.dump(metadata)

    return metadata_yaml


class VisitSequenceArchiveMetadata:
    """Interface to metadata database that tracks sequences
    of visits.

    Parameters
    ----------
    metadata_db_kwargs: `Mapping` or `None`
        A dictionary or other mapping defining the connection
        parameters for connecting to the postgresql database
        that holds the sequence metadata. Keys are passed as keyword
        arguments to ``psycopg2.pool.SimpleConnectionPool``.
        If ``None``, the dictionary is built from the environment
        variables ``VSARCHIVE_PGDATABASE``, ``VSARCHIVE_PGHOST``,
        ``VSARCHIVE_PGUSER``, and ``VSARCHIVE_PGPORT`` (if they
        exist) or corresponding module variables in
        ``rubin_sim.sim_archive.vseqarchiv`` (if they do not).
    metadata_db_schema: `str` or `None`
        The schema in the database holding the metadata. If
        ``None``, defaults to
        ``rubin_sim.sim_archive.vseqarchive.VSARCHIVE_PGSCHEMA``.
        Default is ``None``.

    Notes
    -----
    The visit sequence archive consists of two primary architectural
    components: an archive in which files associated with a sequence
    of visits are stored (including, optionally, the table of visits
    themselves); and postegresql database with tables with metadata
    about the sequneces of visits, including the
    provenance of the visits, the locations of the files associated
    with each sequence of visits, comments and tags associated with
    each sequence of visits, and basic statistics of the visits.
    This class is an interface to the postgresql metadata database.
    """

    def __init__(
        self,
        metadata_db_kwargs: Mapping | None = None,
        metadata_db_schema: str | None = None,
    ):
        if metadata_db_schema is None:
            metadata_db_schema = VSARCHIVE_PGSCHEMA
        assert isinstance(metadata_db_schema, str)

        if not isinstance(metadata_db_kwargs, dict):
            metadata_db_kwargs = {} if metadata_db_kwargs is None else dict(metadata_db_kwargs)
        assert isinstance(metadata_db_kwargs, dict)

        if "database" not in metadata_db_kwargs or metadata_db_kwargs["database"] is None:
            if "VSARCHIVE_PGDATABASE" in os.environ:
                metadata_db_kwargs["database"] = os.environ["VSARCHIVE_PGDATABASE"]
            else:
                metadata_db_kwargs["database"] = VSARCHIVE_PGDATABASE

        if "host" not in metadata_db_kwargs or metadata_db_kwargs["host"] is None:
            if "VSARCHIVE_PGHOST" in os.environ:
                metadata_db_kwargs["host"] = os.environ["VSARCHIVE_PGHOST"]
            else:
                metadata_db_kwargs["host"] = VSARCHIVE_PGHOST

        if "user" not in metadata_db_kwargs or metadata_db_kwargs["user"] is None:
            if "VSARCHIVE_PGUSER" in os.environ:
                metadata_db_kwargs["user"] = os.environ["VSARCHIVE_PGUSER"]
            else:
                metadata_db_kwargs["user"] = VSARCHIVE_PGUSER

        if "port" not in metadata_db_kwargs or metadata_db_kwargs["port"] is None:
            if "VSARCHIVE_PGPORT" in os.environ:
                metadata_db_kwargs["port"] = os.environ["VSARCHIVE_PGPORT"]
            elif VSARCHIVE_PGPORT is not None:
                metadata_db_kwargs["port"] = VSARCHIVE_PGPORT

        self.pg_pool = psycopg2.pool.SimpleConnectionPool(1, 5, **metadata_db_kwargs)
        # On some operations, pandas does not
        # work well directly with the psycopg2
        # connection, but can work with sqlalchemy.
        # Make and alchemy engine that acts as
        # a middle-man between pandas and psycopg2,
        # using the same connection pool.
        # There is a potential issue in that SA
        # does not return connections to the
        # psycopg pool when it is done. But, it
        # has its own pool, and so in expected use
        # it will just get one and keep reusing it.
        self.sa_engine = sqlalchemy.create_engine(
            "postgresql+psycopg2://", creator=self.pg_pool.getconn, pool_pre_ping=True
        )

        self.metadata_db_schema: str = metadata_db_schema

    def query(
        self,
        query: str | sql.SQL | sql.Composed,
        data: dict = {},
        commit: bool = False,
        return_result: bool = True,
    ) -> tuple:
        """Run a simple query on the visit sequence database.

        Parameters
        ----------
        query : `str` or `sql.SQL`
            The query to send
        data : `dict`
            Data to include in the query
        commit : `bool`
            Commit the query (e.g. for an INSERT), defaults to False
        return_result : `bool`
            Return the result, defaults to True

        Returns
        -------
        result : `tuple`
            The result of the query
        """

        connection = None
        try:
            # Get a connection from the pool
            connection = self.pg_pool.getconn()
            with connection.cursor() as cursor:
                search_path_query = sql.SQL("SET search_path TO {};").format(
                    sql.Identifier(self.metadata_db_schema)
                )
                cursor.execute(search_path_query)

                if len(data) > 0:
                    cursor.execute(query, data)
                else:
                    cursor.execute(query)
                result = cursor.fetchall() if return_result else (None,)
            if commit:
                connection.commit()
            else:
                # If a commit was not requested, make sure
                # there were no alterations to the database
                # in this transaction.
                connection.rollback()
        except Exception as e:
            # If something went wrong, restore the
            # connection to a stable state before
            # returning it to the pool.
            if connection:
                try:
                    connection.rollback()
                except Exception:
                    pass
            LOGGER.exception("Failed to execute query: %s", e)
            raise
        finally:
            # Make sure we always return the connection to
            # the pool.
            if connection:
                self.pg_pool.putconn(connection)

        return result

    def create_schema_in_database(self) -> None:
        """Create the visit sequence metadata schema in a database."""

        # Creation of the production schema should be
        # a one-off execution.
        assert "test" in self.metadata_db_schema

        # Make sure the schema does not already exist
        schema_test_query = sql.SQL(
            "SELECT EXISTS(SELECT * FROM information_schema.schemata WHERE schema_name = {});"
        ).format(sql.Placeholder("schema"))
        data = {"schema": self.metadata_db_schema}
        schema_exists_return = self.query(schema_test_query, data, return_result=True)
        schema_exists = schema_exists_return[0][0]
        if schema_exists:
            raise ValueError("Schema already exists.")

        creation_sql_file = Path(__file__).parent.parent.parent / "sql" / "make_vseqmeta.sql"
        with open(creation_sql_file, "r") as f:
            query_template = f.read()

        # Sanity check what we read.
        assert "CREATE SCHEMA" in query_template

        query = sql.SQL(query_template).format(
            sql.Identifier(self.metadata_db_schema), sql.Identifier(self.metadata_db_schema)
        )
        self.query(query, {}, commit=True, return_result=False)
        print("Created test database and schema ", self.metadata_db_schema)

    def pd_read_sql(
        self,
        query_template: str,
        sql_params: list[sql.Composable] | None = None,
        query_params: tuple | None = None,
    ) -> pd.DataFrame:
        """Execute a SQL query using the internal PostgreSQL connection pool
        and return the results as a pandas DataFrame.

        Parameters
        ----------
        query_template : `str`
            A query template.
        sql_params : `list` [`sql.Composable`]
            Elements that will be substituted into the template (e.g. table or
            schema identifiers) using ``psycopg2.sql.SQL.format``.
            To make type checkers happy, you may need to explictly declare
            variables passed as ``list[sql.Composable]``.
        query_params : `tuple`
            Positional parameters that will be bound by sqlalchemy.

        Returns
        -------
        df : `pd.DataFrame`
            The result set of the query, wrapped in a DataFrame.

        Notes
        -----
        This works around limitations in pandas when dealing with
        complex datatypes in raw postgresql connections.
        """

        if sql_params is None:
            sql_params = []
        assert isinstance(sql_params, list)

        if query_params is None:
            query_params = tuple([])
        assert isinstance(query_params, tuple)

        # Pandas sometimes has trouble with postgresql when working with it
        # directly, so interact with it by way of sqlalchemy.
        # But, we need to use postgresql's tools for inserting identifiers
        # like the schema name, followed by the sqlalchemy templating
        # for actual parameters.
        query = sql.SQL(query_template).format(*sql_params)
        conn = None
        try:
            conn = self.pg_pool.getconn()
            text_query = query.as_string(conn)
        finally:
            if conn:
                self.pg_pool.putconn(conn)

        with self.sa_engine.connect() as sa_conn:
            df = pd.read_sql(text_query, sa_conn, params=query_params)

        return df

    def record_visitseq_metadata(
        self,
        visits: pd.DataFrame,
        label: str,
        telescope: str = "simonyi",
        table: str = "visitseq",
        url: str | None = None,
        first_day_obs: date | int | str | None = None,
        last_day_obs: date | int | str | None = None,
        creation_time: Time | None = None,
    ) -> UUID:
        """Record metadata for a new visit sequence.

        Parameters
        ----------
        visits : `pd.DataFrame`
            A DataFrame of visits, with column names following those
            in consdb.
        label : `str`
            A label for the sequence.
        telescope : `str`
            The telescope used, either "simonyi" or "auxtel".
            Defaults to "simonyi".
        table : `str`
            The table in the metadata archive database in which
            to insert the entry: one of ``simulations``,
            ``completed``, ``mixedvisitseq``, or ``visitseq``.
        url : `str`, optional
            A URL for the file from thich the table of visits can
            be download.
        first_day_obs : `date` or `int` or `str`, optional
            The first night of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        last_day_obs : `date` or `int` or `str`, optional
            The last day of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        creation_time : `Time`, optional
            The time the sequence was created, defaults no now.

        Returns
        -------
        visitseq_uuid : `UUID`
            The UUID of the new visit sequence.

        Notes
        -----
        This method is not normally used externally, but may be
        occasionally needed to add non-standard sequences of visits.
        Standard simulations and completed sets of visits should be
        registered with the ``record_simulation_metadata`` and
        ``record_completed_metadata`` methods instead.
        """

        sha256 = compute_visits_sha256(visits)

        columns = [
            sql.Identifier("visitseq_sha256"),
            sql.Identifier("visitseq_label"),
            sql.Identifier("telescope"),
        ]
        data_placeholders = [
            sql.Placeholder("visitseq_sha256"),
            sql.Placeholder("visitseq_label"),
            sql.Placeholder("telescope"),
        ]
        data: Dict[str, Any] = {"visitseq_sha256": sha256, "visitseq_label": label, "telescope": telescope}

        if url is not None:
            columns.append(sql.Identifier("visitseq_url"))
            data_placeholders.append(sql.Placeholder("visitseq_url"))
            data["visitseq_url"] = url

        if first_day_obs is not None:
            columns.append(sql.Identifier("first_day_obs"))
            data_placeholders.append(sql.Placeholder("first_day_obs"))
            data["first_day_obs"] = dayobs_to_date(first_day_obs)

        if last_day_obs is not None:
            columns.append(sql.Identifier("last_day_obs"))
            data_placeholders.append(sql.Placeholder("last_day_obs"))
            data["last_day_obs"] = dayobs_to_date(last_day_obs)

        if creation_time is not None:
            assert not creation_time.masked
            assert creation_time.isscalar
            try:
                creation_datetime = creation_time.utc[0].datetime
            except AttributeError:
                # astropy Times, even those for which .isscalar
                # is true, can sometimes need to be indexed
                # by 0 to get a true scalar, and sometimes
                # cannot be.
                creation_utc = creation_time.utc
                assert isinstance(creation_utc, Time)
                creation_datetime = creation_utc.datetime
            columns.append(sql.Identifier("creation_time"))
            data_placeholders.append(sql.Placeholder("creation_time"))
            data["creation_time"] = creation_datetime

        query = sql.SQL("INSERT INTO {}.{} ({}) VALUES ({}) RETURNING visitseq_uuid").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Identifier(table),
            sql.SQL(", ").join(columns),
            sql.SQL(", ").join(data_placeholders),
        )

        result = self.query(query, data, commit=True)[0][0]
        return result

    def get_visitseq_metadata(
        self,
        visitseq_uuid: UUID,
        table: str = "visitseq",
    ) -> pd.Series:
        """Retrieve metadata for a visit sequence.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence to retrieve metadata for.
        table : `str`, optional
            The table in the metadata archive database in which to search.
            Must be one of "visitseq", "mixedvisitseq", "completed", or
            "simulations".
            Defaults to "visitseq".

        Returns
        -------
        visitseq : `pd.Series`
            A Pandas Series containing the metadata for the visit sequence.

        Raises
        ------
        ValueError
            If the table is not one of "visitseq", "mixedvisitseq",
            "completed", or "simulations".
        """
        if table not in {"visitseq", "mixedvisitseq", "completed", "simulations", "simulations_extra"}:
            raise ValueError()

        query_template = "SELECT * FROM {}.{} WHERE visitseq_uuid=%s"
        sql_params: list[sql.Composable] = [sql.Identifier(self.metadata_db_schema), sql.Identifier(table)]
        query_params = (visitseq_uuid,)
        visitseq = self.pd_read_sql(query_template, sql_params, query_params).iloc[0, :]
        return visitseq

    def set_visitseq_url(self, visitseq_uuid: UUID, visitseq_url: str) -> None:
        """Update the URL for the visits file for a visit sequence.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence to update.
        visitseq_url : `str`
            The new URL for the visit sequence.

        Raises
        ------
        `ValueError`
            If the table is not one of "visitseq", "mixedvisitseq",
            "completed", or "simulations".
        """
        table = self._find_visitseq_table(visitseq_uuid)
        query = sql.SQL("UPDATE {}.{} SET visitseq_url={} WHERE visitseq_uuid={} RETURNING *;").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Identifier(table),
            sql.Placeholder("visitseq_url"),
            sql.Placeholder("visitseq_uuid"),
        )
        data = {"visitseq_url": visitseq_url, "visitseq_uuid": visitseq_uuid}
        self.query(query, data, return_result=False, commit=True)

    def get_visitseq_url(self, visitseq_uuid: UUID) -> str:
        """Retrieve the URL for the file with the visits in a visit sequence.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence to search for.

        Returns
        -------
        url : `str`
            The URL for the visit sequence.

        Raises
        ------
        `ValueError`
            If no URL is found for the visit sequence.
        """
        query = sql.SQL("SELECT visitseq_url FROM {}.visitseq WHERE visitseq_uuid={}").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Placeholder("visitseq_uuid"),
        )
        data = {"visitseq_uuid": visitseq_uuid}
        response = self.query(query, data)
        if len(response) < 1:
            raise ValueError(f"No URL for {visitseq_uuid} found")
        assert len(response) == 1, f"Datatabase has too many visit sequinces with UUID={visitseq_uuid}"
        return response[0][0]

    def record_simulation_metadata(
        self,
        visits: pd.DataFrame,
        label: str,
        telescope: str = "simonyi",
        url: str | None = None,
        first_day_obs: date | int | str | None = None,
        last_day_obs: date | int | str | None = None,
        creation_time: Time | None = None,
        scheduler_version: str | None = None,
        config_url: str | None = None,
        conda_env_sha256: bytes | None = None,
        parent_visitseq_uuid: UUID | None = None,
        sim_runner_kwargs: dict | None = None,
        parent_last_day_obs: str | date | int | None = None,
    ) -> UUID:
        """Record metadata for a new simulation sequence.

        Parameters
        ----------
        visits : pd.DataFrame
            A DataFrame of visits, with column names following those
            in consdb.
        label : str
            A label for the sequence.
        telescope : `str`
            The telescope used, either "simonyi" or "auxtel".
            Defaults to "simonyi".
        url : `str`, optional
            A URL for the file from thich the table of visits can
            be download.
        first_day_obs : `date` or `int` or `str`, optional
            The first night of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        last_day_obs : `date` or `int` or `str`, optional
            The last day of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        creation_time : `Time`, optional
            The time the sequence was created, defaults to now.
        scheduler_version : `str`, optional
            Version of ``rubin_scheduler`` used to run the simulation,
            by default None.
        config_url : `str`, optional
            URL for the config script, perhaps on github,
            by default `None`.
        conda_env_sha256 : `bytes`, optional
            SHA256 hash of output of ``conda list --json``, by default None.
        parent_visitseq_uuid : `UUID`, optional
            UUID of visitseq loaded into scheduler before running,
            by default `None`.
        sim_runner_kwargs : `dict`, optional
            Arguments to ``sim_runner`` as a `dict`, by default `None`.
        parent_last_day_obs : `str`, optional
            day_obs of last visit loaded into scheduler before running,
            by default `None`.

        Returns
        -------
        visitseq_uuid : UUID
            The UUID of the new visit sequence.
        """
        # I would have preferred to use kwargs here, but was not
        # able to get type checking happy with it.
        visitseq_uuid = self.record_visitseq_metadata(
            visits,
            label,
            telescope=telescope,
            table="simulations",
            url=url,
            first_day_obs=first_day_obs,
            last_day_obs=last_day_obs,
            creation_time=creation_time,
        )

        def make_set_clause(column: str) -> sql.Composed:
            set_clause = sql.SQL("{} = {}").format(sql.Identifier(column), sql.Placeholder(column))
            return set_clause

        set_clauses = []
        data: Dict[str, Any] = {}
        if scheduler_version is not None:
            set_clauses.append(make_set_clause("scheduler_version"))
            data["scheduler_version"] = scheduler_version

        if config_url is not None:
            set_clauses.append(make_set_clause("config_url"))
            data["config_url"] = config_url

        if conda_env_sha256 is not None:
            set_clauses.append(make_set_clause("conda_env_sha256"))
            data["conda_env_sha256"] = conda_env_sha256

        if parent_visitseq_uuid is not None:
            set_clauses.append(make_set_clause("parent_visitseq_uuid"))
            data["parent_visitseq_uuid"] = str(parent_visitseq_uuid)

        if sim_runner_kwargs is not None:
            sim_runner_munged_kwargs = {}
            for key in sim_runner_kwargs:
                try:
                    # Throws a TypeError if it cannot convert it to json
                    json_result = json.dumps(sim_runner_kwargs[key])
                    json_failed = len(json_result) > JSON_DUMP_LIMIT
                except TypeError:
                    # Cannot convert to json, just store a string
                    # representation instead.
                    json_failed = True

                # if we can save an argument as serealized json in reasonable
                # space, do, otherwise store a (possibly truncated) result of
                # repr.
                sim_runner_munged_kwargs[key] = (
                    ("Not json serializable", repr(sim_runner_kwargs[key])[:JSON_DUMP_LIMIT])
                    if json_failed
                    else sim_runner_kwargs[key]
                )

            set_clauses.append(make_set_clause("sim_runner_kwargs"))
            data["sim_runner_kwargs"] = psycopg2.extras.Json(sim_runner_munged_kwargs)

        if parent_last_day_obs is not None:
            set_clauses.append(make_set_clause("parent_last_day_obs"))
            data["parent_last_day_obs"] = parent_last_day_obs

        num_columns_to_update = len(set_clauses)

        if num_columns_to_update > 0:
            query = sql.SQL("UPDATE {}.simulations SET {} WHERE visitseq_uuid={} RETURNING *;").format(
                sql.Identifier(self.metadata_db_schema),
                sql.SQL(", ").join(set_clauses),
                sql.Placeholder("visitseq_uuid"),
            )
            data["visitseq_uuid"] = visitseq_uuid

            conn = None
            try:
                conn = self.pg_pool.getconn()
                cursor = conn.cursor()
                cursor.execute(query, data)
                result = cursor.fetchall()

                # Be extra cautious, and check that everything looks
                # reasonable before commiting the update.
                assert len(result) == 1
                cursor.execute("COMMIT;")
            finally:
                if conn:
                    self.pg_pool.putconn(conn)

        return visitseq_uuid

    def update_visitseq_metadata(self, visitseq_uuid: UUID, field: str, value: object) -> None:
        """Update a single metadata field for a visit sequence.
        Automatically finds the table in which to make the update.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The unique identifier of the visit sequence to update.
        field : `str`
            The name of the column in the metadata table to modify.
        value : `object`
            The new value to assign to ``field``.  The type must be
            compatible with the column's database type.
        """
        table = self._find_visitseq_table(visitseq_uuid)
        query = sql.SQL("UPDATE {}.{} SET {}={} WHERE visitseq_uuid={};").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Identifier(table),
            sql.Identifier(field),
            sql.Placeholder("value"),
            sql.Placeholder("visitseq_uuid"),
        )
        data = {"value": value, "visitseq_uuid": visitseq_uuid}
        self.query(query, data, return_result=False, commit=True)

    def record_completed_metadata(
        self,
        visits: pd.DataFrame,
        label: str,
        telescope: str = "simonyi",
        url: str | None = None,
        first_day_obs: date | int | str | None = None,
        last_day_obs: date | int | str | None = None,
        creation_time: Time | None = None,
        query: str | None = None,
    ) -> UUID:
        """Record metadata for a sequence of visits queried from
        the consdb.

        Parameters
        ----------
        visits : `pd.DataFrame`
            A DataFrame of visits, with column names following those
            in consdb.
        label : `str`
            A label for the sequence.
        telescope : `str`
            The telescope used, either "simonyi" or "auxtel".
            Defaults to "simonyi".
        url : `str`, optional
            A URL for the file from thich the table of visits can
            be download.
        first_day_obs : `date` or `int` or `str`, optional
            The first night of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        last_day_obs : `date` or `int` or `str`, optional
            The last day of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        creation_time : `Time`, optional
            The time the sequence was created, defaults no now.
        query : `str`, optional
            The query to the consdb used to get the completed
            visits, by default None.

        Returns
        -------
        visitseq_uuid : `UUID`
            The UUID of the new visit sequence.
        """

        # I would have preferred to use kwargs here, but was not
        # able to get type checking happy with it.
        visitseq_uuid = self.record_visitseq_metadata(
            visits,
            label,
            telescope=telescope,
            table="completed",
            url=url,
            first_day_obs=first_day_obs,
            last_day_obs=last_day_obs,
            creation_time=creation_time,
        )

        if query is not None:
            composed_query = sql.SQL(
                "UPDATE {}.completed SET query={} WHERE visitseq_uuid={} RETURNING *"
            ).format(
                sql.Identifier(self.metadata_db_schema),
                sql.Placeholder("query"),
                sql.Placeholder("visitseq_uuid"),
            )
            data = {"query": query, "visitseq_uuid": visitseq_uuid}

            conn = None
            try:
                conn = self.pg_pool.getconn()
                cursor = conn.cursor()
                cursor.execute(composed_query, data)
                result = cursor.fetchall()

                # Be extra cautious, and check that everything looks
                # reasonable before commiting the update.
                assert len(result) == 1
                cursor.execute("COMMIT;")
            finally:
                if conn:
                    self.pg_pool.putconn(conn)

        return visitseq_uuid

    def record_mixed_metadata(
        self,
        visits: pd.DataFrame,
        label: str,
        last_early_day_obs: date | int | str,
        first_late_day_obs: date | int | str,
        early_parent_uuid: UUID,
        late_parent_uuid: UUID,
        telescope: str = "simonyi",
        url: str | None = None,
        first_day_obs: date | int | str | None = None,
        last_day_obs: date | int | str | None = None,
        creation_time: Time | None = None,
    ) -> UUID:
        """Record metadata for a new mixed visit sequence.

        Parameters
        ----------
        visits : `pd.DataFrame`
            A DataFrame of visits, with column names following those
            in consdb.
        label : `str`
            A label for the sequence.
        last_early_day_obs : `date` or `int` or `str`
            The last day obs of the early parent.
        first_late_day_obs : `date` or `int` or `str`
            The first day obs of the late parent.
        early_parent_uuid : `UUID`
            The UUID of the early parent.
        late_parent_uuid : `UUID`
            The UUID of the late parent.
        telescope : `str`, optional
            The telescope used, either "simonyi" or "auxtel".
            Defaults to "simonyi".
        first_day_obs : `date` or `int` or `str` or `None`, optional
            The first night of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        last_day_obs : `date` or `int` or `str` or `None`, optional
            The last day of observations, defined according
            to SITCOMTN-032 (UTC-12hrs).
        creation_time : `Time` or `None`, optional
            The time the sequence was created, defaults to now.

        Returns
        -------
        visitseq_uuid : `UUID`
            The UUID of the new visit sequence.
        """
        visitseq_uuid = self.record_visitseq_metadata(
            visits,
            label,
            telescope=telescope,
            table="mixedvisitseq",
            url=url,
            first_day_obs=first_day_obs,
            last_day_obs=last_day_obs,
            creation_time=creation_time,
        )

        update_query = sql.SQL("""UPDATE {}.mixedvisitseq
                   SET last_early_day_obs={},
                       first_late_day_obs={},
                       early_parent_uuid={},
                       late_parent_uuid={}\
                   WHERE visitseq_uuid={} RETURNING *
                """).format(
            sql.Identifier(self.metadata_db_schema),
            sql.Placeholder("last_early_day_obs"),
            sql.Placeholder("first_late_day_obs"),
            sql.Placeholder("early_parent_uuid"),
            sql.Placeholder("late_parent_uuid"),
            sql.Placeholder("visitseq_uuid"),
        )

        data = {
            "last_early_day_obs": dayobs_to_date(last_early_day_obs),
            "first_late_day_obs": dayobs_to_date(first_late_day_obs),
            "early_parent_uuid": early_parent_uuid,
            "late_parent_uuid": late_parent_uuid,
            "visitseq_uuid": visitseq_uuid,
        }

        conn = None
        try:
            conn = self.pg_pool.getconn()
            cursor = conn.cursor()
            cursor.execute(update_query, data)
            result = cursor.fetchall()

            # Be extra cautious, and check that everything looks
            # reasonable before commiting the update.
            assert len(result) == 1
            cursor.execute("COMMIT;")
        finally:
            if conn:
                self.pg_pool.putconn(conn)

        return visitseq_uuid

    def is_tagged(self, visitseq_uuid: UUID, tag: str) -> bool:
        """Return whether a visit sequence is tagged with a given tag.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence.
        tag : `str`
            The tag to check for.

        Returns
        -------
        is_tagged : `bool`
            True if the visit sequence is tagged with the given tag,
            False otherwise.
        """
        query = sql.SQL("SELECT COUNT(*)>=1 FROM {}.tags WHERE visitseq_uuid={} AND TAG={}").format(
            sql.Identifier(self.metadata_db_schema), sql.Placeholder("visitseq_uuid"), sql.Placeholder("tag")
        )
        data = {"visitseq_uuid": visitseq_uuid, "tag": tag}
        is_tagged = self.query(query, data)[0][0] > 0
        return is_tagged

    def tag(self, visitseq_uuid: UUID, *tags: str) -> None:
        """Tag a visit sequence with one or more tags.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence.
        *tags : `str`
            One or more tags to add to the visit sequence.
        """
        query = sql.SQL("INSERT INTO {}.tags (visitseq_uuid, tag) VALUES ({}, {})").format(
            sql.Identifier(self.metadata_db_schema), sql.Placeholder("visitseq_uuid"), sql.Placeholder("tag")
        )
        data: Dict[str, Any] = {"visitseq_uuid": visitseq_uuid}
        for tag in tags:
            if not self.is_tagged(visitseq_uuid, tag):
                data["tag"] = tag
                self.query(query, data, commit=True, return_result=False)

    def untag(self, visitseq_uuid: UUID, tag: str) -> None:
        """Untag a visit sequence with a given tag.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence.
        tag : `str`
            The tag to remove.

        Raises
        ------
        `ValueError`
            If the visit sequence is not tagged with the given tag.
        """
        if not self.is_tagged(visitseq_uuid, tag):
            raise ValueError(f"Visit sequence {visitseq_uuid} is not tagged with {tag}")

        query = sql.SQL("DELETE FROM {}.tags WHERE visitseq_uuid={} AND tag={}").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Placeholder("visitseq_uuid"),
            sql.Placeholder("tag"),
        )
        data = {"visitseq_uuid": visitseq_uuid, "tag": tag}
        self.query(query, data, commit=True, return_result=False)

    def comment(self, visitseq_uuid: UUID, comment: str, author: str | None = None) -> None:
        """Attach a comment to a visit sequence in the
        metadata database..

        Parameters
        ----------
        visitseq_uuid: `uuid.UUID`
            The UUID of the visit sequence to comment on.
        comment : `str`
            The comment to attach.
        author : `str`, optional
            The author of the comment.
            Defaults to `None`.
        """
        comment_utc = Time.now().utc
        # mypy cannot figure out that comment_utc cannot be masked
        # so assert that it is not as a hint to it.
        assert isinstance(comment_utc, Time)
        comment_time = comment_utc.datetime
        query = ""
        data = {"visitseq_uuid": visitseq_uuid, "comment_time": comment_time, "comment": comment}
        if author is None:
            query = sql.SQL(
                "INSERT INTO {}.comments (visitseq_uuid, comment_time, comment) VALUES ({}, {}, {})"
            ).format(
                sql.Identifier(self.metadata_db_schema),
                sql.Placeholder("visitseq_uuid"),
                sql.Placeholder("comment_time"),
                sql.Placeholder("comment"),
            )
        else:
            query = sql.SQL(
                "INSERT INTO {}.comments (visitseq_uuid, comment_time, author, comment)"
                + " VALUES ({}, {}, {}, {})"
            ).format(
                sql.Identifier(self.metadata_db_schema),
                sql.Placeholder("visitseq_uuid"),
                sql.Placeholder("comment_time"),
                sql.Placeholder("author"),
                sql.Placeholder("comment"),
            )
            data["author"] = author

        self.query(query, data, commit=True, return_result=False)

    def get_comments(self, visitseq_uuid: UUID) -> pd.DataFrame:
        """Retrieve all comments attached to a specific visit sequence.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence whose comments should be fetched.

        Returns
        -------
        comments: `pd.DataFrame`
            A DataFrame containing the comments for the requested visit
            sequence. The DataFrame has the following columns:

            ``"visitseq_uuid"``
                The ID of the visit sequence.
            ``"comment_time"``
                Timestamp of the comment.
            ``"author"``
                Name of the user who added the comment; may be ``None``.
            ``"comment"``
                The comment text.

            If the visit sequence has no comments, an empty DataFrame with the
            above column names is returned.

        """
        query_template = "SELECT * FROM {}.comments WHERE visitseq_uuid = %s"
        sql_params: list[sql.Composable] = [sql.Identifier(self.metadata_db_schema)]
        query_params = (visitseq_uuid,)
        comments = self.pd_read_sql(query_template, sql_params, query_params)

        # sqlalchemy seems to mess up the dtypes. Fix them.
        comments["comment"] = comments["comment"].astype("string")

        return comments

    def register_file(
        self,
        visitseq_uuid: UUID,
        file_type: str,
        file_sha256: bytes,
        location: str | ResourcePath,
        update: bool = False,
    ) -> None:
        """Register a file in the visit sequence metadata database.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence with which the file should be
            associated.
        file_type : `str`
            The handle for the file type.
            The ``"visits"`` file type is not allowed
            because visit tables are recorded using `set_visitseq_url`.
        file_sha256 : `bytes`
            The hash of the file contents.
        location : `str` or `lsst.resources.ResourcePath`
            The location of the file to be registered.
        update : `bool`, optional
            If ``True`` and a record for the same ``visitseq_uuid`` and
            ``file_type`` already exists, the existing row will be updated with
            the new SHA‑256 hash and URL.  If ``False`` and a record already
            exists a ``ValueError`` is raised.

        Raises
        ------
        `ValueError`
            If ``file_type`` is ``"visits"`` (use `set_visitseq_url` instead),
            or if ``update`` is ``False`` and a record already exists for the
            given ``visitseq_uuid`` and ``file_type``.
        `TypeError`
            If ``location`` is neither a string nor a `ResourcePath`.
        """
        if file_type == "visits":
            raise ValueError("Use set_visitseq_url to register sets of visits themselves")

        file_url: str = ""
        match location:
            case ResourcePath():
                file_url = location.geturl()
            case str():
                file_url = location
            case _:
                raise ValueError("Unrecognised location type")

        # Check whether the file is already registered
        already_exists = False
        try:
            self.get_file_url(visitseq_uuid, file_type)
            already_exists = True
        except ValueError:
            already_exists = False

        if already_exists:
            if not update:
                raise ValueError("A file of type {file_type} is already registered for {visitseq_uuid}")

            query = sql.SQL(
                "UPDATE {}.files SET file_sha256={}, file_url={} WHERE visitseq_uuid={} AND file_type={}"
            ).format(
                sql.Identifier(self.metadata_db_schema),
                sql.Placeholder("file_sha256"),
                sql.Placeholder("file_url"),
                sql.Placeholder("visitseq_uuid"),
                sql.Placeholder("file_type"),
            )
        else:
            query = sql.SQL(
                "INSERT INTO {}.files (visitseq_uuid, file_type, file_sha256, file_url)"
                + " VALUES ({}, {}, {}, {})"
            ).format(
                sql.Identifier(self.metadata_db_schema),
                sql.Placeholder("visitseq_uuid"),
                sql.Placeholder("file_type"),
                sql.Placeholder("file_sha256"),
                sql.Placeholder("file_url"),
            )

        data = {
            "visitseq_uuid": visitseq_uuid,
            "file_type": file_type,
            "file_sha256": file_sha256,
            "file_url": file_url,
        }

        self.query(query, data, commit=True, return_result=False)

    def get_file_url(self, visitseq_uuid: UUID, file_type: str) -> str:
        """Return the URL for a registered file of a given type and
        visit sequence UUID.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence with which the desired
            file is associated.
        file_type : `str`
            Handle for the file type.

        Returns
        -------
        url : `str`
            The URL pointing to the requested file.
        """
        if file_type == "visits":
            # It's as easy to just do it as it is to raise
            # an exception
            return self.get_visitseq_url(visitseq_uuid)

        query = sql.SQL("SELECT file_url FROM {}.files WHERE visitseq_uuid={} AND file_type={}").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Placeholder("visitseq_uuid"),
            sql.Placeholder("file_type"),
        )
        data = {"visitseq_uuid": visitseq_uuid, "file_type": file_type}
        result = self.query(query, data, return_result=True)
        if len(result) < 1:
            raise ValueError(f"No URLs found for {file_type} for visitseq {visitseq_uuid}")
        if len(result) > 1:
            raise ValueError(f"Too many URLs found for {file_type} for visitseq {visitseq_uuid}!")

        url = result[0][0]
        return url

    def get_file_sha256(self, visitseq_uuid: UUID, file_type: str) -> bytes:
        """Retrieve the SHA‑256 digest of a registered file.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence the file is associated with.
        file_type : `str`
            Handle for the file type.

        Returns
        -------
        file_sha256 : `bytes`
            The raw SHA‑256 digest of the file contents.  The value
            is returned as a 32‑byte object (the output of
            ``hashlib.sha256``).
        """

        query = sql.SQL("SELECT file_sha256 FROM {}.files WHERE visitseq_uuid={} AND file_type={}").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Placeholder("visitseq_uuid"),
            sql.Placeholder("file_type"),
        )
        data = {"visitseq_uuid": visitseq_uuid, "file_type": file_type}
        result = self.query(query, data, return_result=True)
        if len(result) < 1:
            raise ValueError(f"No URLs found for {file_type} for visitseq {visitseq_uuid}")
        if len(result) > 1:
            raise ValueError(f"Too many URLs found for {file_type} for visitseq {visitseq_uuid}!")

        file_sha256 = result[0][0].tobytes()
        return file_sha256

    def insert_nightly_stats(self, visitseq_uuid: UUID, nightly_stats: pd.DataFrame) -> None:
        """Instert by-night statistics into associoted with
        a visit sequence into the visit sequence metadata database.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence whose statistics are being
            recorded.
        stats_df : `pd.DataFrame`
            A DataFrame containing the nightly statistics that were
            written to the database.  The columns are:

            ``"day_obs"``
                The observation day.
            ``"value_name"``
                The value for which statistics were computed.
            ``"p05"``
                5th percentile.
            ``"q1"``
                25th percentile (first quartile).
            ``"median"``
                50th percentile (median).
            ``"q3"``
                75th percentile (third quartile).
            ``"p95"``
                95th percentile.
            ``"visitseq_uuid"``
                The UUID of the sequence.
            ``"accumulated"``
                Always ``False`` for data written by
                this method.
        """
        # We are adding a column, but do not want
        # to change the original df, so
        # create a copy.
        stats_df = nightly_stats.copy()
        stats_df["visitseq_uuid"] = visitseq_uuid

        # pandas to_sql fails with the native psycopg2, only
        # works with sqlalchemy
        engine = sqlalchemy.create_engine("postgresql+psycopg2://", creator=self.pg_pool.getconn)
        with engine.connect() as conn:
            num_rows_added = stats_df.to_sql(
                "nightly_stats", conn, schema=self.metadata_db_schema, if_exists="append", index=False
            )
            assert num_rows_added == len(stats_df)

    def query_nightly_stats(self, visitseq_uuid: UUID) -> pd.DataFrame:
        """Query the visit sequence metadata database for
         nightly statistics.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence whose statistics are being
            recorded.

        Returns
        -------
        stats_df : `pd.DataFrame`
            A table of nightly statistics.
        """
        query_template = "SELECT * FROM {}.nightly_stats WHERE visitseq_uuid=%s"
        sql_params: list[sql.Composable] = [sql.Identifier(self.metadata_db_schema)]
        query_params = (visitseq_uuid,)
        stats_df = self.pd_read_sql(query_template, sql_params, query_params)
        return stats_df

    def conda_env_is_saved(self, conda_env_hash: bytes) -> bool:
        """Check whether a conda environment with the given hash is already
        stored in the database.

        Parameters
        ----------
        conda_env_hash : `bytes`
            The SHA‑256 digest of the conda environment JSON representation.
            This value is expected to be the same output returned by
            `compute_conda_env`.

        Returns
        -------
        env_exists : `bool`
            ``True`` if a row with the supplied hash exists in the
            ``conda_env`` table; ``False`` otherwise.
        """
        query = sql.SQL("SELECT EXISTS(SELECT true from {}.conda_env WHERE conda_env_hash={})").format(
            sql.Identifier(self.metadata_db_schema), sql.Placeholder("conda_env_hash")
        )
        data = {"conda_env_hash": conda_env_hash}
        result = self.query(query, data, commit=False, return_result=True)
        env_exists = result[0][0]
        assert isinstance(env_exists, bool)
        return env_exists

    def record_conda_env(
        self, conda_env_hash: bytes | None = None, conda_env_json: str | None = None
    ) -> bytes:
        """Record the current Conda environment in the database.

        Parameters
        ----------
        conda_env_hash : `bytes` or `None`
            The SHA‑256 digest of the Conda environment JSON
            representation. ``None`` (the default)
            uses the current conda environment.
        conda_env_json : `str` or `None`
            The json representing the conda environment.
            ``None`` (the default) uses the current
            conda environment.
        """
        if conda_env_hash is None != conda_env_json is None:
            raise ValueError("conda_env_hash and conda_env_json must both be set, or neither.")

        if conda_env_hash is None:
            conda_env_hash, conda_env_json = compute_conda_env()

        assert isinstance(conda_env_hash, bytes)
        assert isinstance(conda_env_json, str)

        if self.conda_env_is_saved(conda_env_hash):
            warnings.warn("Conda env with hash already exists, not saving again.")
            return conda_env_hash

        query = sql.SQL("INSERT INTO {}.conda_env (conda_env_hash, conda_env) VALUES ({}, {})").format(
            sql.Identifier(self.metadata_db_schema),
            sql.Placeholder("conda_env_hash"),
            sql.Placeholder("conda_env"),
        )
        data = {"conda_env_hash": conda_env_hash, "conda_env": conda_env_json}
        self.query(query, data, commit=True, return_result=False)
        return conda_env_hash

    def _find_visitseq_table(self, visitseq_uuid: UUID) -> str:
        # Figure out which child table the visitseq_uuid is in by
        # querying each in turn.

        for table in ("mixedvisitseq", "completed", "simulations", "visitseq"):
            query = sql.SQL("SELECT COUNT(*) FROM {}.{} WHERE visitseq_uuid={};").format(
                sql.Identifier(self.metadata_db_schema),
                sql.Identifier(table),
                sql.Placeholder("visitseq_uuid"),
            )
            data = {"visitseq_uuid": visitseq_uuid}
            result = self.query(query, data)
            assert isinstance(result[0][0], int)
            num_rows: int = result[0][0]
            if num_rows > 0:
                return table

        raise ValueError("Visit sequence not found.")

    def sims_on_nights(
        self,
        first_day_obs: str | date | int | None = None,
        last_day_obs: str | date | int | None = None,
        tags: tuple[str, ...] = ("prenight", "ideal", "nominal"),
        telescope: str = "simonyi",
        max_simulation_age: int = 2,
    ) -> pd.DataFrame:
        """Return a table of simulations that cover a given night range.

        Parameters
        ----------
        first_day_obs : `str` | `date` | `int` | `None`, optional
            The first observation day of the window for which simulations are
            requested.  It can be specified as a `datetime.date` object, an
            ISO‑formatted string (e.g. ``'2025-12-01'``), or an integer in
            SMTN‑032 format (e.g. ``20251201``).  If ``None`` (the default),
            the method uses the current dayobs.
        last_day_obs : `str` | `date` | `int` | `None`, optional
            The last observation day of the window.  It accepts the same
            types as ``first_day_obs``.  If ``None`` the same value as
            ``first_day_obs`` is used, giving a single‑night query.
        tags : `tuple[str]`, optional
            A sequence of tags that must be present on the simulation record.
            The default tags are ``("prenight", "ideal", "nominal")``.  If
            an empty tuple is supplied, the tag test is omitted.
        telescope : `str`, optional
            The telescope simulated.  The default is ``simonyi``.
        max_simulation_age : `int`, optional
            The maximum age of a simulation in days.
            The default is 2 days.

        Returns
        -------
        vseqs : `pd.DataFrame`
            A table of metadata for matching simulations.
        """

        if first_day_obs is None:
            first_day_obs = datetime.now(timezone(timedelta(hours=-12))).date()
        if last_day_obs is None:
            last_day_obs = first_day_obs

        first_day_obs = dayobs_to_date(first_day_obs)
        last_day_obs = dayobs_to_date(last_day_obs)

        assert isinstance(first_day_obs, date)
        assert isinstance(last_day_obs, date)

        if len(tags) > 0:
            tags_json = json.dumps(list(tags))
            query_template = """
            SELECT *
            FROM {}.simulations_extra AS s
            WHERE %s BETWEEN first_day_obs AND last_day_obs
                AND %s BETWEEN first_day_obs AND last_day_obs
                AND telescope = %s
                AND tags @> %s::JSONB
                AND creation_time >= NOW() - INTERVAL '%s days'
            """
            query_params: Tuple = (first_day_obs, last_day_obs, telescope, tags_json, max_simulation_age)
        else:
            query_template = """
            SELECT *
            FROM {}.simulations_extra AS s
            WHERE %s BETWEEN first_day_obs AND last_day_obs
                AND %s BETWEEN first_day_obs AND last_day_obs
                AND telescope = %
                AND creation_time >= NOW() - INTERVAL '%s days'
            """
            query_params = (first_day_obs, last_day_obs, telescope, max_simulation_age)

        vseqs = self.pd_read_sql(
            query_template,
            [sql.Identifier(self.metadata_db_schema)],
            query_params,
        )
        return vseqs

    def sims_on_night_with_stats(
        self,
        day_obs: str | date | int | None = None,
        tags: tuple[str, ...] = ("prenight", "ideal", "nominal"),
        telescope: str = "simonyi",
        max_simulation_age: int = 2,
    ) -> pd.DataFrame:
        """Return a table of simulations that cover a given night range.

        Parameters
        ----------
        day_obs : `str` | `date` | `int` | `None`, optional
            The day_obs for which to get simulations.
        tags : `tuple[str]`, optional
            A sequence of tags that must be present on the simulation record.
            The default tags are ``("prenight", "ideal", "nominal")``.  If
            an empty tuple is supplied, the tag test is omitted.
        telescope : `str`, optional
            The telescope simulated.  The default is ``simonyi``.
        max_simulation_age : `int`, optional
            The maximum age of a simulation in days.
            The default is 2 days.

        Returns
        -------
        vseqs : `pd.DataFrame`
            A table of metadata for matching simulations.
        """

        if day_obs is None:
            day_obs = datetime.now(timezone(timedelta(hours=-12))).date()

        day_obs = dayobs_to_date(day_obs)
        assert isinstance(day_obs, date)

        if len(tags) > 0:
            tags_json = json.dumps(list(tags))
            query_template = """
            WITH aggstats AS (
                SELECT
                    visitseq_uuid,
                    JSONB_OBJECT_AGG(
                        value_name,
                        TO_JSONB(ns) - 'value_name' - 'visitseq_uuid' - 'day_obs'
                    ) AS stats
                FROM {}.nightly_stats AS ns
                WHERE ns.day_obs = %s
                GROUP BY ns.visitseq_uuid
            )
            SELECT s.*, ns.stats
            FROM {}.simulations_extra AS s
            LEFT JOIN aggstats AS ns ON s.visitseq_uuid=ns.visitseq_uuid
            WHERE %s BETWEEN first_day_obs AND last_day_obs
                AND telescope = %s
                AND tags @> %s::JSONB
                AND creation_time >= NOW() - INTERVAL '%s days'
            """
            query_params: Tuple = (day_obs, day_obs, telescope, tags_json, max_simulation_age)
        else:
            query_template = """
            WITH aggstats AS (
                SELECT
                    visitseq_uuid,
                    JSONB_OBJECT_AGG(
                        value_name,
                        TO_JSONB(ns) - 'value_name' - 'visitseq_uuid' - 'day_obs'
                    ) AS stats
                FROM {}.nightly_stats AS ns
                WHERE ns.day_obs = %s
                GROUP BY ns.visitseq_uuid
            )
            SELECT s.*, ns.stats
            FROM {}.simulations_extra AS s
            LEFT JOIN aggstats AS ns ON s.visitseq_uuid=ns.visitseq_uuid
            WHERE %s BETWEEN first_day_obs AND last_day_obs
                AND telescope = %
                AND creation_time >= NOW() - INTERVAL '%s days'
            """
            query_params = (day_obs, day_obs, telescope, max_simulation_age)

        vseqs = self.pd_read_sql(
            query_template,
            [sql.Identifier(self.metadata_db_schema), sql.Identifier(self.metadata_db_schema)],
            query_params,
        )
        return vseqs

    def import_sim_from_yaml(self, metadata_yaml: str, archive_base: str | None = None) -> UUID:
        """Import a simulation from a YAML metadata description.

        Parameters
        ----------
        metadata_yaml : `str`
            YAML string with simulation metadata.
        archive_base : `str` or `None`
            Base location of the archive to which to copy the visit
            sequence. If ``None``, the visits will not be copied.
            Defaults to ``None``.

        Returns
        -------
        visitseq_uuid : `UUID`
            The UUID of the newly registered visit sequence.
        """

        sim_archive_metadata = yaml.safe_load(metadata_yaml)

        if "label" not in sim_archive_metadata:
            raise ValueError("Metadata yaml must include a label")

        if "files" not in sim_archive_metadata or "observations" not in sim_archive_metadata["files"]:
            raise ValueError("Metadata yaml must include the observations file")

        label = sim_archive_metadata["label"]
        record_sim_kwargs = {"label": label}

        # The prenight labels usually end in the creation time.
        if " ".join(label.split()[-3:-1]) == "run at":
            maybe_date_str = label.split()[-1]
            try:
                record_sim_kwargs["creation_time"] = Time(datetime.fromisoformat(maybe_date_str))
            except ValueError:
                pass

        for key in ["scheduler_version", "telescope"]:
            if key in sim_archive_metadata:
                record_sim_kwargs[key] = sim_archive_metadata[key]

        for side in ["first", "last"]:
            try:
                record_sim_kwargs[f"{side}_day_obs"] = sim_archive_metadata["simulated_dates"][side]
            except KeyError:
                pass

        # Get the visits hdf5 file
        with TemporaryDirectory() as temp_dir:
            visits_h5_fname = Path(temp_dir) / "visits.h5"
            obs_rp = ResourcePath(sim_archive_metadata["files"]["observations"]["url"])
            with obs_rp.as_local() as obs_local_rp:
                opsimdb_to_hdf5(obs_local_rp.ospath, visits_h5_fname)

            record_sim_kwargs["visits"] = pd.read_hdf(visits_h5_fname, "observations")

            sim_uuid = self.record_simulation_metadata(**record_sim_kwargs)

            if archive_base is not None:
                # Import here to avoid circular imports
                from .vseqarchive import add_file

                add_file(self, sim_uuid, visits_h5_fname, "visits", archive_base)

        if "files" in sim_archive_metadata:
            for file_type in sim_archive_metadata["files"]:
                file_rp = ResourcePath(sim_archive_metadata["files"][file_type]["url"])
                file_sha256 = bytes.fromhex(hashlib.sha256(file_rp.read()).hexdigest())
                self.register_file(sim_uuid, file_type, file_sha256, file_rp)

        if "tags" in sim_archive_metadata:
            valid_tags = [t for t in sim_archive_metadata["tags"] if isinstance(t, str)]
            self.tag(sim_uuid, *valid_tags)

        return sim_uuid

    def import_sim_from_prototype_sim_archive(
        self, archive_base: str, sim_date: str | date, sim_index: str | int, proto_sim_archive_uri: str
    ) -> UUID:
        """Import a simulation metadata from the prototype achive.

        Parameters
        ----------
        archive_base : `str`
            Base location where the new archive entry will be written
            (e.g. an S3 bucket or a local directory).  The method
            will write and hdf5 with visits to this location and
            register the resulting URL in the database.
        sim_date : `str` or `datetime.date`
            The ISO‑formatted date (e.g. ``'2025-03-12'``) or a
            ``date`` object that identifies (with the sim_index)
            a simulation in the prototype archive.
        sim_index : `str` or `int`
            The index of the simulation which completes the
            identification of the simulation in the prototype.
        proto_sim_archive_uri : `str`
            Base URI of the prototype simulation archive.  By default
            this is ``'s3://rubin:rubin-scheduler-prenight/opsim/'``.

        Returns
        -------
        visitseq_uuid : `UUID`
            The UUID of the visit sequence that was created and
            registered.
        """

        metadata_yaml = metadata_yaml_from_prototype(sim_date, sim_index, proto_sim_archive_uri)
        sim_uuid = self.import_sim_from_yaml(metadata_yaml, archive_base)
        self.tag(sim_uuid, "from_prototype_sim_archive")

        sim_date_str = sim_date if isinstance(sim_date, str) else sim_date.isoformat()
        self.comment(sim_uuid, f"Imported from prototype sim_archive {sim_date_str}, {sim_index}")
        return sim_uuid

    def sim_metadata_yaml(self, visitseq_uuid: UUID) -> str:
        """Return a YAML representation of a simulation's metadata.

        Parameters
        ----------
        visitseq_uuid : `UUID`
            The unique identifier of the visit sequence for which
            metadata should be exported.

        Returns
        -------
        md_yaml: `str`
            yaml representation of (some of) the simulation metadata
            in the metadata database.
        """

        metadata_seq = self.get_visitseq_metadata(visitseq_uuid, "simulations_extra")

        metadata: dict = {"uuid": str(metadata_seq.visitseq_uuid), "label": str(metadata_seq.visitseq_label)}
        for key in ["telescope", "tags", "scheduler_version"]:
            if metadata_seq[key] is not None:
                metadata[key] = metadata_seq[key]

        if metadata_seq["creation_time"] is not None:
            metadata["creation_time"] = metadata_seq["creation_time"].isoformat()

        if metadata_seq.first_day_obs is not None or metadata_seq.last_day_obs is not None:
            metadata["simulation_dates"] = {}
            if metadata_seq.first_day_obs is not None:
                metadata["simulation_dates"]["first"] = metadata_seq.first_day_obs
            if metadata_seq.last_day_obs is not None:
                metadata["simulation_dates"]["last"] = metadata_seq.last_day_obs

        if metadata_seq.files is not None:
            metadata["files"] = {}
            for file_type in metadata_seq.files:
                metadata["files"][file_type] = {
                    "url": metadata_seq.files[file_type],
                    "name": ResourcePath(metadata_seq.files[file_type]).basename(),
                }

        metadata_yaml = yaml.dump(metadata)
        return metadata_yaml
