from __future__ import annotations

import builtins
import inspect
import os
import re
import sys
from collections import defaultdict
from itertools import chain
from pathlib import PurePosixPath
from typing import (
    TYPE_CHECKING,
    Any,
    Literal,
    NamedTuple,
    TypeVar,
    Union,
    overload,
)

import dj_database_url
import lamindb_setup as ln_setup
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import IntegrityError, ProgrammingError, connections, models, transaction
from django.db.models import CASCADE, PROTECT, Field, Manager, QuerySet
from django.db.models import ForeignKey as django_ForeignKey
from django.db.models.base import ModelBase
from django.db.models.fields.related import (
    ManyToManyField,
    ManyToManyRel,
    ManyToOneRel,
)
from django.db.models.functions import Lower
from lamin_utils import colors, logger
from lamindb_setup import settings as setup_settings
from lamindb_setup._connect_instance import (
    get_owner_name_from_identifier,
    load_instance_settings,
    update_db_using_local,
)
from lamindb_setup.core._docs import doc_args
from lamindb_setup.core._hub_core import connect_instance_hub
from lamindb_setup.core._settings_store import instance_settings_file
from lamindb_setup.core.django import DBToken, db_token_manager
from lamindb_setup.core.upath import extract_suffix_from_path

from lamindb.base import deprecated

from ..base.fields import (
    BooleanField,
    CharField,
    DateTimeField,
    ForeignKey,
    JSONField,
    TextField,
)
from ..base.ids import base62_12
from ..base.types import FieldAttr, StrField
from ..errors import (
    FieldValidationError,
    InvalidArgument,
    NoWriteAccess,
    SQLRecordNameChangeIntegrityError,
    ValidationError,
)
from ._is_versioned import IsVersioned
from .query_manager import QueryManager, _lookup, _search

if TYPE_CHECKING:
    from datetime import datetime

    import pandas as pd

    from .artifact import Artifact
    from .blocks import Block
    from .run import Run, User
    from .transform import Transform


T = TypeVar("T", bound="SQLRecord")
IPYTHON = getattr(builtins, "__IPYTHON__", False)


# -------------------------------------------------------------------------------------
# A note on required fields at the SQLRecord level
#
# As Django does most of its validation on the Form-level, it doesn't offer functionality
# for validating the integrity of an SQLRecord object upon instantation (similar to pydantic)
#
# For required fields, we define them as commonly done on the SQL level together
# with a validator in SQLRecord (validate_required_fields)
#
# This goes against the Django convention, but goes with the SQLModel convention
# (Optional fields can be null on the SQL level, non-optional fields cannot)
#
# Due to Django's convention where CharFieldAttr has pre-configured (null=False, default=""), marking
# a required field necessitates passing `default=None`. Without the validator it would trigger
# an error at the SQL-level, with it, it triggers it at instantiation

# -------------------------------------------------------------------------------------
# A note on class and instance methods of core SQLRecord
#
# All of these are defined and tested within lamindb, in files starting with _{orm_name}.py

# -------------------------------------------------------------------------------------
# A note on maximal lengths of char fields
#
# 100 characters:
#     "Raindrops pitter-pattered on the windowpane, blurring the"
#     "city lights outside, curled up with a mug."
# A good maximal length for a name (title).
#
# 150 characters: We choose this for name maximal length because some users like long names.
#
# 255 characters:
#     "In creating a precise 255-character paragraph, one engages in"
#     "a dance of words, where clarity meets brevity. Every syllable counts,"
#     "illustrating the skill in compact expression, ensuring the essence of the"
#     "message shines through within the exacting limit."


class IsLink:
    pass


def deferred_attribute__repr__(self):
    return f"FieldAttr({self.field.model.__name__}.{self.field.name})"


def unique_constraint_error_in_error_message(error_msg: str) -> bool:
    """Check if the error message indicates a unique constraint violation."""
    return (
        "UNIQUE constraint failed" in error_msg  # SQLite
        or "duplicate key value violates unique constraint" in error_msg  # Postgre
    )


FieldAttr.__repr__ = deferred_attribute__repr__  # type: ignore


class ValidateFields:
    pass


def is_approx_pascal_case(s):
    """Check if the last component of a dotted string is in PascalCase.

    Args:
        s (str): The string to check

    Returns:
        bool: True if the last component is in PascalCase

    Raises:
        ValueError: If the last component doesn't start with a capital letter
    """
    if "[" in s:  # this is because we allow types of form 'script[test_script.py]'
        return True
    last_component = s.split(".")[-1]

    if not last_component[0].isupper():
        raise ValueError(
            f"'{last_component}' should start with a capital letter given you're defining a type"
        )

    return True


def init_self_from_db(self: SQLRecord, existing_record: SQLRecord):
    new_args = [
        getattr(existing_record, field.attname) for field in self._meta.concrete_fields
    ]
    super(self.__class__, self).__init__(*new_args)
    self._state.adding = False  # mimic from_db
    self._state.db = "default"


def update_attributes(record: SQLRecord, attributes: dict[str, str]):
    for key, value in attributes.items():
        if getattr(record, key) != value and value is not None:
            if key not in {"uid", "dtype", "otype", "hash"}:
                logger.warning(f"updated {key} from {getattr(record, key)} to {value}")
                setattr(record, key, value)
            else:
                hash_message = (
                    "recomputing on .save()"
                    if key == "hash"
                    else f"keeping {getattr(record, key)}"
                )
                logger.debug(
                    f"ignoring tentative value {value} for {key}, {hash_message}"
                )


def validate_literal_fields(record: SQLRecord, kwargs) -> None:
    """Validate all Literal type fields in a record.

    Args:
        record: record being validated

    Raises:
        ValidationError: If any field value is not in its Literal's allowed values
    """
    if isinstance(record, IsLink):
        return None
    if record.__class__.__name__ in "Feature":
        return None
    from lamindb.base.types import Dtype, TransformType

    types = {
        "TransformType": TransformType,
        "ArtifactKind": Dtype,
        "Dtype": Dtype,
    }
    errors = {}
    annotations = getattr(record.__class__, "__annotations__", {})
    for field_name, annotation in annotations.items():
        if field_name not in kwargs or kwargs[field_name] is None:
            continue
        value = kwargs[field_name]
        if str(annotation) in types:
            annotation = types[annotation]
        if not hasattr(annotation, "__origin__"):
            continue
        literal_type = annotation if annotation.__origin__ is Literal else None
        if literal_type is None:
            continue
        valid_values = set(literal_type.__args__)
        if value not in valid_values:
            errors[field_name] = (
                f"{field_name}: {colors.yellow(value)} is not a valid value"
                f"\n    → Valid values are: {colors.green(', '.join(sorted(valid_values)))}"
            )
    if errors:
        message = "\n  "
        for _, error in errors.items():
            message += error + "\n  "
        raise FieldValidationError(message)


def validate_fields(record: SQLRecord, kwargs):
    from lamindb.models import (
        Artifact,
        Collection,
        Feature,
        Run,
        Schema,
        Transform,
        ULabel,
    )

    # validate required fields
    # a "required field" is a Django field that has `null=False, default=None`
    required_fields = {
        k.name for k in record._meta.fields if not k.null and k.default is None
    }
    required_fields_not_passed = {k: None for k in required_fields if k not in kwargs}
    kwargs.update(required_fields_not_passed)
    missing_fields = [
        k for k, v in kwargs.items() if v is None and k in required_fields
    ]
    if missing_fields:
        raise FieldValidationError(f"{missing_fields} are required.")
    # ensure the exact length of the internal uid for core entities
    if "uid" in kwargs and record.__class__ in {
        Artifact,
        Collection,
        Transform,
        Run,
        ULabel,
        Feature,
        Schema,
    }:
        uid_max_length = record.__class__._meta.get_field(
            "uid"
        ).max_length  # triggers FieldDoesNotExist
        if len(kwargs["uid"]) != uid_max_length:  # triggers KeyError
            if not (
                record.__class__ is Schema and len(kwargs["uid"]) == 16
            ):  # no error for schema
                raise ValidationError(
                    f"`uid` must be exactly {uid_max_length} characters long, got {len(kwargs['uid'])}."
                )
    # validate is_type
    if "is_type" in kwargs and "name" in kwargs and kwargs["is_type"]:
        if kwargs["name"].endswith("s"):
            logger.warning(
                f"name '{kwargs['name']}' for type ends with 's', in case you're naming with plural, consider the singular for a type name"
            )
        is_approx_pascal_case(kwargs["name"])
    # validate literals
    validate_literal_fields(record, kwargs)


def suggest_records_with_similar_names(
    record: SQLRecord, name_field: str, kwargs
) -> SQLRecord | None:
    """Returns a record if found exact match, otherwise None.

    Logs similar matches if found.
    """
    if kwargs.get(name_field) is None or not isinstance(kwargs.get(name_field), str):
        return None
    # need to perform an additional request to find the exact match
    # previously, this was inferred from the truncated/fuzzy search below
    # but this isn't reliable: https://laminlabs.slack.com/archives/C04FPE8V01W/p1737812808563409
    # the below needs to be .first() because there might be multiple records with the same
    # name field in case the record is versioned (e.g. for Transform key)
    if hasattr(record.__class__, "type"):
        if kwargs.get("type", None) is None:
            subset = record.__class__.filter(type__isnull=True)
        else:
            subset = record.__class__.filter(type=kwargs["type"])
    else:
        subset = record.__class__
    exact_match = subset.filter(**{name_field: kwargs[name_field]}).first()
    if exact_match is not None:
        return exact_match
    queryset = _search(
        subset,
        kwargs[name_field],
        field=name_field,
        truncate_string=True,
        limit=3,
    )
    if not queryset.exists():  # empty queryset
        return None
    s, it, nots, record_text = (
        ("", "it", "s", "a record")
        if len(queryset) == 1
        else ("s", "one of them", "", "records")
    )
    similar_names = ", ".join(f"'{getattr(record, name_field)}'" for record in queryset)
    msg = f"you are trying to create a record with name='{kwargs[name_field]}' but {record_text} with similar {name_field}{s} exist{nots}: {similar_names}. Did you mean to load {it}?"
    logger.warning(f"{msg}")

    return None


def delete_record(record: BaseSQLRecord, is_soft: bool = True):
    def delete():
        if is_soft:
            record.branch_id = -1
            record.save()
        else:
            super(BaseSQLRecord, record).delete()

    # deal with versioned records
    # if _ovewrite_version = True, there is only a single version and
    # no need to set the new latest version because all versions are deleted
    # when deleting the latest version
    if (
        isinstance(record, IsVersioned)
        and record.is_latest
        and not getattr(record, "_overwrite_versions", False)
    ):
        new_latest = (
            record.__class__.objects.using(record._state.db)
            .filter(is_latest=False, uid__startswith=record.stem_uid)
            .exclude(branch_id=-1)  # exclude candidates in the trash
            .order_by("-created_at")
            .first()
        )
        if new_latest is not None:
            new_latest.is_latest = True
            if is_soft:
                record.is_latest = False
            with transaction.atomic():
                new_latest.save()
                delete()
            logger.important_hint(f"new latest version is: {new_latest}")
            return None
    # deal with all other cases of the nested if condition now
    delete()


RECORD_REGISTRY_EXAMPLE = """Example::

        from lamindb import SQLRecord, fields

        # sub-classing `SQLRecord` creates a new registry
        class Experiment(SQLRecord):
            name: str = fields.CharField()

        # instantiating `Experiment` creates a record `experiment`
        experiment = Experiment(name="my experiment")

        # you can save the record to the database
        experiment.save()

        # `Experiment` refers to the registry, which you can query
        df = Experiment.filter(name__startswith="my ").to_dataframe()
"""


# this is the metaclass for SQLRecord
@doc_args(RECORD_REGISTRY_EXAMPLE)
class Registry(ModelBase):
    """Metaclass for :class:`~lamindb.models.SQLRecord`.

    Each `Registry` *object* is a `SQLRecord` *class* and corresponds to a table in the metadata SQL database.

    You work with `Registry` objects whenever you use *class methods* of `SQLRecord`.

    You call any subclass of `SQLRecord` a "registry" and their objects "records". A `SQLRecord` object corresponds to a row in the SQL table.

    If you want to create a new registry, you sub-class `SQLRecord`.

    {}

    Note: `Registry` inherits from Django's `ModelBase`.
    """

    _available_fields: set[str] = None

    def __new__(cls, name, bases, attrs, **kwargs):
        new_class = super().__new__(cls, name, bases, attrs, **kwargs)
        return new_class

    # below creates a sensible auto-complete behavior that differs across the
    # class and instance level in Jupyter Editors it doesn't have any effect for
    # static type analyzer like pylance used in VSCode
    def __dir__(cls):
        # this is needed to bring auto-complete on the class-level back
        # https://laminlabs.slack.com/archives/C04FPE8V01W/p1717535625268849
        # Filter class attributes, excluding instance methods
        exclude_instance_methods = "sphinx" not in sys.modules
        # https://laminlabs.slack.com/archives/C04FPE8V01W/p1721134595920959

        def include_attribute(attr_name, attr_value):
            if attr_name.startswith("__"):
                return False
            if exclude_instance_methods and callable(attr_value):
                return isinstance(attr_value, (classmethod, staticmethod, type))
            return True

        # check also inherited attributes
        if hasattr(cls, "mro"):
            attrs = chain(*(c.__dict__.items() for c in cls.mro()))
        else:
            attrs = cls.__dict__.items()

        result = []
        for attr_name, attr_value in attrs:
            if attr_name not in result and include_attribute(attr_name, attr_value):
                result.append(attr_name)

        # Add non-dunder attributes from Registry
        for attr in dir(Registry):
            if not attr.startswith("__") and attr not in result:
                result.append(attr)
        return result

    def __repr__(cls) -> str:
        return registry_repr(cls)

    @doc_args(_lookup.__doc__)
    def lookup(
        cls,
        field: StrField | None = None,
        return_field: StrField | None = None,
        keep: Literal["first", "last", False] = "first",
    ) -> NamedTuple:
        """{}"""  # noqa: D415
        return _lookup(cls=cls, field=field, return_field=return_field, keep=keep)

    def filter(cls, *queries, **expressions) -> QuerySet:
        """Query records.

        Args:
            queries: One or multiple `Q` objects.
            expressions: Fields and values passed as Django query expressions.

        Returns:
            A :class:`~lamindb.models.QuerySet`.

        See Also:
            - Guide: :doc:`docs:registries`
            - Django documentation: `Queries <https://docs.djangoproject.com/en/stable/topics/db/queries/>`__

        Examples:
            >>> ln.ULabel(name="my label").save()
            >>> ln.ULabel.filter(name__startswith="my").to_dataframe()
        """
        from .query_set import QuerySet

        _using_key = None
        if "_using_key" in expressions:
            _using_key = expressions.pop("_using_key")

        return QuerySet(model=cls, using=_using_key).filter(*queries, **expressions)

    def get(
        cls: type[T],
        idlike: int | str | None = None,
        **expressions,
    ) -> T:
        """Get a single record.

        Args:
            idlike: Either a uid stub, uid or an integer id.
            expressions: Fields and values passed as Django query expressions.

        Raises:
            :exc:`docs:lamindb.errors.DoesNotExist`: In case no matching record is found.

        See Also:
            - Guide: :doc:`docs:registries`
            - Django documentation: `Queries <https://docs.djangoproject.com/en/stable/topics/db/queries/>`__

        Examples:

            ::

                ulabel = ln.ULabel.get("FvtpPJLJ")
                ulabel = ln.ULabel.get(name="my-label")
        """
        from .query_set import QuerySet

        return QuerySet(model=cls).get(idlike, **expressions)

    def to_dataframe(
        cls,
        include: str | list[str] | None = None,
        features: bool | list[str] | str = False,
        limit: int = 100,
    ) -> pd.DataFrame:
        """Convert to `pd.DataFrame`.

        By default, shows all direct fields, except `updated_at`.

        Use arguments `include` or `feature` to include other data.

        Args:
            include: Related fields to include as columns. Takes strings of
                form `"ulabels__name"`, `"cell_types__name"`, etc. or a list
                of such strings.
            features: If a list of feature names, filters
                :class:`~lamindb.Feature` down to these features.
                If `True`, prints all features with dtypes in the core schema module.
                If `"queryset"`, infers the features used within the set of artifacts or records.
                Only available for `Artifact` and `Record`.
            limit: Maximum number of rows to display from a Pandas DataFrame.
                Defaults to 100 to reduce database load.

        Examples:

            Include the name of the creator in the `DataFrame`:

            >>> ln.ULabel.to_dataframe(include="created_by__name"])

            Include display of features for `Artifact`:

            >>> df = ln.Artifact.to_dataframe(features=True)
            >>> ln.view(df)  # visualize with type annotations

            Only include select features:

            >>> df = ln.Artifact.to_dataframe(features=["cell_type_by_expert", "cell_type_by_model"])
        """
        query_set = cls.filter()
        if hasattr(cls, "updated_at"):
            query_set = query_set.order_by("-updated_at")
        return query_set[:limit].to_dataframe(include=include, features=features)

    @deprecated(new_name="to_dataframe")
    def df(
        cls,
        include: str | list[str] | None = None,
        features: bool | list[str] | str = False,
        limit: int = 100,
    ) -> pd.DataFrame:
        return cls.to_dataframe(include, features, limit)

    @doc_args(_search.__doc__)
    def search(
        cls,
        string: str,
        *,
        field: StrField | None = None,
        limit: int | None = 20,
        case_sensitive: bool = False,
    ) -> QuerySet:
        """{}"""  # noqa: D415
        return _search(
            cls=cls,
            string=string,
            field=field,
            limit=limit,
            case_sensitive=case_sensitive,
        )

    def using(
        cls,
        instance: str | None,
    ) -> QuerySet:
        """Use a non-default LaminDB instance.

        Args:
            instance: An instance identifier of form "account_handle/instance_name".

        Examples:
            >>> ln.ULabel.using("account_handle/instance_name").search("ULabel7", field="name")
                        uid    score
            name
            ULabel7  g7Hk9b2v  100.0
            ULabel5  t4Jm6s0q   75.0
            ULabel6  r2Xw8p1z   75.0
        """
        from .query_set import QuerySet

        # connection already established
        if instance in connections:
            return QuerySet(model=cls, using=instance)
        # we're in the default instance
        if instance is None or instance == "default":
            return QuerySet(model=cls, using=None)

        owner, name = get_owner_name_from_identifier(instance)
        current_instance_owner_name: list[str] = setup_settings.instance.slug.split("/")
        if [owner, name] == current_instance_owner_name:
            return QuerySet(model=cls, using=None)

        # move on to different instances
        cache_using_filepath = (
            setup_settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
        )
        settings_file = instance_settings_file(name, owner)
        if not settings_file.exists():
            result = connect_instance_hub(owner=owner, name=name)
            if isinstance(result, str):
                raise RuntimeError(
                    f"Failed to load instance {instance}, please check your permissions!"
                )
            iresult, _ = result
            # this can happen if querying via an old instance name
            if [iresult.get("owner"), iresult["name"]] == current_instance_owner_name:
                return QuerySet(model=cls, using=None)
            # do not use {} syntax below, it gives rise to a dict if the schema modules
            # are empty and then triggers a TypeError in missing_members = source_modules - target_modules
            source_modules = set(  # noqa
                [mod for mod in iresult["schema_str"].split(",") if mod != ""]
            )
            # this just retrives the full connection string from iresult
            db = update_db_using_local(iresult, settings_file)
            cache_using_filepath.write_text(
                f"{iresult['lnid']}\n{iresult['schema_str']}", encoding="utf-8"
            )
            # need to set the token if it is a fine_grained_access and the user is jwt (not public)
            is_fine_grained_access = (
                iresult["fine_grained_access"] and iresult["db_permissions"] == "jwt"
            )
            # access_db can take both: the dict from connect_instance_hub and isettings
            into_db_token = iresult
        else:
            isettings = load_instance_settings(settings_file)
            source_modules = isettings.modules
            db = isettings.db
            cache_using_filepath.write_text(
                f"{isettings.uid}\n{','.join(source_modules)}", encoding="utf-8"
            )
            # need to set the token if it is a fine_grained_access and the user is jwt (not public)
            is_fine_grained_access = (
                isettings._fine_grained_access and isettings._db_permissions == "jwt"
            )
            # access_db can take both: the dict from connect_instance_hub and isettings
            into_db_token = isettings

        target_modules = setup_settings.instance.modules
        if missing_members := source_modules - target_modules:
            logger.info(
                f"in transfer, source lamindb instance has additional modules: {', '.join(missing_members)}"
            )

        add_db_connection(db, instance)
        if is_fine_grained_access:
            db_token = DBToken(into_db_token)
            db_token_manager.set(db_token, instance)
        return QuerySet(model=cls, using=instance)

    def __get_module_name__(cls) -> str:
        schema_module_name = cls.__module__.split(".")[0]
        module_name = schema_module_name.replace("lnschema_", "")
        if module_name == "lamindb":
            module_name = "core"
        return module_name

    def __get_name_with_module__(cls) -> str:
        module_name = cls.__get_module_name__()
        if module_name == "core":
            module_prefix = ""
        else:
            module_prefix = f"{module_name}."
        return f"{module_prefix}{cls.__name__}"

    def __get_available_fields__(cls) -> set[str]:
        if cls._available_fields is None:
            available_fields = set()
            for field in cls._meta.get_fields():
                if not (field_name := field.name).startswith(("_", "links_")):
                    available_fields.add(field_name)
                    if isinstance(field, django_ForeignKey):
                        available_fields.add(field_name + "_id")
            if cls.__name__ == "Artifact":
                available_fields.add("visibility")  # backward compat
                available_fields.add("_branch_code")  # backward compat
                available_fields.add("transform")
            cls._available_fields = available_fields
        return cls._available_fields


class BaseSQLRecord(models.Model, metaclass=Registry):
    """Basic metadata record.

    It has the same methods as SQLRecord, but doesn't have the additional fields.

    It's mainly used for IsLinks and similar.
    """

    objects = QueryManager()

    class Meta:
        abstract = True
        base_manager_name = "objects"

    def __init__(self, *args, **kwargs):
        skip_validation = kwargs.pop("_skip_validation", False)
        if not args:
            if not os.getenv("LAMINDB_MULTI_INSTANCE") == "true":
                if (
                    issubclass(self.__class__, SQLRecord)
                    and self.__class__.__name__ != "Storage"
                    # do not save bionty entities in restricted spaces by default
                    and self.__class__.__module__ != "bionty.models"
                ):
                    from lamindb import context as run_context

                    if run_context.space is not None:
                        current_space = run_context.space
                    elif setup_settings.space is not None:
                        current_space = setup_settings.space

                    if current_space is not None:
                        if "space_id" in kwargs:
                            # space_id takes precedence over space
                            # https://claude.ai/share/f045e5dc-0143-4bc5-b8a4-38309229f75e
                            if kwargs["space_id"] == 1:  # ignore default space
                                kwargs.pop("space_id")
                                kwargs["space"] = current_space
                        elif "space" in kwargs:
                            if kwargs["space"] is None:
                                kwargs["space"] = current_space
                        else:
                            kwargs["space"] = current_space
                if issubclass(
                    self.__class__, SQLRecord
                ) and self.__class__.__name__ not in {"Storage", "Source"}:
                    from lamindb import context as run_context

                    if run_context.branch is not None:
                        current_branch = run_context.branch
                    elif setup_settings.branch is not None:
                        current_branch = setup_settings.branch

                    if current_branch is not None:
                        # branch_id takes precedence over branch
                        # https://claude.ai/share/f045e5dc-0143-4bc5-b8a4-38309229f75e
                        if "branch_id" in kwargs:
                            if kwargs["branch_id"] == 1:  # ignore default branch
                                kwargs.pop("branch_id")
                                kwargs["branch"] = current_branch
                        elif "branch" in kwargs:
                            if kwargs["branch"] is None:
                                kwargs["branch"] = current_branch
                        else:
                            kwargs["branch"] = current_branch
            if skip_validation:
                super().__init__(**kwargs)
            else:
                from ..core._settings import settings
                from .can_curate import CanCurate
                from .collection import Collection
                from .transform import Transform

                validate_fields(self, kwargs)

                # do not search for names if an id is passed; this is important
                # e.g. when synching ids from the notebook store to lamindb
                has_consciously_provided_uid = False
                if "_has_consciously_provided_uid" in kwargs:
                    has_consciously_provided_uid = kwargs.pop(
                        "_has_consciously_provided_uid"
                    )
                if (
                    isinstance(self, (CanCurate, Collection, Transform))
                    and settings.creation.search_names
                    and not has_consciously_provided_uid
                ):
                    name_field = getattr(self, "_name_field", "name")
                    exact_match = suggest_records_with_similar_names(
                        self, name_field, kwargs
                    )
                    if exact_match is not None:
                        if "version" in kwargs:
                            if kwargs["version"] is not None:
                                version_comment = " and version"
                                existing_record = self.__class__.filter(
                                    **{
                                        name_field: kwargs[name_field],
                                        "version": kwargs["version"],
                                    }
                                ).one_or_none()
                            else:
                                # for a versioned record, an exact name match is not a criterion
                                # for retrieving a record in case `version` isn't passed -
                                # we'd always pull out many records with exactly the same name
                                existing_record = None
                        else:
                            version_comment = ""
                            existing_record = exact_match
                        if existing_record is not None:
                            logger.important(
                                f"returning {self.__class__.__name__.lower()} with same"
                                f" {name_field}{version_comment}: '{kwargs[name_field]}'"
                            )
                            init_self_from_db(self, existing_record)
                            update_attributes(self, kwargs)
                            return None
                super().__init__(**kwargs)
                if isinstance(self, ValidateFields):
                    # this will trigger validation against django validators
                    try:
                        if hasattr(self, "clean_fields"):
                            self.clean_fields()
                        else:
                            self._Model__clean_fields()
                    except DjangoValidationError as e:
                        message = _format_django_validation_error(self, e)
                        raise FieldValidationError(message) from e
        elif len(args) != len(self._meta.concrete_fields):
            raise FieldValidationError(
                f"Use keyword arguments instead of positional arguments, e.g.: {self.__class__.__name__}(name='...')."
            )
        else:
            super().__init__(*args)
        track_current_key_and_name_values(self)

    def save(self: T, *args, **kwargs) -> T:
        """Save.

        Always saves to the default database.
        """
        using_key = None
        if "using" in kwargs:
            using_key = kwargs["using"]
        transfer_config = kwargs.pop("transfer", None)
        db = self._state.db
        pk_on_db = self.pk
        artifacts: list = []
        if self.__class__.__name__ == "Collection" and self.id is not None:
            # when creating a new collection without being able to access artifacts
            artifacts = self.ordered_artifacts.to_list()
        pre_existing_record = None
        # consider records that are being transferred from other databases
        transfer_logs: dict[str, list[str]] = {
            "mapped": [],
            "transferred": [],
            "run": None,
        }
        if db is not None and db != "default" and using_key is None:
            if isinstance(self, IsVersioned):
                if not self.is_latest:
                    raise NotImplementedError(
                        "You are attempting to transfer a record that's not the latest in its version history. This is currently not supported."
                    )
            pre_existing_record = transfer_to_default_db(
                self, using_key, transfer_logs=transfer_logs
            )
        self._revises: IsVersioned
        if pre_existing_record is not None:
            init_self_from_db(self, pre_existing_record)
        else:
            check_key_change(self)
            check_name_change(self)
            try:
                # save versioned record in presence of self._revises
                if isinstance(self, IsVersioned) and self._revises is not None:
                    assert self._revises.is_latest  # noqa: S101
                    revises = self._revises
                    revises.is_latest = False
                    with transaction.atomic():
                        revises._revises = None  # ensure we don't start a recursion
                        revises.save()
                        super().save(*args, **kwargs)  # type: ignore
                    self._revises = None
                # save unversioned record
                else:
                    super().save(*args, **kwargs)
            except (IntegrityError, ProgrammingError) as e:
                error_msg = str(e)
                # two possible error messages for hash duplication
                # "duplicate key value violates unique constraint"
                # "UNIQUE constraint failed"
                if (
                    self.__class__.__name__ in {"Transform", "Artifact", "Collection"}
                    and isinstance(e, IntegrityError)
                    and "hash" in error_msg
                    and unique_constraint_error_in_error_message(error_msg)
                ):
                    pre_existing_record = self.__class__.get(hash=self.hash)
                    logger.warning(
                        f"returning {self.__class__.__name__.lower()} with same hash: {pre_existing_record}"
                    )
                    init_self_from_db(self, pre_existing_record)
                elif (
                    self.__class__.__name__ == "Storage"
                    and isinstance(e, IntegrityError)
                    and ("root" in error_msg or "uid" in error_msg)
                    and unique_constraint_error_in_error_message(error_msg)
                ):
                    # even if uid was in the error message, we can retrieve based on
                    # the root because it's going to be the same root
                    pre_existing_record = self.__class__.get(root=self.root)
                    init_self_from_db(self, pre_existing_record)
                elif (
                    isinstance(e, IntegrityError)
                    and ("ontology_id" in error_msg or "uid" in error_msg)
                    and (
                        "UNIQUE constraint failed" in error_msg
                        or "duplicate key value violates unique constraint" in error_msg
                    )
                ):
                    if "UNIQUE constraint failed" in error_msg:  # sqlite
                        constraint_fields = [
                            f.split(".")[-1]
                            for f in error_msg.removeprefix(
                                "UNIQUE constraint failed: "
                            ).split(", ")
                        ]
                    else:  # postgres
                        constraint_fields = [
                            error_msg.split('"')[1]
                            .split('"')[0]
                            .removesuffix("_key")
                            .split("_")[-1]  # field name
                        ]
                    # here we query against the all branches with .objects
                    pre_existing_record = self.__class__.objects.get(
                        **{f: getattr(self, f) for f in constraint_fields}
                    )
                    if pre_existing_record.branch_id == 1:
                        logger.warning(
                            f"returning {self.__class__.__name__} record with same {', '.join(constraint_fields)}: '{', '.join([str(getattr(self, f)) for f in constraint_fields])}'"
                        )
                    else:
                        # modifies the fields of the existing record with new values of self
                        # TODO: parents should be properly dealt with
                        self._parents: list = []
                        field_names = [i.name for i in self.__class__._meta.fields]
                        update_attributes(
                            pre_existing_record,
                            {f: getattr(self, f) for f in field_names},
                        )
                        pre_existing_record.save()
                    init_self_from_db(self, pre_existing_record)
                elif (
                    isinstance(e, ProgrammingError)
                    and "new row violates row-level security policy" in error_msg
                    and (
                        (is_locked := getattr(self, "is_locked", False))
                        or hasattr(self, "space")
                    )
                ):
                    if is_locked:
                        no_write_msg = "It is not allowed to modify or create locked ('is_locked=True') records."
                    else:
                        no_write_msg = (
                            f"You're not allowed to write to the space '{self.space.name}'.\n"
                            "Please contact administrators of the space if you need write access."
                        )
                    raise NoWriteAccess(no_write_msg) from None
                elif (
                    isinstance(e, ProgrammingError)
                    and "permission denied for table" in error_msg
                    and (isettings := setup_settings.instance)._db_permissions
                    == "public"
                ):
                    slug = isettings.slug
                    raise NoWriteAccess(
                        f"You are trying to write to '{slug}' with public (read-only) permissions.\n"
                        "Please contact administrators to make you a collaborator if you need write access.\n"
                        f"If you are already a collaborator, please do 'lamin connect {slug}' in console, "
                        "restart the python session and try again."
                    ) from None
                else:
                    raise
            # call the below in case a user makes more updates to the record
            track_current_key_and_name_values(self)
        # perform transfer of many-to-many fields
        # only supported for Artifact and Collection records
        if db is not None and db != "default" and using_key is None:
            if self.__class__.__name__ == "Collection":
                if len(artifacts) > 0:
                    logger.info("transfer artifacts")
                    for artifact in artifacts:
                        artifact.save()
                    self.artifacts.add(*artifacts)
            if hasattr(self, "labels") and transfer_config == "annotations":
                from copy import copy

                # here we go back to original record on the source database
                self_on_db = copy(self)
                self_on_db._state.db = db
                self_on_db.pk = pk_on_db  # manually set the primary key
                self.features._add_from(self_on_db, transfer_logs=transfer_logs)
                self.labels.add_from(self_on_db, transfer_logs=transfer_logs)
            for k, v in transfer_logs.items():
                if k != "run" and len(v) > 0:
                    logger.important(f"{k}: {', '.join(v)}")

        if self.__class__.__name__ in {
            "Artifact",
            "Transform",
            "Run",
            "ULabel",
            "Feature",
            "Schema",
            "Collection",
            "Reference",
        } and not (
            self.__class__.__name__ == "Artifact" and self.kind == "__lamindb_run__"
        ):
            import lamindb as ln

            if ln.context.project is not None:
                self.projects.add(ln.context.project)
        return self

    def delete(self, permanent: bool | None = None) -> None:
        """Delete.

        Args:
            permanent: For consistency, `False` raises an error, as soft delete is impossible.
        """
        if permanent is False:
            raise ValueError(
                f"Soft delete is not possible for {self.__class__.__name__}, "
                "use 'permanent=True' or 'permanent=None' for permanent deletion."
            )

        delete_record(self, is_soft=False)


class Space(BaseSQLRecord):
    """Workspaces with managed access for specific users or teams.

    Guide: :doc:`docs:access`.

    All data in this registry is synchronized from LaminHub so that spaces can be shared and reused across multiple LaminDB instances.
    """

    class Meta:
        app_label = "lamindb"
        constraints = [
            models.UniqueConstraint(Lower("name"), name="unique_space_name_lower")
        ]

    id: int = models.SmallAutoField(primary_key=True)
    """Internal id, valid only in one DB instance."""
    name: str = models.CharField(max_length=100, db_index=True)
    """Name of space."""
    uid: str = CharField(
        editable=False,
        unique=True,
        max_length=12,
        default=base62_12,
        db_index=True,
    )
    """Universal id."""
    description: str | None = TextField(null=True)
    """Description of space."""
    created_at: datetime = DateTimeField(
        editable=False, db_default=models.functions.Now(), db_index=True
    )
    """Time of creation of record."""
    created_by: User = ForeignKey(
        "User", CASCADE, default=None, related_name="+", null=True
    )
    """Creator of space."""
    blocks: Block
    """Blocks that annotate this space."""

    @overload
    def __init__(
        self,
        name: str,
        description: str | None = None,
    ): ...

    @overload
    def __init__(
        self,
        *db_args,
    ): ...

    def __init__(
        self,
        *args,
        **kwargs,
    ):
        if not args and "uid" not in kwargs:
            warn = False
            msg = ""
            isettings = setup_settings.instance
            if (dialect := isettings.dialect) != "postgresql":
                warn = True
                msg = f"on {dialect} databases"
            elif not isettings.is_on_hub:
                warn = True
                msg = "on local instances"
            if warn:
                logger.warning(
                    f"creating spaces manually {msg} is possible for demo purposes, "
                    "but does *not* affect access permissions"
                )
        super().__init__(*args, **kwargs)


class Branch(BaseSQLRecord):
    """Branches for change management with archive and trash states.

    Every `SQLRecord` has a `branch` field, which dictates where a record appears in queries & searches.
    """

    class Meta:
        app_label = "lamindb"
        constraints = [
            models.UniqueConstraint(Lower("name"), name="unique_branch_name_lower")
        ]

    # below isn't fully implemented but a roadmap
    # - 3: template (hidden in queries & searches)
    # - 2: locked (same as default, but locked for edits except for space admins)
    # - 1: default (visible in queries & searches)
    # - 0: archive (hidden, meant to be kept, locked for edits for everyone)
    # - -1: trash (hidden, scheduled for deletion)

    # An integer higher than >3 codes a branch that can be used for collaborators to create drafts
    # that can be merged onto the main branch in an experience akin to a Pull Request. The mapping
    # onto a semantic branch name is handled through LaminHub.

    id: int = models.AutoField(primary_key=True)
    """An integer id that's synchronized for a family of coupled database instances.

    Among all LaminDB instances, this id is arbitrary and non-unique.
    """
    name: str = models.CharField(max_length=100, db_index=True)
    """Name of branch."""
    uid: str = CharField(
        editable=False,
        unique=True,
        max_length=12,
        default=base62_12,
        db_index=True,
    )
    """Universal id.

    This id is useful if one wants to apply the same patch to many database instances.
    """
    space: Space = ForeignKey(Space, PROTECT, default=1, db_default=1, related_name="+")
    """The space associated with the branch."""
    description: str | None = TextField(null=True)
    """Description of branch."""
    created_at: datetime = DateTimeField(
        editable=False, db_default=models.functions.Now(), db_index=True
    )
    """Time of creation of record."""
    created_by: User = ForeignKey(
        "User", CASCADE, default=None, related_name="+", null=True
    )
    """Creator of branch."""
    blocks: Block
    """Blocks that annotate this branch."""

    @overload
    def __init__(
        self,
        name: str,
        description: str | None = None,
    ): ...

    @overload
    def __init__(
        self,
        *db_args,
    ): ...

    def __init__(
        self,
        *args,
        **kwargs,
    ):
        super().__init__(*args, **kwargs)


@doc_args(RECORD_REGISTRY_EXAMPLE)
class SQLRecord(BaseSQLRecord, metaclass=Registry):
    """Metadata record.

    Every `SQLRecord` is a data model that comes with a registry in form of a SQL
    table in your database.

    Sub-classing `SQLRecord` creates a new registry while instantiating a `SQLRecord`
    creates a new record.

    {}

    `SQLRecord`'s metaclass is :class:`~lamindb.models.Registry`.

    `SQLRecord` inherits from Django's `Model` class. Why does LaminDB call it `SQLRecord`
    and not `Model`? The term `SQLRecord` can't lead to confusion with statistical,
    machine learning or biological models.
    """

    # we need the db_default when not interacting via django directly on a required field
    branch: Branch = ForeignKey(
        Branch,
        PROTECT,
        default=1,
        db_default=1,
        db_column="_branch_code",
        related_name="+",
    )
    """Whether record is on a branch or in another "special state"."""
    space: Space = ForeignKey(Space, PROTECT, default=1, db_default=1, related_name="+")
    """The space in which the record lives."""
    is_locked: bool = BooleanField(default=False, db_default=False)
    """Whether the record is locked for edits."""
    _aux: dict[str, Any] | None = JSONField(default=None, db_default=None, null=True)
    """Auxiliary field for dictionary-like metadata."""

    class Meta:
        abstract = True

    @property
    @deprecated("branch_id")
    def _branch_code(self) -> int:
        """Deprecated alias for `branch`."""
        return self.branch_id

    @_branch_code.setter
    def _branch_code(self, value: int):
        self.branch_id = value

    def restore(self) -> None:
        """Restore from trash onto the main branch."""
        self.branch_id = 1
        self.save()

    def delete(self, permanent: bool | None = None, **kwargs) -> None:
        """Delete record.

        Args:
            permanent: Whether to permanently delete the record (skips trash).
                If `None`, performs soft delete if the record is not already in the trash.

        Examples:

            For any `SQLRecord` object `record`, call:

            >>> record.delete()
        """
        if self._state.adding:
            logger.warning("record is not yet saved, delete has no effect")
            return
        name_with_module = self.__class__.__get_name_with_module__()

        if name_with_module == "Artifact":
            # this first check means an invalid delete fails fast rather than cascading through
            # database and storage permission errors
            isettings = setup_settings.instance
            if self.storage.instance_uid != isettings.uid and (
                kwargs["storage"] or kwargs["storage"] is None
            ):
                from ..errors import IntegrityError
                from .storage import Storage

                raise IntegrityError(
                    "Cannot simply delete artifacts outside of this instance's managed storage locations."
                    "\n(1) If you only want to delete the metadata record in this instance, pass `storage=False`"
                    f"\n(2) If you want to delete the artifact in storage, please load the managing lamindb instance (uid={self.storage.instance_uid})."
                    f"\nThese are all managed storage locations of this instance:\n{Storage.filter(instance_uid=isettings.uid).to_dataframe()}"
                )

        # change branch_id to trash
        trash_branch_id = -1
        if self.branch_id > trash_branch_id and permanent is not True:
            delete_record(self, is_soft=True)
            logger.warning(f"moved record to trash (branch_id = -1): {self}")
            return

        # permanent delete
        if permanent is None:
            response = input(
                f"Record {self.uid} is already in trash! Are you sure you want to delete it from your"
                " database? You can't undo this action. (y/n) "
            )
            confirm_delete = response == "y"
        else:
            confirm_delete = permanent

        if confirm_delete:
            if name_with_module == "Run":
                from .run import delete_run_artifacts

                delete_run_artifacts(self)
            elif name_with_module == "Transform":
                from .transform import delete_transform_relations

                delete_transform_relations(self)
            elif name_with_module == "Artifact":
                from .artifact import delete_permanently

                delete_permanently(
                    self, storage=kwargs["storage"], using_key=kwargs["using_key"]
                )
            if name_with_module != "Artifact":
                super().delete()


def _format_django_validation_error(record: SQLRecord, e: DjangoValidationError):
    """Pretty print Django validation errors."""
    errors = {}
    if hasattr(e, "error_dict"):
        error_dict = e.error_dict
    else:
        error_dict = {"__all__": e.error_list}

    for field_name, error_list in error_dict.items():
        for error in error_list:
            if hasattr(error, "message"):
                msg = error.message
            else:
                msg = str(error)

            if field_name == "__all__":
                errors[field_name] = f"{colors.yellow(msg)}"
            else:
                current_value = getattr(record, field_name, None)
                errors[field_name] = (
                    f"{field_name}: {colors.yellow(current_value)} is not valid\n    → {msg}"
                )

    if errors:
        message = "\n  "
        for _, error in errors.items():
            message += error + "\n  "

        return message


def _get_record_kwargs(record_class) -> list[tuple[str, str]]:
    """Gets the parameters of a SQLRecord from the overloaded signature.

    Example:
        >>> get_record_params(bt.Organism)
        >>> [('name', 'str'), ('taxon_id', 'str | None'), ('scientific_name', 'str | None')]
    """
    source = inspect.getsource(record_class)

    # Find first overload that's not *db_args
    pattern = r"@overload\s+def __init__\s*\(([\s\S]*?)\):\s*\.{3}"
    overloads = re.finditer(pattern, source)

    for single_overload in overloads:
        params_block = single_overload.group(1)
        # This is an additional safety measure if the overloaded signature that we're
        # looking for is not at the top but a "db_args" constructor
        if "*db_args" in params_block:
            continue

        params = []
        for line in params_block.split("\n"):
            line = line.strip()
            if not line or "self" in line:
                continue

            # Extract name and type annotation
            # The regex pattern finds parameter definitions like:
            # Simple: name: str
            # With default: age: int = 0
            # With complex types: items: List[str] = []
            param_pattern = (
                r"(\w+)"  # Parameter name
                r"\s*:\s*"  # Colon with optional whitespace
                r"((?:[^=,]|"  # Type hint: either non-equals/comma chars
                r"(?<=\[)[^[\]]*"  # or contents within square brackets
                r"(?=\]))+)"  # looking ahead for closing bracket
                r"(?:\s*=\s*"  # Optional default value part
                r"([^,]+))?"  # Default value: anything but comma
            )
            match = re.match(param_pattern, line)
            if not match:
                continue

            name, type_str = match.group(1), match.group(2).strip()

            # Keep type as string instead of evaluating
            params.append((name, type_str))

        return params

    return []


def get_name_field(
    registry: type[SQLRecord] | QuerySet | Manager,
    *,
    field: StrField | None = None,
) -> str:
    """Get the 1st char or text field from the registry."""
    if isinstance(registry, (QuerySet, Manager)):
        registry = registry.model
    model_field_names = [i.name for i in registry._meta.fields]

    # set to default name field
    if field is None:
        if hasattr(registry, "_name_field"):
            field = registry._meta.get_field(registry._name_field)
        elif "name" in model_field_names:
            field = registry._meta.get_field("name")
        else:
            # first char or text field that doesn't contain "id"
            for i in registry._meta.fields:
                if "id" in i.name:
                    continue
                if i.get_internal_type() in {"CharField", "TextField"}:
                    field = i
                    break

        # no default name field can be found
        if field is None:
            raise ValueError(
                f"Do not know which field to use as name file for registry {registry}, please pass field"
            )
        else:
            field = field.name  # type:ignore
    if not isinstance(field, str):
        try:
            field = field.field.name
        except AttributeError:
            raise TypeError(
                "please pass a SQLRecord string field, e.g., `CellType.name`!"
            ) from None

    return field


def add_db_connection(db: str, using: str):
    db_config = dj_database_url.config(
        default=db, conn_max_age=600, conn_health_checks=True
    )
    db_config["TIME_ZONE"] = "UTC"
    db_config["OPTIONS"] = {}
    db_config["AUTOCOMMIT"] = True
    connections.settings[using] = db_config


REGISTRY_UNIQUE_FIELD = {"storage": "root", "ulabel": "name"}


def update_fk_to_default_db(
    records: SQLRecord | list[SQLRecord] | QuerySet,
    fk: str,
    using_key: str | None,
    transfer_logs: dict,
):
    # here in case it is an iterable, we are checking only a single record
    # and set the same fks for all other records because we do this only
    # for certain fks where they have to the same for the whole bulk
    # see transfer_fk_to_default_db_bulk
    # todo: but this has to be changed i think, it is not safe as it is now - Sergei
    record = records[0] if isinstance(records, (list, QuerySet)) else records
    if getattr(record, f"{fk}_id", None) is not None:
        # set the space of the transferred record to the current space
        if fk == "space":
            # for space we set the record's space to the current space
            from lamindb import context

            # the default space has id=1
            fk_record_default = Space.get(1) if context.space is None else context.space
        # process non-space fks
        else:
            fk_record = getattr(record, fk)
            field = REGISTRY_UNIQUE_FIELD.get(fk, "uid")
            fk_record_default = fk_record.__class__.filter(
                **{field: getattr(fk_record, field)}
            ).one_or_none()
            if fk_record_default is None:
                from copy import copy

                fk_record_default = copy(fk_record)
                transfer_to_default_db(
                    fk_record_default, using_key, save=True, transfer_logs=transfer_logs
                )
        # re-set the fks to the newly saved ones in the default db
        if isinstance(records, (list, QuerySet)):
            for r in records:
                setattr(r, f"{fk}", None)
                setattr(r, f"{fk}_id", fk_record_default.id)
        else:
            setattr(records, f"{fk}", None)
            setattr(records, f"{fk}_id", fk_record_default.id)


FKBULK = [
    "organism",
    "source",
    "report",  # Run
]


def transfer_fk_to_default_db_bulk(
    records: list | QuerySet, using_key: str | None, transfer_logs: dict
):
    for fk in FKBULK:
        update_fk_to_default_db(records, fk, using_key, transfer_logs=transfer_logs)


def get_transfer_run(record) -> Run:
    from lamindb import settings
    from lamindb.core._context import context
    from lamindb.models import Run, Transform
    from lamindb.models.artifact import WARNING_RUN_TRANSFORM

    slug = record._state.db
    owner, name = get_owner_name_from_identifier(slug)
    cache_using_filepath = (
        ln_setup.settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
    )
    if not cache_using_filepath.exists():
        raise SystemExit("Need to call .using() before")
    instance_uid = cache_using_filepath.read_text().split("\n")[0]
    key = f"__lamindb_transfer__/{instance_uid}"
    uid = instance_uid + "0000"
    transform = Transform.filter(uid=uid).one_or_none()
    if transform is None:
        search_names = settings.creation.search_names
        settings.creation.search_names = False
        transform = Transform(  # type: ignore
            uid=uid, description=f"Transfer from `{slug}`", key=key, type="function"
        ).save()
        settings.creation.search_names = search_names
    # use the global run context to get the initiated_by_run run id
    if context.run is not None:
        initiated_by_run = context.run
    else:
        if not settings.creation.artifact_silence_missing_run_warning:
            logger.warning(WARNING_RUN_TRANSFORM)
        initiated_by_run = None
    # it doesn't seem to make sense to create new runs for every transfer
    run = Run.filter(transform=transform, initiated_by_run=initiated_by_run).first()
    if run is None:
        run = Run(transform=transform, initiated_by_run=initiated_by_run).save()  # type: ignore
        run.initiated_by_run = initiated_by_run  # so that it's available in memory
    return run


def transfer_to_default_db(
    record: SQLRecord,
    using_key: str | None,
    *,
    transfer_logs: dict,
    save: bool = False,
    transfer_fk: bool = True,
) -> SQLRecord | None:
    if record._state.db is None or record._state.db == "default":
        return None
    registry = record.__class__
    logger.debug(f"transferring {registry.__name__} record {record.uid} to default db")
    record_on_default = registry.objects.filter(uid=record.uid).one_or_none()
    record_str = f"{record.__class__.__name__}(uid='{record.uid}')"
    if transfer_logs["run"] is None:
        transfer_logs["run"] = get_transfer_run(record)
    if record_on_default is not None:
        transfer_logs["mapped"].append(record_str)
        return record_on_default
    else:
        transfer_logs["transferred"].append(record_str)

    if hasattr(record, "created_by_id"):
        record.created_by = None
        record.created_by_id = ln_setup.settings.user.id
    # run & transform
    run = transfer_logs["run"]
    if hasattr(record, "run_id"):
        record.run = None
        record.run_id = run.id
    # deal with denormalized transform FK on artifact and collection
    if hasattr(record, "transform_id"):
        record.transform = None
        record.transform_id = run.transform_id
    # transfer other foreign key fields
    fk_fields = [
        i.name
        for i in record._meta.fields
        if i.get_internal_type() == "ForeignKey"
        if i.name not in {"created_by", "run", "transform", "branch"}
    ]
    if not transfer_fk:
        # don't transfer fk fields that are already bulk transferred
        fk_fields = [fk for fk in fk_fields if fk not in FKBULK]
    for fk in fk_fields:
        update_fk_to_default_db(record, fk, using_key, transfer_logs=transfer_logs)
    record.id = None
    record._state.db = "default"
    if save:
        record.save()
    return None


def track_current_key_and_name_values(record: SQLRecord):
    from lamindb.models import Artifact

    # below, we're using __dict__ to avoid triggering the refresh from the database
    # which can lead to a recursion
    if isinstance(record, Artifact):
        record._old_key = record.__dict__.get("key")  # type: ignore
        record._old_suffix = record.__dict__.get("suffix")  # type: ignore
    elif hasattr(record, "_name_field"):
        record._old_name = record.__dict__.get(record._name_field)


def check_name_change(record: SQLRecord):
    """Warns if a record's name has changed."""
    from lamindb.models import (
        Artifact,
        Collection,
        Feature,
        Record,
        Schema,
        Storage,
        Transform,
    )

    if (
        not record.pk
        or not hasattr(record, "_old_name")
        or not hasattr(record, "_name_field")
    ):
        return

    # checked in check_key_change or not checked at all
    if isinstance(record, (Artifact, Collection, Transform)):
        return

    # renaming feature sets is not checked
    if isinstance(record, Schema):
        return

    old_name = record._old_name
    new_name = getattr(record, record._name_field)
    registry = record.__class__.__name__

    if old_name != new_name:
        # when a label is renamed, only raise a warning if it has a feature
        if hasattr(record, "artifacts") and not isinstance(record, (Record, Storage)):
            linked_records = (
                record.artifacts.through.filter(
                    label_ref_is_name=True, **{f"{registry.lower()}_id": record.pk}
                )
                .exclude(feature_id=None)  # must have a feature
                .distinct()
            )
            artifact_ids = linked_records.to_list("artifact__uid")
            n = len(artifact_ids)
            if n > 0:
                s = "s" if n > 1 else ""
                logger.error(
                    f"You are trying to {colors.red('rename label')} from '{old_name}' to '{new_name}'!\n"
                    f"   → The following {n} artifact{s} {colors.red('will no longer be validated')}: {artifact_ids}\n\n"
                    f"{colors.bold('To rename this label')}, make it external:\n"
                    f"   → run `artifact.labels.make_external(label)`\n\n"
                    f"After renaming, consider re-curating the above artifact{s}:\n"
                    f'   → in each dataset, manually modify label "{old_name}" to "{new_name}"\n'
                    f"   → run `ln.Curator`\n"
                )
                raise SQLRecordNameChangeIntegrityError

        # when a feature is renamed
        elif isinstance(record, Feature):
            # only internal features are associated with schemas
            linked_artifacts = Artifact.filter(feature_sets__features=record).to_list(
                "uid"
            )
            n = len(linked_artifacts)
            if n > 0:
                s = "s" if n > 1 else ""
                logger.error(
                    f"You are trying to {colors.red('rename feature')} from '{old_name}' to '{new_name}'!\n"
                    f"   → The following {n} artifact{s} {colors.red('will no longer be validated')}: {linked_artifacts}\n\n"
                    f"{colors.bold('To rename this feature')}, make it external:\n"
                    "   → run `artifact.features.make_external(feature)`\n\n"
                    f"After renaming, consider re-curating the above artifact{s}:\n"
                    f"   → in each dataset, manually modify feature '{old_name}' to '{new_name}'\n"
                    f"   → run `ln.Curator`\n"
                )
                raise SQLRecordNameChangeIntegrityError


def check_key_change(record: Union[Artifact, Transform]):
    """Errors if a record's key has falsely changed."""
    from .artifact import Artifact

    if not isinstance(record, Artifact) or not hasattr(record, "_old_key"):
        return
    if record._old_suffix != record.suffix:  # type: ignore
        raise InvalidArgument(
            f"Changing the `.suffix` of an artifact is not allowed! You tried to change it from '{record._old_suffix}' to '{record.suffix}'."  # type: ignore
        )

    old_key = record._old_key  # type: ignore
    new_key = record.key

    if old_key != new_key:
        if not record._key_is_virtual:
            raise InvalidArgument(
                f"Changing a non-virtual key of an artifact is not allowed! You tried to change it from '{old_key}' to '{new_key}'."
            )
        if old_key is not None:
            old_key_suffix = extract_suffix_from_path(
                PurePosixPath(old_key), arg_name="key"
            )
            assert old_key_suffix == record.suffix, (  # noqa: S101
                old_key_suffix,
                record.suffix,
            )
        else:
            old_key_suffix = record.suffix
        new_key_suffix = extract_suffix_from_path(
            PurePosixPath(new_key), arg_name="key"
        )
        if old_key_suffix != new_key_suffix:
            raise InvalidArgument(
                f"The suffix '{new_key_suffix}' of the provided key is incorrect, it should be '{old_key_suffix}'."
            )


def format_field_value(value: datetime | str | Any) -> Any:
    from datetime import datetime

    if isinstance(value, datetime):
        return value.strftime("%Y-%m-%d %H:%M:%S %Z")

    if isinstance(value, str):
        try:
            value = datetime.fromisoformat(value)
            value = value.strftime("%Y-%m-%d %H:%M:%S %Z")
        except ValueError:
            pass
        return f"'{value}'"
    else:
        return value


class SQLRecordInfo:
    def __init__(self, registry: Registry):
        self.registry = registry

    def _get_type_for_field(self, field_name: str) -> str:
        field = self.registry._meta.get_field(field_name)
        related_model_name = (
            field.related_model.__name__
            if hasattr(field, "related_model") and field.related_model
            else None
        )
        return related_model_name if related_model_name else field.get_internal_type()

    def _get_base_class_fields(self) -> list[str]:
        return [
            field.name
            for base in self.registry.__bases__
            if hasattr(base, "_meta")
            for field in base._meta.get_fields()
        ]

    def _reorder_fields_by_class(self, fields_to_order: list[Field]) -> list[Field]:
        """Reorders the fields so that base class fields come last."""
        non_base_class_fields = [
            field
            for field in fields_to_order
            if field.name not in self._get_base_class_fields()
        ]
        found_base_class_fields = [
            field
            for field in fields_to_order
            if field.name in self._get_base_class_fields()
        ]
        return non_base_class_fields + found_base_class_fields

    def get_simple_fields(self, return_str: bool = False) -> Any:
        simple_fields = [
            field
            for field in self.registry._meta.get_fields()
            if not (
                isinstance(field, ManyToOneRel)
                or isinstance(field, ManyToManyRel)
                or isinstance(field, ManyToManyField)
                or isinstance(field, ForeignKey)
                or field.name.startswith("_")
                or field.name == "id"
            )
        ]
        simple_fields = self._reorder_fields_by_class(simple_fields)
        if not return_str:
            return simple_fields
        else:
            repr_str = f"  {colors.italic('Simple fields')}\n"
            if simple_fields:
                repr_str += "".join(
                    [
                        f"    .{field_name.name}: {self._get_type_for_field(field_name.name)}\n"
                        for field_name in simple_fields
                    ]
                )
            return repr_str

    def get_relational_fields(self, return_str: bool = False):
        # we ignore ManyToOneRel because it leads to so much clutter in the API
        # also note that our general guideline is to have related_name="+"
        # for ForeignKey fields
        relational_fields = (ManyToOneRel, ManyToManyRel, ManyToManyField, ForeignKey)

        class_specific_relational_fields = [
            field
            for field in self.registry._meta.fields + self.registry._meta.many_to_many
            if isinstance(field, relational_fields)
            and not field.name.startswith(("links_", "_"))
        ]

        non_class_specific_relational_fields = [
            field
            for field in self.registry._meta.get_fields()
            if isinstance(field, relational_fields)
            and not field.name.startswith(("links_", "_"))
        ]
        non_class_specific_relational_fields = self._reorder_fields_by_class(
            non_class_specific_relational_fields
        )

        # Ensure that class specific fields (e.g. Artifact) come before non-class specific fields (e.g. collection)
        filtered_non_class_specific = [
            field
            for field in non_class_specific_relational_fields
            if field not in class_specific_relational_fields
        ]
        ordered_relational_fields = (
            class_specific_relational_fields + filtered_non_class_specific
        )

        core_module_fields = []
        external_modules_fields = []
        for field in ordered_relational_fields:
            field_name = repr(field).split(": ")[1][:-1]
            if field_name.count(".") == 1 and "lamindb" not in field_name:
                external_modules_fields.append(field)
            else:
                core_module_fields.append(field)

        def _get_related_field_type(field) -> str:
            field_type = (
                field.related_model.__get_name_with_module__()
                .replace(
                    "Artifact", ""
                )  # some fields have an unnecessary 'Artifact' in their name
                .replace(
                    "Collection", ""
                )  # some fields have an unnecessary 'Collection' in their name
            )
            return (
                self._get_type_for_field(field.name)
                if not field_type.strip()
                else field_type
            )

        core_module_fields_formatted = [
            f"    .{field.name}: {_get_related_field_type(field)}\n"
            for field in core_module_fields
        ]
        external_modules_fields_formatted = [
            f"    .{field.name}: {_get_related_field_type(field)}\n"
            for field in external_modules_fields
        ]

        if not return_str:
            external_modules_fields_by_modules = defaultdict(list)
            for field_str, field in zip(
                external_modules_fields_formatted, external_modules_fields
            ):
                field_type = field_str.split(":")[1].split()[0]
                module_name = field_type.split(".")[0]
                external_modules_fields_by_modules[module_name].append(field)
            return core_module_fields, external_modules_fields_by_modules
        else:
            repr_str = ""

            # Non-external relational fields
            if core_module_fields:
                repr_str += f"  {colors.italic('Relational fields')}\n"
                repr_str += "".join(core_module_fields_formatted)

            # External relational fields
            external_modules = set()
            for field in external_modules_fields_formatted:
                field_type = field.split(":")[1].split()[0]
                external_modules.add(field_type.split(".")[0])

            if external_modules:
                # We want Bionty to show up before other modules
                external_modules = (
                    ["bionty"] + sorted(external_modules - {"bionty"})  # type: ignore
                    if "bionty" in external_modules
                    else sorted(external_modules)
                )
                for ext_module in external_modules:
                    ext_module_fields = [
                        field
                        for field in external_modules_fields_formatted
                        if ext_module in field
                    ]

                    if ext_module_fields:
                        repr_str += (
                            f"  {colors.italic(f'{ext_module.capitalize()} fields')}\n"
                        )
                        repr_str += "".join(ext_module_fields)

            return repr_str


def registry_repr(cls):
    """Shows fields."""
    repr_str = f"{colors.green(cls.__name__)}\n"
    info = SQLRecordInfo(cls)
    repr_str += info.get_simple_fields(return_str=True)
    repr_str += info.get_relational_fields(return_str=True)
    repr_str = repr_str.rstrip("\n")
    return repr_str


def record_repr(
    self: SQLRecord, include_foreign_keys: bool = True, exclude_field_names=None
) -> str:
    if exclude_field_names is None:
        exclude_field_names = ["id", "updated_at", "source_code"]
    field_names = [
        field.name
        for field in self._meta.fields
        if (not isinstance(field, ForeignKey) and field.name not in exclude_field_names)
    ]
    if include_foreign_keys:
        field_names += [
            f"{field.name}_id"
            for field in self._meta.fields
            if isinstance(field, ForeignKey)
        ]
    if "created_at" in field_names:
        field_names.remove("created_at")
        field_names.append("created_at")
    if "is_locked" in field_names:
        field_names.remove("is_locked")
        field_names.append("is_locked")
    if field_names[0] != "uid" and "uid" in field_names:
        field_names.remove("uid")
        field_names.insert(0, "uid")
    fields_str = {}
    for k in field_names:
        if k == "n" and getattr(self, k) < 0:
            # only needed for Schema
            continue
        if not k.startswith("_") and hasattr(self, k):
            value = getattr(self, k)
            # Force strip the time component of the version
            if k == "version" and value:
                fields_str[k] = f"'{str(value).split()[0]}'"
            else:
                fields_str[k] = format_field_value(value)
    fields_joined_str = ", ".join(
        [f"{k}={fields_str[k]}" for k in fields_str if fields_str[k] is not None]
    )
    return f"{self.__class__.__name__}({fields_joined_str})"


SQLRecord.__repr__ = record_repr  # type: ignore
SQLRecord.__str__ = record_repr  # type: ignore


class Migration(BaseSQLRecord):
    app = CharField(max_length=255)
    name = CharField(max_length=255)
    applied: datetime = DateTimeField()

    class Meta:
        db_table = "django_migrations"
        app_label = "lamindb"
        managed = False


LinkORM = IsLink  # backward compat
Record = SQLRecord  # backward compat
BasicRecord = BaseSQLRecord  # backward compat
RecordInfo = SQLRecordInfo  # backward compat
