import json
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Any, Literal, TypeVar, cast, overload

from pydantic import BaseModel, TypeAdapter, ValidationError

from cognite.neat._data_model.models.dms import (
    Constraint,
    ConstraintAdapter,
    ContainerPropertyDefinition,
    ContainerReference,
    ContainerRequest,
    DataModelRequest,
    Index,
    IndexAdapter,
    NodeReference,
    RequestSchema,
    SpaceRequest,
    UniquenessConstraintDefinition,
    ViewReference,
    ViewRequest,
    ViewRequestProperty,
    ViewRequestPropertyAdapter,
)
from cognite.neat._data_model.models.dms._constants import DATA_MODEL_DESCRIPTION_MAX_LENGTH
from cognite.neat._data_model.models.entities import ParsedEntity, parse_entity
from cognite.neat._exceptions import DataModelImportException
from cognite.neat._issues import ModelSyntaxError
from cognite.neat._utils.text import humanize_collection
from cognite.neat._utils.validation import ValidationContext, humanize_validation_error

from .data_classes import (
    CREATOR_KEY,
    CREATOR_MARKER,
    DMSContainer,
    DMSEnum,
    DMSNode,
    DMSProperty,
    DMSView,
    EntityTableFilter,
    RAWFilterTableFilter,
    TableDMS,
    TableViewFilter,
)
from .source import TableSource

T_BaseModel = TypeVar("T_BaseModel", bound=BaseModel)


@dataclass
class ReadViewProperty:
    prop_id: str
    row_no: int
    view_property: ViewRequestProperty


@dataclass
class ReadContainerProperty:
    prop_id: str
    row_no: int
    container_property: ContainerPropertyDefinition


@dataclass
class ReadIndex:
    prop_id: str
    order: int | None
    row_no: int
    index_id: str
    index: Index


@dataclass
class ReadConstraint:
    prop_id: str
    order: int | None
    row_no: int
    constraint_id: str
    constraint: Constraint


@dataclass
class ReadProperties:
    """Read properties from the properties table.

    Attributes:
        container: A mapping from container entity to a mapping of property identifier to container property definition.
        view: A mapping from view entity to a mapping of property identifier to view property definition.
        indices: A mapping from (container entity, index identifier) to a list of read indices
        constraints: A mapping from (container entity, constraint identifier) to a list of read constraints
    """

    container: dict[tuple[ParsedEntity, str], list[ReadContainerProperty]] = field(
        default_factory=lambda: defaultdict(list)
    )
    view: dict[tuple[ParsedEntity, str], list[ReadViewProperty]] = field(default_factory=lambda: defaultdict(list))
    indices: dict[tuple[ParsedEntity, str], list[ReadIndex]] = field(default_factory=lambda: defaultdict(list))
    constraints: dict[tuple[ParsedEntity, str], list[ReadConstraint]] = field(default_factory=lambda: defaultdict(list))


@dataclass
class ProcessedProperties:
    container: dict[ParsedEntity, dict[str, ContainerPropertyDefinition]] = field(
        default_factory=lambda: defaultdict(dict)
    )
    view: dict[ParsedEntity, dict[str, ViewRequestProperty]] = field(default_factory=lambda: defaultdict(dict))
    indices: dict[ParsedEntity, dict[str, Index]] = field(default_factory=lambda: defaultdict(dict))
    constraints: dict[ParsedEntity, dict[str, Constraint]] = field(default_factory=lambda: defaultdict(dict))


class DMSTableReader:
    """Reads a TableDMS object and converts it to a RequestSchema.


    Args:
        default_space (str): The default space to use when no space is given in an entity.
        default_version (str): The default version to use when no version is given in an entity.
        source (TableSource): The source of the table data, used for error reporting.

    Raises:
        DataModelImportError: If there are any errors in the data model.

    Attributes:
        errors (list[ModelSyntaxError]): A list of errors encountered during parsing.

    Class Attributes:
        Sheets: This is used to create error messages. It ensures that the column names matches
            the names in the table, even if they are renamed in the code.
        PropertyColumns: This is used to create error messages for the properties table.
            It ensures that the column names matches the names in the table, even if they are renamed in the code.
        ContainerColumns: This is used to create error messages for the containers table.
            It ensures that the column names matches the names in the table, even if they are renamed in the code.
        ViewColumns: This is used to create error messages for the views table.
            It ensures that the column names matches the names in the table, even if they are renamed in the code.

    """

    CELL_MISSING = "#N/A"

    # The following classes are used when creating error messages. They ensure that the column names
    # matches the names in the table, even if they are renamed in the code.
    # Note that this is not a complete list of all columns, only those that are used in error messages.
    class Sheets:
        metadata = cast(str, TableDMS.model_fields["metadata"].validation_alias)
        properties = cast(str, TableDMS.model_fields["properties"].validation_alias)
        containers = cast(str, TableDMS.model_fields["containers"].validation_alias)
        views = cast(str, TableDMS.model_fields["views"].validation_alias)
        nodes = cast(str, TableDMS.model_fields["nodes"].validation_alias)

    class PropertyColumns:
        view = cast(str, DMSProperty.model_fields["view"].validation_alias)
        view_property = cast(str, DMSProperty.model_fields["view_property"].validation_alias)
        connection = cast(str, DMSProperty.model_fields["connection"].validation_alias)
        value_type = cast(str, DMSProperty.model_fields["value_type"].validation_alias)
        min_count = cast(str, DMSProperty.model_fields["min_count"].validation_alias)
        max_count = cast(str, DMSProperty.model_fields["max_count"].validation_alias)
        default = cast(str, DMSProperty.model_fields["default"].validation_alias)
        auto_increment = cast(str, DMSProperty.model_fields["auto_increment"].validation_alias)
        container = cast(str, DMSProperty.model_fields["container"].validation_alias)
        container_property = cast(str, DMSProperty.model_fields["container_property"].validation_alias)
        container_property_name = cast(str, DMSProperty.model_fields["container_property_name"].validation_alias)
        container_property_description = cast(
            str, DMSProperty.model_fields["container_property_description"].validation_alias
        )
        index = cast(str, DMSProperty.model_fields["index"].validation_alias)
        constraint = cast(str, DMSProperty.model_fields["constraint"].validation_alias)

    class ContainerColumns:
        container = cast(str, DMSContainer.model_fields["container"].validation_alias)
        constraint = cast(str, DMSContainer.model_fields["constraint"].validation_alias)

    class ViewColumns:
        view = cast(str, DMSView.model_fields["view"].validation_alias)
        filter = cast(str, DMSView.model_fields["filter"].validation_alias)

    def __init__(self, default_space: str, default_version: str, source: TableSource) -> None:
        self.default_space = default_space
        self.default_version = default_version
        self.source = source
        self.errors: list[ModelSyntaxError] = []

    def read_tables(self, tables: TableDMS) -> RequestSchema:
        space_request = self.read_space(self.default_space)
        node_types = self.read_nodes(tables.nodes)
        enum_collections = self.read_enum_collections(tables.enum)
        container_ref_by_entity = self.read_entity_by_container_ref(tables.containers)
        view_ref_by_entity = self.read_entity_by_view_ref(tables.views)
        read = self.read_properties(tables.properties, enum_collections, container_ref_by_entity, view_ref_by_entity)
        processed = self.process_properties(read)
        containers = self.read_containers(tables.containers, processed)
        views, valid_view_entities = self.read_views(tables.views, processed.view)
        data_model = self.read_data_model(tables, valid_view_entities)

        if self.errors:
            raise DataModelImportException(self.errors) from None
        return RequestSchema(
            dataModel=data_model, views=views, containers=containers, spaces=[space_request], nodeTypes=node_types
        )

    def read_space(self, space: str) -> SpaceRequest:
        space_request = self._validate_obj(SpaceRequest, {"space": space}, (self.Sheets.metadata,), field_name="value")
        if space_request is None:
            # If space is invalid, we stop parsing to avoid raising an error for every place the space is used.
            raise DataModelImportException(self.errors) from None
        return space_request

    def read_nodes(self, nodes: list[DMSNode]) -> list[NodeReference]:
        node_refs: list[NodeReference] = []
        for row_no, row in enumerate(nodes):
            data = self._create_node_ref(row.node)
            instantiated = self._validate_obj(NodeReference, data, (self.Sheets.nodes, row_no))
            if instantiated is not None:
                node_refs.append(instantiated)
        return node_refs

    @staticmethod
    def read_enum_collections(enum_rows: list[DMSEnum]) -> dict[str, dict[str, Any]]:
        enum_collections: dict[str, dict[str, Any]] = defaultdict(dict)
        for row in enum_rows:
            enum_collections[row.collection][row.value] = {
                "name": row.name,
                "description": row.description,
            }
        return enum_collections

    def read_entity_by_container_ref(self, containers: list[DMSContainer]) -> dict[ContainerReference, ParsedEntity]:
        entity_by_container_ref: dict[ContainerReference, ParsedEntity] = {}
        for container in containers:
            data = self._create_container_ref(container.container)
            try:
                container_ref = ContainerReference.model_validate(data)
            except ValidationError:
                # Error will be reported when reading the containers
                continue
            entity_by_container_ref[container_ref] = container.container
        return entity_by_container_ref

    def read_entity_by_view_ref(self, views: list[DMSView]) -> dict[ViewReference, ParsedEntity]:
        entity_by_view_ref: dict[ViewReference, ParsedEntity] = {}
        for view in views:
            data = self._create_view_ref(view.view)
            try:
                view_ref = ViewReference.model_validate(data)
            except ValidationError:
                # Error will be reported when reading the views
                continue
            entity_by_view_ref[view_ref] = view.view
        return entity_by_view_ref

    def read_properties(
        self,
        properties: list[DMSProperty],
        enum_collections: dict[str, dict[str, Any]],
        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
        view_ref_by_entity: dict[ViewReference, ParsedEntity],
    ) -> ReadProperties:
        read = ReadProperties()
        view_entities = set(view_ref_by_entity.values())
        container_entities = set(container_ref_by_entity.values())
        for row_no, prop in enumerate(properties):
            self._process_view_property(prop, read, row_no, view_ref_by_entity, view_entities)
            if prop.container is None or prop.container_property is None:
                # This is when the property is an edge or reverse direct relation property.
                continue
            self._process_container_property(
                prop, read, enum_collections, row_no, container_ref_by_entity, container_entities
            )
            self._process_index(prop, read, row_no, container_ref_by_entity, container_entities)
            self._process_constraint(prop, read, row_no, container_ref_by_entity, container_entities)
        return read

    def process_properties(self, read: ReadProperties) -> ProcessedProperties:
        return ProcessedProperties(
            container=self.create_container_properties(read),
            view=self.create_view_properties(read),
            indices=self.create_indices(read),
            constraints=self.create_constraints(read),
        )

    def create_container_properties(
        self, read: ReadProperties
    ) -> dict[ParsedEntity, dict[str, ContainerPropertyDefinition]]:
        container_props: dict[ParsedEntity, dict[str, ContainerPropertyDefinition]] = defaultdict(dict)
        for (container_entity, prop_id), prop_list in read.container.items():
            if len(prop_list) == 0:
                # Should not happen
                continue
            container_props[container_entity][prop_id] = prop_list[0].container_property
            if len(prop_list) > 1 and self._are_definitions_different(prop_list):
                # If multiple view properties are mapping to the same container property,
                # the container property definitions must be the same.
                rows_str = humanize_collection(
                    [self.source.adjust_row_number(self.Sheets.properties, p.row_no) for p in prop_list]
                )
                container_columns_str = humanize_collection(
                    [
                        self.PropertyColumns.connection,
                        self.PropertyColumns.value_type,
                        self.PropertyColumns.min_count,
                        self.PropertyColumns.max_count,
                        self.PropertyColumns.default,
                        self.PropertyColumns.auto_increment,
                        self.PropertyColumns.container_property_name,
                        self.PropertyColumns.container_property_description,
                        self.PropertyColumns.index,
                        self.PropertyColumns.constraint,
                    ]
                )
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {self.source.location((self.Sheets.properties,))} "
                            f"when the column {self.PropertyColumns.container!r} and "
                            f"{self.PropertyColumns.container_property!r} are the same, "
                            f"all the container columns ({container_columns_str}) must be the same. "
                            f"Inconsistent definitions for container '{container_entity!s} "
                            f"and {prop_id!r}' found in rows {rows_str}."
                        )
                    )
                )
        return container_props

    def _are_definitions_different(self, prop_list: list[ReadContainerProperty]) -> bool:
        if len(prop_list) < 2:
            return False
        first_def = prop_list[0].container_property
        for prop in prop_list[1:]:
            if first_def != prop.container_property:
                return True
        return False

    def create_view_properties(self, read: ReadProperties) -> dict[ParsedEntity, dict[str, ViewRequestProperty]]:
        view_props: dict[ParsedEntity, dict[str, ViewRequestProperty]] = defaultdict(dict)
        for (view_entity, prop_id), prop_list in read.view.items():
            if len(prop_list) == 0:
                # Should not happen
                continue
            view_props[view_entity][prop_id] = prop_list[0].view_property
            if len(prop_list) > 1:
                # Safeguard against duplicated rows for view properties.
                rows_str = humanize_collection(
                    [self.source.adjust_row_number(self.Sheets.properties, p.row_no) for p in prop_list]
                )
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {self.source.location((self.Sheets.properties,))} the combination of columns "
                            f"{self.PropertyColumns.view!r} and {self.PropertyColumns.view_property!r} must be unique. "
                            f"Duplicated entries for view '{view_entity!s}' and "
                            f"property '{prop_id!s}' found in rows {rows_str}."
                        )
                    )
                )

        return view_props

    def create_indices(self, read: ReadProperties) -> dict[ParsedEntity, dict[str, Index]]:
        indices: dict[ParsedEntity, dict[str, Index]] = defaultdict(dict)
        for (container_entity, index_id), index_list in read.indices.items():
            if len(index_list) == 0:
                continue
            # Remove duplicates based on prop_id, keeping the first occurrence
            # Note that we have already validated that the index definitions are the same
            index_list = list({read_index.prop_id: read_index for read_index in index_list}.values())
            index = index_list[0].index
            if len(index_list) == 1:
                indices[container_entity][index_id] = index
                continue
            if missing_order := [idx for idx in index_list if idx.order is None]:
                row_str = humanize_collection(
                    [self.source.adjust_row_number(self.Sheets.properties, idx.row_no) for idx in missing_order]
                )
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In table {self.Sheets.properties!r} column {self.PropertyColumns.index!r}: "
                            f"the index {index_id!r} on container {container_entity!s} is defined on multiple "
                            f"properties. This requires the 'order' attribute to be set. It is missing in rows "
                            f"{row_str}."
                        )
                    )
                )
                continue
            index.properties = [idx.prop_id for idx in sorted(index_list, key=lambda x: x.order or 999)]
            indices[container_entity][index_id] = index
        return indices

    def create_constraints(self, read: ReadProperties) -> dict[ParsedEntity, dict[str, Constraint]]:
        constraints: dict[ParsedEntity, dict[str, Constraint]] = defaultdict(dict)
        for (container_entity, constraint_id), constraint_list in read.constraints.items():
            if len(constraint_list) == 0:
                continue
            # Remove duplicates based on prop_id, keeping the first occurrence
            # Note that we have already validated that the constraint definitions are the same
            constraint_list = list(
                {read_constraint.prop_id: read_constraint for read_constraint in constraint_list}.values()
            )
            constraint = constraint_list[0].constraint
            if len(constraint_list) == 1 or not isinstance(constraint, UniquenessConstraintDefinition):
                constraints[container_entity][constraint_id] = constraint
                continue
            if missing_order := [c for c in constraint_list if c.order is None]:
                row_str = humanize_collection(
                    [self.source.adjust_row_number(self.Sheets.properties, c.row_no) for c in missing_order]
                )
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In table {self.Sheets.properties!r} column {self.PropertyColumns.constraint!r}: "
                            f"the uniqueness constraint {constraint_id!r} on container {container_entity!s} is defined "
                            f"on multiple properties. This requires the 'order' attribute to be set. It is missing in "
                            f"rows {row_str}."
                        )
                    )
                )
                continue
            constraint.properties = [c.prop_id for c in sorted(constraint_list, key=lambda x: x.order or 999)]
            constraints[container_entity][constraint_id] = constraint
        return constraints

    def _process_view_property(
        self,
        prop: DMSProperty,
        read: ReadProperties,
        row_no: int,
        view_ref_by_entity: dict[ViewReference, ParsedEntity],
        view_entities: set[ParsedEntity],
    ) -> None:
        loc = (self.Sheets.properties, row_no)
        data = self.read_view_property(prop, loc)
        view_prop = self._validate_adapter(ViewRequestPropertyAdapter, data, loc)
        if view_prop is None:
            return None
        if prop.view in view_entities:
            read.view[(prop.view, prop.view_property)].append(ReadViewProperty(prop.view_property, row_no, view_prop))
            return None
        # The view entity was not found in the views table. This could either be because the view is missing,
        # or because either the view entity in the Properties table and the View table are specified with/without
        # default space/version inconsistently.
        try:
            view_ref = ViewReference.model_validate(self._create_view_ref(prop.view))
        except ValidationError:
            # Error will be reported when reading the views
            return None
        if view_ref in view_ref_by_entity:
            view_ref_entity = view_ref_by_entity[view_ref]
            read.view[(view_ref_entity, prop.view_property)].append(
                ReadViewProperty(prop.view_property, row_no, view_prop)
            )
        else:
            self.errors.append(
                ModelSyntaxError(
                    message=(
                        f"In {self.source.location(loc)} the View '{prop.view!s}' "
                        f"was not found in the {self.Sheets.views!r} table."
                    )
                )
            )
        return None

    def _process_container_property(
        self,
        prop: DMSProperty,
        read: ReadProperties,
        enum_collections: dict[str, dict[str, Any]],
        row_no: int,
        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
        container_entities: set[ParsedEntity],
    ) -> None:
        loc = (self.Sheets.properties, row_no)
        data = self.read_container_property(prop, enum_collections, loc=loc)
        container_prop = self._validate_obj(ContainerPropertyDefinition, data, loc)
        if container_prop is None:
            return None
        if not (prop.container and prop.container_property):
            return None
        if prop.container in container_entities:
            read.container[(prop.container, prop.container_property)].append(
                ReadContainerProperty(prop.container_property, row_no, container_prop)
            )
            return None
        # The container entity was not found in the containers table. This could either be because the container
        # is missing, or because either the container entity in the Properties table and the Container table are
        # specified with/without default space/version inconsistently.
        try:
            container_ref = ContainerReference.model_validate(self._create_container_ref(prop.container))
        except ValidationError:
            # Error will be reported when reading the containers
            return None
        if container_ref in container_ref_by_entity:
            container_ref_entity = container_ref_by_entity[container_ref]
            read.container[(container_ref_entity, prop.container_property)].append(
                ReadContainerProperty(prop.container_property, row_no, container_prop)
            )
        # Container can be in CDF, this will be reported by a validator later.
        return None

    def _process_index(
        self,
        prop: DMSProperty,
        read: ReadProperties,
        row_no: int,
        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
        container_entities: set[ParsedEntity],
    ) -> None:
        if prop.index is None or prop.container_property is None or prop.container is None:
            return

        loc = (self.Sheets.properties, row_no, self.PropertyColumns.index)
        for index in prop.index:
            data = self.read_index(index, prop.container_property)
            created = self._validate_adapter(IndexAdapter, data, loc)
            if created is None:
                continue
            order = self._read_order(index.properties, loc)

            if prop.container in container_entities:
                read.indices[(prop.container, index.suffix)].append(
                    ReadIndex(
                        prop_id=prop.container_property,
                        order=order,
                        row_no=row_no,
                        index_id=index.suffix,
                        index=created,
                    )
                )
                continue
            # The container entity was not found in the containers table. This could either be because the
            # container is missing, or because either the container entity in the Properties table and the
            # Container table are specified with/without default space/version inconsistently.

            try:
                container_ref = ContainerReference.model_validate(self._create_container_ref(prop.container))
            except ValidationError:
                # Error will be reported when reading the containers
                continue
            if container_ref in container_ref_by_entity:
                container_ref_entity = container_ref_by_entity[container_ref]
                read.indices[(container_ref_entity, index.suffix)].append(
                    ReadIndex(
                        prop_id=prop.container_property,
                        order=order,
                        row_no=row_no,
                        index_id=index.suffix,
                        index=created,
                    )
                )
            else:
                # Error is reported when reading the property.
                ...

    def _read_order(self, properties: dict[str, Any], loc: tuple[str | int, ...]) -> int | None:
        if "order" not in properties:
            return None
        try:
            return int(properties["order"])
        except ValueError:
            self.errors.append(
                ModelSyntaxError(
                    message=f"In {self.source.location(loc)} invalid order value '{properties['order']}'. "
                    "Must be an integer."
                )
            )
            return None

    @staticmethod
    def read_index(index: ParsedEntity, prop_id: str) -> dict[str, Any]:
        return {
            "indexType": index.prefix,
            "properties": [prop_id],
            **index.properties,
        }

    def _process_constraint(
        self,
        prop: DMSProperty,
        read: ReadProperties,
        row_no: int,
        container_ref_by_entity: dict[ContainerReference, ParsedEntity],
        container_entities: set[ParsedEntity],
    ) -> None:
        if prop.constraint is None or prop.container_property is None or prop.container is None:
            return
        loc = (self.Sheets.properties, row_no, self.PropertyColumns.constraint)
        for constraint in prop.constraint:
            data = self.read_property_constraint(constraint, prop.container_property)
            created = self._validate_adapter(ConstraintAdapter, data, loc)
            if created is None:
                continue
            order = self._read_order(constraint.properties, loc)

            if prop.container in container_entities:
                read.constraints[(prop.container, constraint.suffix)].append(
                    ReadConstraint(
                        prop_id=prop.container_property,
                        order=order,
                        constraint_id=constraint.suffix,
                        row_no=row_no,
                        constraint=created,
                    )
                )
                continue
            # The container entity was not found in the containers table. This could either be because the
            # container is missing, or because either the container entity in the Properties table and the
            # Container table are specified with/without default space/version inconsistently.
            try:
                container_ref = ContainerReference.model_validate(self._create_container_ref(prop.container))
            except ValidationError:
                # Error will be reported when reading the containers
                continue
            if container_ref in container_ref_by_entity:
                container_ref_entity = container_ref_by_entity[container_ref]
                read.constraints[(container_ref_entity, constraint.suffix)].append(
                    ReadConstraint(
                        prop_id=prop.container_property,
                        order=order,
                        constraint_id=constraint.suffix,
                        row_no=row_no,
                        constraint=created,
                    )
                )
            else:
                # Error is reported when reading the property.
                ...

    @staticmethod
    def read_property_constraint(constraint: ParsedEntity, prop_id: str) -> dict[str, Any]:
        return {"constraintType": constraint.prefix, "properties": [prop_id], **constraint.properties}

    def read_view_property(self, prop: DMSProperty, loc: tuple[str | int, ...]) -> dict[str, Any]:
        """Reads a single view property from a given row in the properties table.

        The type of property (core, edge, reverse direct relation) is determined based on the connection column
        as follows:
        1. If the connection is empty or 'direct' it is a core property.
        2. If the connection is 'edge' it is an edge property.
        3. If the connection is 'reverse' it is a reverse direct relation property
        4. Otherwise, it is an error.

        Args:
            prop (DMSProperty): The property row to read.
            loc (tuple[str | int, ...]): The location of the property in the source for error reporting.

        Returns:
            ViewRequestProperty: The parsed view property.
        """

        if prop.connection is None or prop.connection.suffix == "direct":
            return self.read_core_view_property(prop)
        elif prop.connection.suffix == "edge":
            return self.read_edge_view_property(prop, loc)
        elif prop.connection.suffix == "reverse":
            return self.read_reverse_direct_relation_view_property(prop)
        else:
            self.errors.append(
                ModelSyntaxError(
                    message=f"In {self.source.location(loc)} invalid connection type '{prop.connection.suffix}'. "
                )
            )
            return {}

    def read_core_view_property(self, prop: DMSProperty) -> dict[str, Any]:
        source: dict[str, str | None] | None = None
        if prop.connection is not None and prop.value_type.suffix != self.CELL_MISSING:
            source = self._create_view_ref(prop.value_type)

        return dict(
            connectionType="primary_property",
            name=prop.name,
            description=prop.description,
            container=self._create_container_ref(prop.container),
            containerPropertyIdentifier=prop.container_property,
            source=source,
        )

    def read_edge_view_property(self, prop: DMSProperty, loc: tuple[str | int, ...]) -> dict[str, Any]:
        if prop.connection is None:
            return {}
        edge_source: dict[str, str | None] | None = None
        if "edgeSource" in prop.connection.properties:
            edge_source = self._create_view_ref_unparsed(
                prop.connection.properties["edgeSource"], (*loc, self.PropertyColumns.connection, "edgeSource")
            )
        return dict(
            connectionType="single_edge_connection" if prop.max_count == 1 else "multi_edge_connection",
            name=prop.name,
            description=prop.description,
            source=self._create_view_ref(prop.value_type),
            type=self._create_node_ref_unparsed(
                prop.connection.properties.get("type"),
                prop.view,
                prop.view_property,
                (*loc, self.PropertyColumns.connection, "type"),
            ),
            edgeSource=edge_source,
            direction=prop.connection.properties.get("direction", "outwards"),
        )

    def read_reverse_direct_relation_view_property(
        self,
        prop: DMSProperty,
    ) -> dict[str, Any]:
        if prop.connection is None:
            return {}
        view_ref = self._create_view_ref(prop.value_type)
        return dict(
            connectionType="single_reverse_direct_relation" if prop.max_count == 1 else "multi_reverse_direct_relation",
            name=prop.name,
            description=prop.description,
            source=view_ref,
            through={
                "source": view_ref,
                "identifier": prop.connection.properties.get("property"),
            },
        )

    def read_container_property(
        self, prop: DMSProperty, enum_collections: dict[str, dict[str, Any]], loc: tuple[str | int, ...]
    ) -> dict[str, Any]:
        data_type = self._read_data_type(prop, enum_collections, loc)
        return dict(
            immutable=prop.immutable,
            nullable=prop.min_count == 0 or prop.min_count is None,
            autoIncrement=prop.auto_increment,
            defaultValue=prop.default,
            description=prop.container_property_description,
            name=prop.container_property_name,
            type=data_type,
        )

    def _read_data_type(
        self, prop: DMSProperty, enum_collections: dict[str, dict[str, Any]], loc: tuple[str | int, ...]
    ) -> dict[str, Any]:
        # Implementation to read the container property type from DMSProperty
        is_list = None if prop.max_count is None else prop.max_count > 1
        max_list_size: int | None = None
        if is_list and prop.max_count is not None:
            max_list_size = prop.max_count

        args: dict[str, Any] = {
            "maxListSize": max_list_size,
            "list": is_list,
            "type": "direct" if prop.connection is not None else prop.value_type.suffix,
        }
        args.update(prop.value_type.properties)
        if "container" in args and prop.connection is not None:
            # Direct relation constraint.
            args["container"] = self._create_container_ref_unparsed(
                prop.connection.properties["container"], (*loc, self.PropertyColumns.connection, "container")
            )
        if args["type"] == "enum" and "collection" in prop.value_type.properties:
            args["values"] = enum_collections.get(prop.value_type.properties["collection"], {})
        return args

    def read_containers(
        self, containers: list[DMSContainer], properties: ProcessedProperties
    ) -> list[ContainerRequest]:
        # Implementation to read containers from DMSContainer list
        containers_requests: list[ContainerRequest] = []
        rows_by_seen: dict[ParsedEntity, list[int]] = defaultdict(list)
        for row_no, container in enumerate(containers):
            property_constraints = properties.constraints.get(container.container, {})
            require_constraints = self.read_container_constraints(container, row_no)
            if conflict := set(property_constraints.keys()).intersection(set(require_constraints.keys())):
                conflict_str = humanize_collection(conflict)
                location_str = self.source.location((self.Sheets.containers, row_no, self.ContainerColumns.constraint))
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {location_str} the container '{container.container!s}' has constraints defined "
                            f"with the same identifier(s) as the uniqueness constraint defined in the "
                            f"{self.Sheets.properties} sheet. Ensure that the identifiers are unique. "
                            f"Conflicting identifiers: {conflict_str}. "
                        )
                    )
                )
            constraints = {**property_constraints, **require_constraints}
            container_request = self._validate_obj(
                ContainerRequest,
                dict(
                    **self._create_container_ref(container.container),
                    usedFor=container.used_for,
                    name=container.name,
                    description=container.description,
                    properties=properties.container[container.container],
                    indexes=properties.indices.get(container.container),
                    constraints=constraints or None,
                ),
                (self.Sheets.containers, row_no),
            )
            if container_request is None:
                continue
            if container.container in rows_by_seen:
                rows_by_seen[container.container].append(row_no)
            else:
                containers_requests.append(container_request)
                rows_by_seen[container.container] = [row_no]
        for entity, rows in rows_by_seen.items():
            if len(rows) > 1:
                rows_str = humanize_collection([self.source.adjust_row_number(self.Sheets.containers, r) for r in rows])
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {self.source.location((self.Sheets.containers,))} the values in "
                            f"column {self.ContainerColumns.container!r} must be unique. "
                            f"Duplicated entries for container '{entity!s}' found in rows {rows_str}."
                        )
                    )
                )
        return containers_requests

    def read_container_constraints(self, container: DMSContainer, row_no: int) -> dict[str, Constraint]:
        constraints: dict[str, Constraint] = {}
        if not container.constraint:
            return constraints
        for entity in container.constraint:
            loc = self.Sheets.containers, row_no, self.ContainerColumns.constraint
            if entity.prefix != "requires":
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {self.source.location(loc)} the constraint '{entity.suffix}' on container "
                            f"'{container.container!s}' has an invalid type '{entity.prefix}'. Only 'requires' "
                            f"constraints are supported at the container level."
                        )
                    )
                )
                continue

            if "require" not in entity.properties:
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {self.source.location(loc)} the constraint '{entity.suffix}' on container "
                            f"'{container.container!s}' is missing the "
                            f"'require' property which is required for container level constraints."
                        )
                    )
                )
                continue
            data = {
                "constraintType": entity.prefix,
                "require": self._create_container_ref_unparsed(entity.properties["require"], loc),
            }
            created = self._validate_adapter(ConstraintAdapter, data, loc)
            if created is None:
                continue
            constraints[entity.suffix] = created
        return constraints

    def read_views(
        self,
        views: list[DMSView],
        properties: dict[ParsedEntity, dict[str, ViewRequestProperty]],
    ) -> tuple[list[ViewRequest], set[ParsedEntity]]:
        views_requests: list[ViewRequest] = []
        rows_by_seen: dict[ParsedEntity, list[int]] = defaultdict(list)
        for row_no, view in enumerate(views):
            view_request = self._validate_obj(
                ViewRequest,
                dict(
                    **self._create_view_ref(view.view),
                    name=view.name,
                    description=view.description,
                    implements=[self._create_view_ref(impl) for impl in view.implements] if view.implements else None,
                    filter=self._create_filter_dict(view.filter, row_no) if view.filter else None,
                    properties=properties.get(view.view, {}),
                ),
                (self.Sheets.views, row_no),
            )
            if view_request is None:
                continue
            if view.view in rows_by_seen:
                rows_by_seen[view.view].append(row_no)
            else:
                views_requests.append(view_request)
                rows_by_seen[view.view] = [row_no]
        for entity, rows in rows_by_seen.items():
            if len(rows) > 1:
                rows_str = humanize_collection([self.source.adjust_row_number(self.Sheets.views, r) for r in rows])
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {self.source.location((self.Sheets.views,))} the values in "
                            f"column {self.ViewColumns.view!r} must be unique. "
                            f"Duplicated entries for view '{entity!s}' found in rows {rows_str}."
                        )
                    )
                )
        return views_requests, set(rows_by_seen.keys())

    def _create_filter_dict(self, filter: TableViewFilter, row_no: int) -> dict[str, Any] | None:
        if isinstance(filter, RAWFilterTableFilter):
            try:
                return json.loads(filter.filter)
            except ValueError as e:
                self.errors.append(
                    ModelSyntaxError(
                        message=(
                            f"In {self.source.location((self.Sheets.views, row_no, self.ViewColumns.filter))} "
                            f"must be valid json. Got error {e!s}"
                        )
                    )
                )
            return None
        elif isinstance(filter, EntityTableFilter):
            return self._create_entity_filter_dict(filter)
        else:
            # This is unreachable due to validation of the TableViewFilter model.
            raise RuntimeError(f"Unknown filter type {filter.__class__.__name__}")

    def _create_entity_filter_dict(self, filter: EntityTableFilter) -> dict[str, Any]:
        """Creates the filter dictionary from an EntityTableFilter."""
        if filter.type == "hasData":
            return {
                "hasData": [{**self._create_container_ref(entity), "type": "container"} for entity in filter.entities]
            }
        elif filter.type == "nodeType":
            if len(filter.entities) == 1:
                return {"equals": {"property": ["node", "type"], "value": self._create_node_ref(filter.entities[0])}}
            else:
                return {
                    "in": {
                        "property": ["node", "type"],
                        "values": [self._create_node_ref(entity) for entity in filter.entities],
                    }
                }
        else:
            # This is unreachable due to validation of the EntityTableFilter model.
            raise RuntimeError(f"Unknown filter type {filter.__class__.__name__}")

    def read_data_model(self, tables: TableDMS, valid_view_entities: set[ParsedEntity]) -> DataModelRequest:
        data: dict[str, Any] = {
            **{meta.key: meta.value for meta in tables.metadata},
            "views": [self._create_view_ref(view.view) for view in tables.views if view.view in valid_view_entities],
        }
        if description := self._create_description_field(data):
            data["description"] = description
        model = self._validate_obj(DataModelRequest, data, (self.Sheets.metadata,), field_name="value")
        if model is None:
            # This is the last step, so we can raise the error here.
            raise DataModelImportException(self.errors) from None
        return model

    def _create_description_field(self, data: dict[str, Any]) -> str | None:
        """DataModelRequest does not have a 'creator' field, this is a special addition that the Neat tables
        format supports (and recommends using). To keep it, Neat adds it to the suffix of the description field.
        """
        if CREATOR_KEY not in data and CREATOR_KEY.title() not in data:
            return None
        creator_val = data.pop(CREATOR_KEY, data.pop(CREATOR_KEY.title(), None))

        if not creator_val:
            return None

        creator = str(creator_val)
        # We do a split/join to clean up any spaces around commas. Ensuring that we have a consistent
        # canonical format.
        cleaned_creator = ", ".join(item.strip() for item in creator.split(","))
        if not cleaned_creator:
            return None
        suffix = f"{CREATOR_MARKER}{cleaned_creator}"
        description = data.get("description", "")
        if len(description) + len(suffix) > DATA_MODEL_DESCRIPTION_MAX_LENGTH:
            description = description[: DATA_MODEL_DESCRIPTION_MAX_LENGTH - len(suffix) - 4] + "..."
        if description:
            description = f"{description} {suffix}"
        else:
            description = suffix
        return description

    def _parse_entity(self, entity: str, loc: tuple[str | int, ...]) -> ParsedEntity | None:
        try:
            parsed = parse_entity(entity)
        except ValueError as e:
            self.errors.append(
                ModelSyntaxError(message=f"In {self.source.location(loc)} failed to parse entity '{entity}': {e!s}")
            )
            return None
        return parsed

    def _create_view_ref_unparsed(self, entity: str, loc: tuple[str | int, ...]) -> dict[str, str | None]:
        parsed = self._parse_entity(entity, loc)
        if parsed is None:
            return dict()
        return self._create_view_ref(parsed)

    def _create_view_ref(self, entity: ParsedEntity | None) -> dict[str, str | None]:
        if entity is None or entity.suffix == "":
            # If no suffix is given, we cannot create a valid reference.
            return dict()
        space, version = entity.prefix, entity.properties.get("version")
        if space == "":
            space = self.default_space
            # Only if default space is used, we can use default version.
            if version is None:
                version = self.default_version
        return {
            "space": space,
            "externalId": entity.suffix,
            "version": version,
        }

    def _create_container_ref_unparsed(self, entity: str, loc: tuple[str | int, ...]) -> dict[str, str]:
        parsed = self._parse_entity(entity, loc)
        if parsed is None:
            return dict()
        return self._create_container_ref(parsed)

    def _create_container_ref(self, entity: ParsedEntity | None) -> dict[str, str]:
        if entity is None or entity.suffix == "":
            # If no suffix is given, we cannot create a valid reference.
            return dict()
        return {
            "space": entity.prefix or self.default_space,
            "externalId": entity.suffix,
        }

    def _create_node_ref_unparsed(
        self, entity: str | None, view: ParsedEntity, view_prop: str, loc: tuple[str | int, ...]
    ) -> dict[str, str | None]:
        if entity is None:
            # Use default
            return self._create_node_ref(None, view, view_prop)
        parsed = self._parse_entity(entity, loc)
        if parsed is None:
            return dict()
        return self._create_node_ref(parsed, view, view_prop)

    @overload
    def _create_node_ref(
        self, entity: ParsedEntity, *, view: None = None, view_prop: None = None
    ) -> dict[str, str | None]: ...

    @overload
    def _create_node_ref(
        self, entity: ParsedEntity | None, view: ParsedEntity, view_prop: str
    ) -> dict[str, str | None]: ...

    def _create_node_ref(
        self, entity: ParsedEntity | None, view: ParsedEntity | None = None, view_prop: str | None = None
    ) -> dict[str, str | None]:
        if entity is None or entity.suffix == "":
            if view is None or view_prop is None:
                return dict()
            # If no suffix is given, we fallback to the view's property
            return {
                "space": view.prefix or self.default_space,
                "externalId": f"{view.suffix}.{view_prop}",
            }
        return {
            "space": entity.prefix or self.default_space,
            "externalId": entity.suffix,
        }

    def _validate_obj(
        self,
        obj: type[T_BaseModel],
        data: dict,
        parent_loc: tuple[str | int, ...],
        field_name: Literal["field", "column", "value"] = "column",
    ) -> T_BaseModel | None:
        try:
            return obj.model_validate(data)
        except ValidationError as e:
            self._add_error_messages(e, parent_loc, field_name=field_name)
            return None

    def _validate_adapter(
        self, adapter: TypeAdapter[T_BaseModel], data: dict[str, Any], parent_loc: tuple[str | int, ...]
    ) -> T_BaseModel | None:
        try:
            return adapter.validate_python(data, strict=True)
        except ValidationError as e:
            self._add_error_messages(e, parent_loc, field_name="column")
            return None

    def _add_error_messages(
        self,
        error: ValidationError,
        parent_loc: tuple[str | int, ...],
        field_name: Literal["field", "column", "value"] = "column",
    ) -> None:
        context = ValidationContext(
            parent_loc=parent_loc,
            humanize_location=self.source.location,
            field_name=field_name,
            missing_required_descriptor="empty" if field_name == "column" else "missing",
        )

        if field_renaming := self.source.field_mapping(parent_loc[0]):
            context.field_renaming = field_renaming

        seen: set[str] = set()
        for error_details in error.errors(include_input=True, include_url=False):
            message = humanize_validation_error(error_details, context)
            if message in seen:
                continue

            seen.add(message)
            self.errors.append(ModelSyntaxError(message=message))
