"""A structure for representing collections of similar objects."""

import hashlib
import json
import logging
from collections import defaultdict
from pathlib import Path
from typing import (
    TYPE_CHECKING,
    Any,
    ClassVar,
    Dict,
    Generic,
    Iterable,
    Iterator,
    List,
    Optional,
    TextIO,
    Tuple,
    Type,
    Union,
)

import numpy as np
from linkml_runtime import SchemaView
from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
from linkml_runtime.linkml_model.meta import ArrayExpression
from pydantic import BaseModel

from linkml_store.api.types import DatabaseType
from linkml_store.index import get_indexer
from linkml_store.utils.format_utils import load_objects, load_objects_from_url
from linkml_store.utils.object_utils import clean_empties
from linkml_store.utils.patch_utils import PatchDict, apply_patches_to_list, patches_from_objects_lists

try:
    from linkml.validator.report import ValidationResult
except ImportError:
    ValidationResult = None

from linkml_store.api.config import CollectionConfig
from linkml_store.api.queries import Query, QueryResult
from linkml_store.index.indexer import Indexer

if TYPE_CHECKING:
    from linkml_store.api.database import Database

logger = logging.getLogger(__name__)

OBJECT = Union[Dict[str, Any], BaseModel, Type]

DEFAULT_FACET_LIMIT = 100
IDENTIFIER = str
FIELD_NAME = str


class Collection(Generic[DatabaseType]):
    """
    A collection is an organized set of objects of the same or similar type.

    - For relational databases, a collection is typically a table
    - For document databases such as MongoDB, a collection is the native type
    - For a file system, a collection could be a single tabular file such as Parquet or CSV.

    Collection objects are typically not created directly - instead they are generated
    from a parent :class:`.Database` object:

    >>> from linkml_store import Client
    >>> client = Client()
    >>> db = client.attach_database("duckdb", alias="test")
    >>> collection = db.create_collection("Person")
    >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
    >>> collection.insert(objs)
    """

    # name: str
    parent: Optional[DatabaseType] = None
    _indexers: Optional[Dict[str, Indexer]] = None
    _initialized: Optional[bool] = None
    # hidden: Optional[bool] = False

    metadata: Optional[CollectionConfig] = None
    default_index_name: ClassVar[str] = "simple"

    def __init__(
        self, name: str, parent: Optional["Database"] = None, metadata: Optional[CollectionConfig] = None, **kwargs
    ):
        self.parent = parent
        if metadata:
            self.metadata = metadata
        else:
            self.metadata = CollectionConfig(type=name, **kwargs)
            if not self.metadata.alias:
                self.metadata.alias = name
            if not self.metadata.type:
                self.metadata.type = name
        # if name is not None and self.metadata.name is not None and name != self.metadata.name:
        #    raise ValueError(f"Name mismatch: {name} != {self.metadata.name}")

    @property
    def hidden(self) -> bool:
        """
        True if the collection is hidden.

        An example of a hidden collection is a collection that indexes another
        collection

        :return: True if the collection is hidden
        """
        # return self.metadata.hidden

    @property
    def target_class_name(self):
        """
        Return the name of the class that this collection represents

        This MUST be a LinkML class name

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("duckdb", alias="test")
        >>> collection = db.create_collection("Person", alias="persons")
        >>> collection.target_class_name
        'Person'

        >>> collection = db.create_collection("Organization")
        >>> collection.target_class_name
        'Organization'
        >>> collection.alias
        'Organization'

        :return: name of the class which members of this collection instantiate
        """
        # TODO: this is a shim layer until we can normalize on this
        if self.metadata.type:
            return self.metadata.type
        return self.alias

    @property
    def alias(self):
        """
        Return the primary name/alias used for the collection.

        This MAY be the name of the LinkML class, but it may be desirable
        to have an alias, for example "persons" which collects all instances
        of class Person.

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("duckdb", alias="test")
        >>> collection = db.create_collection("Person", alias="persons")
        >>> collection.alias
        'persons'

        If no explicit alias is provided, then the target class name is used:

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("duckdb", alias="test")
        >>> collection = db.create_collection("Person")
        >>> collection.alias
        'Person'

        The alias SHOULD be used for Table names in SQL.

        For nested data, the alias SHOULD be used as the key; e.g

        .. code-block:: json

           { "persons": [ { "name": "Alice" }, { "name": "Bob" } ] }

        :return:
        """
        # TODO: this is a shim layer until we can normalize on this
        if self.metadata.alias:
            return self.metadata.alias
        return self.target_class_name

    def replace(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
        """
        Replace entire collection with objects.

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("duckdb", alias="test")
        >>> collection = db.create_collection("Person")
        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
        >>> collection.insert(objs)

        :param objs:
        :param kwargs:
        :return:
        """
        self.delete_where({})
        self.insert(objs, **kwargs)

    def insert(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
        """
        Add one or more objects to the collection.

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("duckdb", alias="test")
        >>> collection = db.create_collection("Person")
        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
        >>> collection.insert(objs)

        :param objs:
        :param kwargs:
        :return:
        """
        raise NotImplementedError

    def index(
        self,
        objs: Union[OBJECT, List[OBJECT]],
        index_name: Optional[str] = None,
        replace: bool = False,
        unique: bool = False,
        **kwargs,
    ) -> None:
        """
        Index objects in the collection.

        :param objs:
        :param index_name:
        :param replace: replace the index, or not
        :param unique: boolean used to declare the index unique or not
        :param kwargs:
        :return:
        """
        raise NotImplementedError

    def upsert(
        self,
        objs: Union[OBJECT, List[OBJECT]],
        filter_fields: List[str],
        update_fields: Union[List[str], None] = None,
        **kwargs,
    ):
        """
        Add one or more objects to the collection.

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("mongodb", alias="test")
        >>> collection = db.create_collection("Person")
        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
        >>> collection.upsert(objs)

        :param objs:
        :param filter_fields: List of field names to use as the filter for matching existing collections.
        :param update_fields: List of field names to include in the update. If None, all fields are updated.
        :param kwargs:

        :return:
        """
        raise NotImplementedError

    def _pre_query_hook(self, query: Optional[Query] = None, **kwargs):
        """
        Pre-query hook.

        This is called before a query is executed. It is used to materialize derivations and indexes.
        :param query:
        :param kwargs:
        :return:
        """
        logger.debug(f"Pre-query hook (state: {self._initialized}; Q= {query}")  # if logging.info, this is very noisy.
        if not self._initialized:
            self._materialize_derivations()
            self._initialized = True

    def _pre_insert_hook(self, objs: List[OBJECT], **kwargs):
        if self.metadata.validate_modifications:
            errors = list(self.iter_validate_collection(objs))
            if errors:
                raise ValueError(f"Validation errors: {errors}")

    def _post_insert_hook(self, objs: List[OBJECT], **kwargs):
        self._initialized = True
        patches = [{"op": "add", "path": "/0", "value": obj} for obj in objs]
        self._broadcast(patches, **kwargs)
        self._post_modification_hook(**kwargs)

    def _post_delete_hook(self, **kwargs):
        self._post_modification_hook(**kwargs)

    def _post_modification_hook(self, **kwargs):
        for indexer in self.indexers.values():
            ix_collection_name = self.get_index_collection_name(indexer)
            ix_collection = self.parent.get_collection(ix_collection_name)
            # Currently updating the source triggers complete reindexing
            # TODO: make this more efficient by only deleting modified
            ix_collection.delete_where({})

    def delete(self, objs: Union[OBJECT, List[OBJECT]], **kwargs) -> Optional[int]:
        """
        Delete one or more objects from the collection.

        First let's set up a collection:

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("duckdb", alias="test")
        >>> collection = db.create_collection("Person")
        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
        >>> collection.insert(objs)
        >>> collection.find({}).num_rows
        2

        Now let's delete an object:

        >>> collection.delete(objs[0])
        >>> collection.find({}).num_rows
        1

        Deleting the same object again should have no effect:

        >>> collection.delete(objs[0])
        >>> collection.find({}).num_rows
        1

        :param objs:
        :param kwargs:
        :return:
        """
        raise NotImplementedError

    def delete_where(self, where: Optional[Dict[str, Any]] = None, missing_ok=True, **kwargs) -> Optional[int]:
        """
        Delete objects that match a query.

        First let's set up a collection:

        >>> from linkml_store import Client
        >>> client = Client()
        >>> db = client.attach_database("duckdb", alias="test")
        >>> collection = db.create_collection("Person")
        >>> objs = [{"id": "P1", "name": "John", "age_in_years": 30}, {"id": "P2", "name": "Alice", "age_in_years": 25}]
        >>> collection.insert(objs)

        Now let's delete an object:

        >>> collection.delete_where({"id": "P1"})
        >>> collection.find({}).num_rows
        1

        Match everything:

        >>> collection.delete_where({})
        >>> collection.find({}).num_rows
        0

        :param where: where conditions
        :param missing_ok: if True, do not raise an error if the collection does not exist
        :param kwargs:
        :return: number of objects deleted (or -1 if unsupported)
        """
        raise NotImplementedError

    def update(self, objs: Union[OBJECT, List[OBJECT]], **kwargs):
        """
        Update one or more objects in the collection.

        :param objs:
        :param kwargs:
        :return:
        """
        raise NotImplementedError

    def _create_query(self, **kwargs) -> Query:
        return Query(from_table=self.alias, **kwargs)

    def query(self, query: Query, **kwargs) -> QueryResult:
        """
        Run a query against the collection.

        First let's load a collection:

        >>> from linkml_store import Client
        >>> from linkml_store.utils.format_utils import load_objects
        >>> client = Client()
        >>> db = client.attach_database("duckdb")
        >>> collection = db.create_collection("Country")
        >>> objs = load_objects("tests/input/countries/countries.jsonl")
        >>> collection.insert(objs)

        Now let's run a query:

        TODO

        :param query:
        :param kwargs:
        :return:
        """
        self._pre_query_hook()
        return self.parent.query(query, **kwargs)

    def query_facets(
        self, where: Optional[Dict] = None, facet_columns: List[str] = None, facet_limit=DEFAULT_FACET_LIMIT, **kwargs
    ) -> Dict[str, List[Tuple[Any, int]]]:
        """
        Run a query to get facet counts for one or more columns.

        This function takes a database connection, a Query object, and a list of column names.
        It generates and executes a facet count query for each specified column and returns
        the results as a dictionary where the keys are the column names and the values are
        pandas DataFrames containing the facet counts.

        The facet count query is generated by modifying the original query's WHERE clause
        to exclude conditions directly related to the facet column. This allows for counting
        the occurrences of each unique value in the facet column while still applying the
        other filtering conditions.

        :param con: A DuckDB database connection.
        :param query: A Query object representing the base query.
        :param facet_columns: A list of column names to get facet counts for.
        :param facet_limit:
        :return: A dictionary where keys are column names and values are tuples
                 containing the facet counts for each unique value in the respective column.
        """
        raise NotImplementedError

    def get(self, ids: Optional[List[IDENTIFIER]], **kwargs) -> QueryResult:
        """
        Get one or more objects by ID.

        :param ids:
        :param kwargs:
        :return:
        """
        id_field = self.identifier_attribute_name
        if not id_field:
            raise ValueError(f"No identifier for {self.name}")
        if len(ids) == 1:
            return self.find({id_field: ids[0]})
        else:
            return self.find({id_field: {"$in": ids}})

    def get_one(self, id: IDENTIFIER, **kwargs) -> Optional[OBJECT]:
        """
        Get one object by ID.

        :param id:
        :param kwargs:
        :return:
        """
        if not id:
            raise ValueError("Must pass an ID")
        id_field = self.identifier_attribute_name
        if not id_field:
            raise ValueError(f"No identifier for {self.name}")
        w = {id_field: id}
        qr = self.find(w)
        if qr.num_rows == 1:
            return qr.rows[0]
        return None

    def find(
        self,
        where: Optional[Any] = None,
        select_cols: Optional[List[str]] = None,
        **kwargs,
    ) -> QueryResult:
        """
        Find objects in the collection using a where query.

        As an example, first load a collection:

        >>> from linkml_store import Client
        >>> from linkml_store.utils.format_utils import load_objects
        >>> client = Client()
        >>> db = client.attach_database("duckdb")
        >>> collection = db.create_collection("Country")
        >>> objs = load_objects("tests/input/countries/countries.jsonl")
        >>> collection.insert(objs)

        Now let's find all objects:

        >>> qr = collection.find({})
        >>> qr.num_rows
        20

        We can do a more restrictive query:

        >>> qr = collection.find({"code": "FR"})
        >>> qr.num_rows
        1
        >>> qr.rows[0]["name"]
        'France'


        :param where:
        :param select_cols:
        :param kwargs:
        :return:
        """
        query = self._create_query(
            where_clause=where,
            select_cols=select_cols,
        )
        self._pre_query_hook(query)
        return self.query(query, **kwargs)

    def find_iter(self, where: Optional[Any] = None, page_size=100, **kwargs) -> Iterator[OBJECT]:
        """
        Find objects in the collection using a where query.

        :param where:
        :param kwargs:
        :return:
        """
        total_rows = None
        offset = 0
        if page_size < 1:
            raise ValueError(f"Invalid page size: {page_size}")
        while True:
            qr = self.find(where=where, offset=offset, limit=page_size, **kwargs)
            if total_rows is None:
                total_rows = qr.num_rows
            if not qr.rows:
                return
            for row in qr.rows:
                yield row
            offset += page_size
            if offset >= total_rows:
                break
        return

    def search(
        self,
        query: str,
        where: Optional[Any] = None,
        index_name: Optional[str] = None,
        limit: Optional[int] = None,
        select_cols: Optional[List[str]] = None,
        mmr_relevance_factor: Optional[float] = None,
        **kwargs,
    ) -> QueryResult:
        """
        Search the collection using a text-based index index.

        Example:

        >>> from linkml_store import Client
        >>> from linkml_store.utils.format_utils import load_objects
        >>> client = Client()
        >>> db = client.attach_database("duckdb")
        >>> collection = db.create_collection("Country")
        >>> objs = load_objects("tests/input/countries/countries.jsonl")
        >>> collection.insert(objs)

        Now let's index, using the simple trigram-based index

        >>> index = get_indexer("simple")
        >>> _ = collection.attach_indexer(index)

        Now let's find all objects:

        >>> qr = collection.search("France")
        >>> score, top_obj = qr.ranked_rows[0]
        >>> assert score > 0.1
        >>> top_obj["code"]
        'FR'

        :param query:
        :param where:
        :param index_name:
        :param limit:
        :param select_cols:
        :param kwargs:
        :return:
        """
        self._pre_query_hook()
        if index_name is None:
            if len(self.indexers) == 1:
                index_name = list(self.indexers.keys())[0]
            else:
                logger.warning("Multiple indexes found. Using default index.")
                index_name = self.default_index_name
        ix_coll = self.parent.get_collection(self._index_collection_name(index_name))
        if index_name not in self.indexers:
            logger.debug(f"Indexer not found: {index_name} -- creating")
            ix = get_indexer(index_name)
            if not self._indexers:
                self._indexers = {}
            self._indexers[index_name] = ix
        ix = self.indexers.get(index_name)
        if not ix:
            raise ValueError(f"No index named {index_name}")
        logger.debug(f"Using indexer {type(ix)} with name {index_name}")
        if ix_coll.size() == 0:
            logger.info(f"Index {index_name} is empty; indexing all objects")
            all_objs = self.find(limit=-1).rows
            if all_objs:
                # print(f"Index {index_name} is empty; indexing all objects {len(all_objs)}")
                self.index_objects(all_objs, index_name, replace=True, **kwargs)
                assert ix_coll.size() > 0
        qr = ix_coll.find(where=where, limit=-1, **kwargs)
        index_col = ix.index_field

        # TODO: optimize this for large indexes
        def row2array(row):
            v = row[index_col]
            if isinstance(v, str):
                # sqlite stores arrays as strings
                v = json.loads(v)
            return np.array(v, dtype=float)

        vector_pairs = [(row, row2array(row)) for row in qr.rows]
        results = ix.search(query, vector_pairs, limit=limit, mmr_relevance_factor=mmr_relevance_factor, **kwargs)
        for r in results:
            del r[1][index_col]
        if select_cols:
            new_results = []
            for r in results:
                new_results.append((r[0], {k: v for k, v in r[1].items() if k in select_cols}))
            results = new_results
        new_qr = QueryResult(num_rows=len(results))
        new_qr.ranked_rows = results
        new_qr.rows = [r[1] for r in results]
        return new_qr

    def group_by(
        self,
        group_by_fields: List[str],
        inlined_field="objects",
        agg_map: Optional[Dict[str, str]] = None,
        where: Optional[Dict] = None,
        **kwargs,
    ) -> QueryResult:
        """
        Group objects in the collection by a column.

        :param group_by:
        :param where:
        :param kwargs:
        :return:
        """
        if isinstance(group_by_fields, str):
            group_by_fields = [group_by_fields]
        df = self.find(where=where, limit=-1).rows_dataframe

        # Handle the case where agg_map is None
        if agg_map is None:
            agg_map = {}

        pk_fields = agg_map.get("first", []) + group_by_fields
        list_fields = agg_map.get("list", [])
        if not list_fields:
            list_fields = [a for a in df.columns if a not in pk_fields]

        grouped_objs = defaultdict(list)
        for _, row in df.iterrows():
            pk = tuple(row[pk_fields])
            grouped_objs[pk].append({k: row[k] for k in list_fields})
        results = []
        for pk, objs in grouped_objs.items():
            top_obj = {k: v for k, v in zip(pk_fields, pk)}
            top_obj[inlined_field] = objs
            results.append(top_obj)
        r = QueryResult(num_rows=len(results), rows=results)
        return r

    @property
    def is_internal(self) -> bool:
        """
        Check if the collection is internal.

        Internal collections are hidden by default. Examples of internal collections
        include shadow "index" collections

        :return:
        """
        if not self.alias:
            raise ValueError(f"Collection has no alias: {self} // {self.metadata}")
        return self.alias.startswith("internal__")

    def exists(self) -> Optional[bool]:
        """
        Check if the collection exists.

        :return:
        """
        cd = self.class_definition()
        return cd is not None and cd.attributes

    def load_from_source(self, load_if_exists=False):
        """
        Load objects from the source location.

        :param load_if_exists:
        :return:
        """
        if not load_if_exists and self.exists():
            return
        metadata = self.metadata
        if metadata.source:
            source = metadata.source
            kwargs = source.arguments or {}
            if source.local_path:
                objects = load_objects(
                    metadata.source.local_path,
                    format=source.format,
                    expected_type=source.expected_type,
                    compression=source.compression,
                    select_query=source.select_query,
                    **kwargs,
                )
            elif metadata.source.url:
                objects = load_objects_from_url(
                    metadata.source.url,
                    format=source.format,
                    expected_type=source.expected_type,
                    compression=source.compression,
                    select_query=source.select_query,
                    **kwargs,
                )
            else:
                raise ValueError("No source local_path or url provided")
            self.insert(objects)

    def _check_if_initialized(self) -> bool:
        return self._initialized

    def _materialize_derivations(self, **kwargs):
        metadata = self.metadata
        if not metadata.derived_from:
            logger.info(f"No metadata for {self.alias}; no derivations")
            return
        if self._check_if_initialized():
            logger.info(f"Already initialized {self.alias}; no derivations")
            return
        parent_db = self.parent
        client = parent_db.parent
        # cd = self.class_definition()
        for derivation in metadata.derived_from:
            # TODO: optimize this; utilize underlying engine
            logger.info(f"Deriving from {derivation}")
            if derivation.database:
                db = client.get_database(derivation.database)
            else:
                db = parent_db
            if derivation.collection:
                coll = db.get_collection(derivation.collection)
            else:
                coll = self
            coll.class_definition()
            source_obj_iter = coll.find_iter(derivation.where or {})
            mappings = derivation.mappings
            if not mappings:
                raise ValueError(f"No mappings for {self.name}")
            target_class_name = self.target_class_name
            from linkml_map.session import Session

            session = Session()
            session.set_source_schema(db.schema_view.schema)
            session.set_object_transformer(
                {
                    "class_derivations": {
                        target_class_name: {
                            "populated_from": coll.target_class_name,
                            "slot_derivations": mappings,
                        },
                    }
                },
            )
            logger.debug(f"Session Spec: {session.object_transformer}")
            tr_objs = []
            for source_obj in source_obj_iter:
                tr_obj = session.transform(source_obj, source_type=coll.target_class_name)
                tr_objs.append(tr_obj)
            if not tr_objs:
                raise ValueError(f"No objects derived from {coll.name}")
            self.insert(tr_objs)
            self.commit()

    def size(self) -> int:
        """
        Return the number of objects in the collection.

        :return: The number of objects in the collection.
        """
        return self.find({}, limit=1).num_rows

    def rows_iter(self) -> Iterable[OBJECT]:
        """
        Return an iterator over the objects in the collection.

        :return:
        """
        yield from self.find({}, limit=-1).rows

    @property
    def rows(self) -> List[OBJECT]:
        """
        Return a list of objects in the collection.

        :return:
        """
        return list(self.rows_iter())

    def ranked_rows(self) -> List[Tuple[float, OBJECT]]:
        """
        Return a list of objects in the collection, with scores.
        """
        return [(n, obj) for n, obj in enumerate(self.rows_iter())]

    def attach_indexer(
        self, index: Union[Indexer, str], name: Optional[str] = None, auto_index=True, **kwargs
    ) -> Indexer:
        """
        Attach an index to the collection.

        As an example, first let's create a collection in a database:

        >>> from linkml_store import Client
        >>> from linkml_store.utils.format_utils import load_objects
        >>> client = Client()
        >>> db = client.attach_database("duckdb")
        >>> collection = db.create_collection("Country")
        >>> objs = load_objects("tests/input/countries/countries.jsonl")
        >>> collection.insert(objs)

        We will create two indexes - one that indexes the whole object
        (default behavior), the other one indexes the name only

        >>> full_index = get_indexer("simple")
        >>> full_index.name = "full"
        >>> name_index = get_indexer("simple", text_template="{name}")
        >>> name_index.name = "name"
        >>> _ = collection.attach_indexer(full_index)
        >>> _ = collection.attach_indexer(name_index)

        Now let's find objects using the full index, using the string "France".
        We expect the country France to be the top hit, but the score will
        be less than zero because we did not match all fields in the object.

        >>> qr = collection.search("France", index_name="full")
        >>> score, top_obj = qr.ranked_rows[0]
        >>> assert score > 0.1
        >>> assert score < 0.5
        >>> top_obj["code"]
        'FR'

        Now using the name index

        >>> qr = collection.search("France", index_name="name")
        >>> score, top_obj = qr.ranked_rows[0]
        >>> assert score > 0.99
        >>> top_obj["code"]
        'FR'

        :param index:
        :param name:
        :param auto_index: Automatically index all objects in the collection
        :param kwargs:
        :return:
        """
        if isinstance(index, str):
            index = get_indexer(index)
        if name:
            index.name = name
        if not index.name:
            index.name = type(index).__name__.lower()
        index_name = index.name
        if not index_name:
            raise ValueError("Index must have a name")
        if not self._indexers:
            self._indexers = {}
        self._indexers[index_name] = index
        if auto_index:
            all_objs = self.find(limit=-1).rows
            logger.info(f"Auto-indexing {len(all_objs)} objects")
            self.index_objects(all_objs, index_name, replace=True, **kwargs)
        return index

    def get_index_collection_name(self, indexer: Indexer) -> str:
        return self._index_collection_name(indexer.name)

    def _index_collection_name(self, index_name: str) -> str:
        """
        Create a name for a special collection that holds index data

        :param index_name:
        :param indexer:
        :return:
        """
        return f"internal__index__{self.alias}__{index_name}"

    def index_objects(self, objs: List[OBJECT], index_name: str, replace=False, **kwargs):
        """
        Index a list of objects using a specified index.

        By default, the indexed objects will be stored in a shadow
        collection in the same database, with additional fields for the index vector

        TODO: Support batch_size parameter for processing large collections
        TODO: Implement parallel indexing for multiple objects
        TODO: Add progress reporting for long-running index operations
        TODO: Support incremental indexing (only index new/changed items)

        :param objs:
        :param index_name: e.g. simple, llm
        :param replace:
        :param kwargs:
        :return:
        """
        ix = self._indexers.get(index_name, None)
        if not ix:
            raise ValueError(f"No index named {index_name}")
        ix_coll_name = self._index_collection_name(index_name)
        ix_coll = self.parent.get_collection(ix_coll_name, create_if_not_exists=True)
        if not ix_coll.metadata:
            ix_coll.metadata = CollectionConfig()
        if not ix_coll.metadata.additional_properties:
            ix_coll.metadata.additional_properties = {}
        #for k in ["name", "index_type", "index_field", "index_value_field"]:
        #    ix_coll.metadata.additional_properties[k] = getattr(ix, k)
        for k, v in ix.model_dump().items():
            ix_coll.metadata.additional_properties[k] = v
        ix_coll.store_metadata()
        # TODO: Process vectors in batches rather than all at once
        vectors = [list(float(e) for e in v) for v in ix.objects_to_vectors(objs)]
        objects_with_ix = []
        index_col = ix.index_field
        # TODO: implement this
        index_value_col = ix.index_value_field
        for obj, vector in zip(objs, vectors):
            # TODO: id field
            objects_with_ix.append({**obj, **{index_col: vector}})
        if replace:
            schema = self.parent.schema_view.schema
            logger.info(f"Checking if {ix_coll_name} is in {schema.classes.keys()}")
            if ix_coll_name in schema.classes:
                ix_coll.delete_where()

        # TODO: Use bulk insert operations for better performance
        logger.info(f"Inserting {len(objects_with_ix)} objects into {ix_coll_name}")
        ix_coll.insert(objects_with_ix, **kwargs)
        ix_coll.commit()

    def list_index_names(self) -> List[str]:
        """
        Return a list of index names

        :return:
        """
        return list(self._indexers.keys())

    @property
    def indexers(self) -> Dict[str, Indexer]:
        """
        Return a list of indexers

        :return:
        """
        return self._indexers if self._indexers else {}

    def peek(self, limit: Optional[int] = None) -> QueryResult:
        """
        Return the first N objects in the collection

        :param limit:
        :return:
        """
        q = self._create_query()
        return self.query(q, limit=limit)

    def class_definition(self) -> Optional[ClassDefinition]:
        """
        Return the class definition for the collection.

        If no schema has been explicitly set, and the native database does not
        have a schema, then a schema will be induced from the objects in the collection.

        :return:
        """
        sv: SchemaView = self.parent.schema_view
        if sv:
            cls = sv.get_class(self.target_class_name)
            # if not cls:
            #     logger.warning(f"{self.target_class_name} not in {sv.all_classes().keys()} ")
            # cls = sv.schema.classes[self.target_class_name]
            if cls and not cls.attributes:
                if not sv.class_induced_slots(cls.name):
                    for att in self._induce_attributes():
                        cls.attributes[att.name] = att
                    sv.set_modified()
            return cls
        return None

    def _induce_attributes(self) -> List[SlotDefinition]:
        result = self.find({}, limit=-1)
        cd = self.induce_class_definition_from_objects(result.rows, max_sample_size=None)
        return list(cd.attributes.values())

    @property
    def identifier_attribute_name(self) -> Optional[str]:
        """
        Return the name of the identifier attribute for the collection.

        AKA the primary key.

        :return: The name of the identifier attribute, if one exists.
        """
        cd = self.class_definition()
        if cd:
            for att in self.parent.schema_view.class_induced_slots(cd.name):
                if att.identifier:
                    return att.name
        return None

    def set_identifier_attribute_name(self, name: str):
        """
        Set the name of the identifier attribute for the collection.

        AKA the primary key.

        :param name: The name of the identifier attribute.
        """
        cd = self.class_definition()
        if not cd:
            raise ValueError(f"Cannot find class definition for {self.target_class_name}")
        id_att = None
        candidates = []
        sv: SchemaView = self.parent.schema_view
        cls = sv.get_class(cd.name)
        existing_id_slot = sv.get_identifier_slot(cls.name)
        if existing_id_slot:
            if existing_id_slot.name == name:
                return
            existing_id_slot.identifier = False
        for att in cls.attributes.values():
            candidates.append(att.name)
            if att.name == name:
                att.identifier = True
                id_att = att
            else:
                att.identifier = False
        if not id_att:
            raise ValueError(f"No attribute found with name {name} in {candidates}")
        sv.set_modified()

    def object_identifier(self, obj: OBJECT, auto=True) -> Optional[IDENTIFIER]:
        """
        Return the identifier for an object.

        :param obj:
        :param auto: If True, generate an identifier if one does not exist.
        :return:
        """
        pk = self.identifier_attribute_name
        if pk in obj:
            return obj[pk]
        elif auto:
            # TODO: use other unique keys if no primary key
            as_str = str(obj)
            md5 = hashlib.md5(as_str.encode()).hexdigest()
            return md5
        else:
            return None

    def induce_class_definition_from_objects(
        self, objs: List[OBJECT], max_sample_size: Optional[int] = None
    ) -> ClassDefinition:
        """
        Induce a class definition from a list of objects.

        This uses a heuristic procedure to infer the class definition from a list of objects.
        In general it is recommended you explicitly provide a schema.

        :param objs:
        :param max_sample_size:
        :return:
        """
        # TODO: use schemaview
        if max_sample_size is None:
            max_sample_size = 10
        if not self.target_class_name:
            raise ValueError(f"No target_class_name for {self.alias}")
        cd = ClassDefinition(self.target_class_name)
        keys = defaultdict(list)
        for obj in objs[0:max_sample_size]:
            if isinstance(obj, BaseModel):
                obj = obj.model_dump()
            if not isinstance(obj, dict):
                logger.warning(f"Skipping non-dict object: {obj}")
                continue
            for k, v in obj.items():
                keys[k].append(v)
        for k, vs in keys.items():
            if k == "_id":
                continue
            multivalueds = []
            inlineds = []
            rngs = []
            exact_dimensions_list = []
            for v in vs:
                if v is None:
                    continue
                if isinstance(v, np.ndarray):
                    rngs.append("float")
                    exact_dimensions_list.append(v.shape)
                    break
                if isinstance(v, list):
                    # sample first item. TODO: more robust strategy
                    v = v[0] if v else None
                    multivalueds.append(True)
                elif isinstance(v, dict):
                    pass
                    # TODO: check if this is a nested object or key-value list
                    # v = list(v.values())[0]
                    # multivalueds.append(True)
                else:
                    multivalueds.append(False)
                if not v:
                    continue
                if isinstance(v, str):
                    rng = "string"
                elif isinstance(v, bool):
                    rng = "boolean"
                elif isinstance(v, int):
                    rng = "integer"
                elif isinstance(v, float):
                    rng = "float"
                elif isinstance(v, dict):
                    rng = None
                    inlineds.append(True)
                else:
                    # raise ValueError(f"No mappings for {type(v)} // v={v}")
                    rng = None
                    inlineds.append(False)
                rngs.append(rng)
            multivalued = any(multivalueds)
            inlined = any(inlineds)
            if multivalued and False in multivalueds:
                logger.info(f"Mixed list non list: {vs} // inferred= {multivalueds}")
            # if not rngs:
            #    raise AssertionError(f"Empty rngs for {k} = {vs}")
            rng = rngs[0] if rngs else None
            for other_rng in rngs:
                coercions = {
                    ("integer", "float"): "float",
                }
                if rng != other_rng:
                    if (rng, other_rng) in coercions:
                        rng = coercions[(rng, other_rng)]
                    elif (other_rng, rng) in coercions:
                        rng = coercions[(other_rng, rng)]
                    else:
                        raise ValueError(f"Conflict: {rng} != {other_rng} for {vs}")
            logger.debug(f"Inducing {k} as {rng} {multivalued} {inlined}")
            inlined_as_list = inlined and multivalued
            cd.attributes[k] = SlotDefinition(
                k, range=rng, multivalued=multivalued, inlined=inlined, inlined_as_list=inlined_as_list
            )
            if exact_dimensions_list:
                array_expr = ArrayExpression(exact_number_dimensions=len(exact_dimensions_list[0]))
                cd.attributes[k].array = array_expr
        sv = self.parent.schema_view
        sv.schema.classes[self.target_class_name] = cd
        sv.set_modified()
        return cd

    def import_data(self, location: Union[Path, str, TextIO], **kwargs):
        """
        Import data from a file or stream

        :param location:
        :param kwargs:
        :return:
        """
        raise NotImplementedError

    def export_data(self, location: Union[Path, str, TextIO], **kwargs):
        """
        Export data to a file or stream

        :param location:
        :param kwargs:
        :return:
        """
        raise NotImplementedError

    def apply_patches(self, patches: List[PatchDict], **kwargs):
        """
        Apply a patch to the collection.

        Patches conform to the JSON Patch format.

        :param patches:
        :param kwargs:
        :return:
        """
        all_objs = self.find(limit=-1).rows
        primary_key = self.identifier_attribute_name
        if not primary_key:
            raise ValueError(f"No primary key for {self.target_class_name}")
        new_objs = apply_patches_to_list(all_objs, patches, primary_key=primary_key, **kwargs)
        self.replace(new_objs)

    def diff(self, other: "Collection", **kwargs) -> List[PatchDict]:
        """
        Diff two collections.

        :param other: The collection to diff against
        :param kwargs:
        :return:
        """
        src_objs = self.find(limit=-1).rows
        tgt_objs = other.find(limit=-1).rows
        primary_key = self.identifier_attribute_name
        if not primary_key:
            raise ValueError(f"No primary key for {self.target_class_name}")
        patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)
        return patches_from_objects_lists(src_objs, tgt_objs, primary_key=primary_key)

    def iter_validate_collection(
        self, objects: Optional[Iterable[OBJECT]] = None, **kwargs
    ) -> Iterator["ValidationResult"]:
        """
        Validate the contents of the collection

        :param kwargs:
        :param objects: objects to validate
        :return: iterator over validation results
        """
        from linkml.validator import JsonschemaValidationPlugin, Validator

        validation_plugins = [JsonschemaValidationPlugin(closed=True)]
        validator = Validator(self.parent.schema_view.schema, validation_plugins=validation_plugins)
        cd = self.class_definition()
        if not cd:
            raise ValueError(f"Cannot find class definition for {self.target_class_name}")
        type_designator = None
        for att in self.parent.schema_view.class_induced_slots(cd.name):
            if att.designates_type:
                type_designator = att.name
        class_name = cd.name
        if objects is None:
            objects = self.find_iter(**kwargs)
        for obj in objects:
            obj = clean_empties(obj)
            v_class_name = class_name
            if type_designator is not None:
                # TODO: move type designator logic to core linkml
                this_class_name = obj.get(type_designator)
                if this_class_name:
                    if ":" in this_class_name:
                        this_class_name = this_class_name.split(":")[-1]
                    v_class_name = this_class_name
            yield from validator.iter_results(obj, v_class_name)

    def commit(self):
        """
        Commit changes to the collection.

        :return:
        """
        pass

    def _broadcast(self, *args, **kwargs):
        self.parent.broadcast(self, *args, **kwargs)

    def store_metadata(self, replace=True):
        """
        Store the metadata for the collection.
        """
        if not self.metadata:
            return
        this_collection_name = self.alias
        metadata_collection_name = f"{this_collection_name}__metadata"
        metadata_collection = self.parent.get_collection(metadata_collection_name, create_if_not_exists=True)
        metadata_dict = self.metadata.model_dump()
        if replace:
            metadata_collection.replace(metadata_dict)
        else:
            metadata_collection.insert(metadata_dict)
