"""Database logic."""
import asyncio
import logging
import os
from base64 import urlsafe_b64decode, urlsafe_b64encode
from copy import deepcopy
from typing import Any, Iterable, Type

import attr
import orjson
from fastapi import HTTPException
from opensearchpy import RequestError, exceptions, helpers
from opensearchpy.helpers.query import Q
from opensearchpy.helpers.search import Search
from starlette.requests import Request

import stac_fastapi.sfeos_helpers.filter as filter_module
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, get_bool_env
from stac_fastapi.extensions.core.transaction.request import (
    PartialCollection,
    PartialItem,
    PatchOperation,
)
from stac_fastapi.opensearch.config import (
    AsyncOpensearchSettings as AsyncSearchSettings,
)
from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
from stac_fastapi.sfeos_helpers.database import (
    ItemAlreadyExistsError,
    add_bbox_shape_to_collection,
    apply_collections_bbox_filter_shared,
    apply_collections_datetime_filter_shared,
    apply_collections_free_text_filter_shared,
    apply_free_text_filter_shared,
    apply_intersects_filter_shared,
    check_item_exists_in_alias,
    check_item_exists_in_alias_sync,
    create_index_templates_shared,
    delete_item_index_shared,
    get_queryables_mapping_shared,
    index_alias_by_collection_id,
    mk_actions,
    mk_item_id,
    populate_sort_shared,
    retry_on_connection_error,
    retry_on_datetime_not_found,
    return_date,
    validate_refresh,
)
from stac_fastapi.sfeos_helpers.database.query import (
    ES_MAX_URL_LENGTH,
    add_collections_to_body,
)
from stac_fastapi.sfeos_helpers.database.utils import (
    add_hidden_filter,
    merge_to_operations,
    operations_to_script,
)
from stac_fastapi.sfeos_helpers.mappings import (
    AGGREGATION_MAPPING,
    COLLECTIONS_INDEX,
    DEFAULT_SORT,
    ES_COLLECTIONS_MAPPINGS,
    ITEM_INDICES,
    ITEMS_INDEX_PREFIX,
    Geometry,
)
from stac_fastapi.sfeos_helpers.search_engine import (
    BaseIndexInserter,
    BaseIndexSelector,
    DatetimeIndexInserter,
    IndexInsertionFactory,
    IndexSelectorFactory,
)
from stac_fastapi.types.errors import ConflictError, NotFoundError
from stac_fastapi.types.links import resolve_links
from stac_fastapi.types.stac import Collection, Item

logger = logging.getLogger(__name__)


async def create_index_templates() -> None:
    """
    Create index templates for the Collection and Item indices.

    Returns:
        None

    """
    await create_index_templates_shared(settings=AsyncSearchSettings())


async def create_collection_index() -> None:
    """
    Create the index for a Collection. The settings of the index template will be used implicitly.

    Returns:
        None

    """
    client = AsyncSearchSettings().create_client

    index = f"{COLLECTIONS_INDEX}-000001"

    exists = await client.indices.exists(index=index)
    if not exists:
        await client.indices.create(
            index=index,
            body={
                "aliases": {COLLECTIONS_INDEX: {}},
                "mappings": ES_COLLECTIONS_MAPPINGS,
            },
        )
    await client.close()


async def delete_item_index(collection_id: str) -> None:
    """Delete the index for items in a collection.

    Args:
        collection_id (str): The ID of the collection whose items index will be deleted.

    Notes:
        This function delegates to the shared implementation in delete_item_index_shared.
    """
    await delete_item_index_shared(
        settings=AsyncSearchSettings(), collection_id=collection_id
    )


@attr.s
class DatabaseLogic(BaseDatabaseLogic):
    """Database logic."""

    async_settings: AsyncSearchSettings = attr.ib(factory=AsyncSearchSettings)
    sync_settings: SyncSearchSettings = attr.ib(factory=SyncSearchSettings)

    async_index_selector: BaseIndexSelector = attr.ib(init=False)
    async_index_inserter: BaseIndexInserter = attr.ib(init=False)

    client = attr.ib(init=False)
    sync_client = attr.ib(init=False)

    def __attrs_post_init__(self):
        """Initialize clients after the class is instantiated."""
        self.client = self.async_settings.create_client
        self.sync_client = self.sync_settings.create_client
        self.async_index_inserter = IndexInsertionFactory.create_insertion_strategy(
            self.client
        )
        self.async_index_selector = IndexSelectorFactory.create_selector(self.client)

    item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer)
    collection_serializer: Type[CollectionSerializer] = attr.ib(
        default=CollectionSerializer
    )

    extensions: list[str] = attr.ib(default=attr.Factory(list))

    aggregation_mapping: dict[str, dict[str, Any]] = AGGREGATION_MAPPING

    # constants for field names
    # they are used in multiple methods
    # and could be overwritten in subclasses used with alternate opensearch mappings.
    PROPERTIES_DATETIME_FIELD = os.getenv(
        "STAC_FIELD_PROP_DATETIME", "properties.datetime"
    )
    PROPERTIES_START_DATETIME_FIELD = os.getenv(
        "STAC_FIELD_PROP_START_DATETIME", "properties.start_datetime"
    )
    PROPERTIES_END_DATETIME_FIELD = os.getenv(
        "STAC_FIELD_PROP_END_DATETIME", "properties.end_datetime"
    )
    COLLECTION_FIELD = os.getenv("STAC_FIELD_COLLECTION", "collection")
    GEOMETRY_FIELD = os.getenv("STAC_FIELD_GEOMETRY", "geometry")

    @staticmethod
    def __nested_field__(field: str):
        """Convert opensearch field to nested field format."""
        return field.replace(".", "__")

    """CORE LOGIC"""

    @retry_on_connection_error
    async def get_all_collections(
        self,
        token: str | None,
        limit: int,
        request: Request,
        sort: list[dict[str, Any]] | None = None,
        bbox: list[float] | None = None,
        q: list[str] | None = None,
        filter: dict[str, Any] | None = None,
        query: dict[str, dict[str, Any]] | None = None,
        datetime: str | None = None,
    ) -> tuple[list[dict[str, Any]], str | None, int | None]:
        """Retrieve a list of collections from OpenSearch, supporting pagination.

        Args:
            token (str | None): The pagination token.
            limit (int): The number of results to return.
            request (Request): The FastAPI request object.
            sort (list[dict[str, Any]] | None): Optional sort parameter from the request.
            bbox (list[float] | None): Bounding box to filter collections by spatial extent.
            q (list[str] | None): Free text search terms.
            query (dict[str, dict[str, Any]] | None): Query extension parameters.
            filter (dict[str, Any] | None): Structured query in CQL2 format.
            datetime (str | None): Temporal filter.

        Returns:
            A tuple of (collections, next pagination token if any).
        """
        # Format the sort parameter
        formatted_sort = []
        if sort:
            for item in sort:
                field = item.get("field")
                direction = item.get("direction", "asc")
                if field:
                    formatted_sort.append({field: {"order": direction}})
            # Always include id as a secondary sort to ensure consistent pagination
            if not any("id" in item for item in formatted_sort):
                formatted_sort.append({"id": {"order": "asc"}})
        else:
            formatted_sort = [{"id": {"order": "asc"}}]

        body = {
            "sort": formatted_sort,
            "size": limit,
        }

        # Handle search_after token - split by '|' to get all sort values
        search_after = None
        if token:
            try:
                # The token should be a pipe-separated string of sort values
                # e.g., "2023-01-01T00:00:00Z|collection-1"
                search_after = token.split("|")
                # If the number of sort fields doesn't match token parts, ignore the token
                if len(search_after) != len(formatted_sort):
                    search_after = None
            except Exception:
                search_after = None

            if search_after is not None:
                body["search_after"] = search_after

        # Build the query part of the body
        query_parts = []

        # Apply free text query if provided
        if q:
            free_text_query = apply_collections_free_text_filter_shared(q)
            if free_text_query:
                query_parts.append(free_text_query)

        # Apply structured filter if provided
        if filter:
            # Convert string filter to dict if needed
            if isinstance(filter, str):
                filter = orjson.loads(filter)
            # Convert the filter to an OpenSearch query using the filter module
            es_query = filter_module.to_es(await self.get_queryables_mapping(), filter)
            query_parts.append(es_query)

        # Apply query extension if provided
        if query:
            try:
                # First create a search object to apply filters
                search = Search(index=COLLECTIONS_INDEX)

                # Process each field and operator in the query
                for field_name, expr in query.items():
                    for op, value in expr.items():
                        # For collections, we don't need to prefix with 'properties__'
                        field = field_name
                        # Apply the filter using apply_stacql_filter
                        search = self.apply_stacql_filter(
                            search=search, op=op, field=field, value=value
                        )

                # Convert the search object to a query dict and add it to query_parts
                search_dict = search.to_dict()
                if "query" in search_dict:
                    query_parts.append(search_dict["query"])

            except Exception as e:
                logger.error(f"Error converting query to OpenSearch: {e}")
                # If there's an error, add a query that matches nothing
                query_parts.append({"bool": {"must_not": {"match_all": {}}}})
                raise

        # Apply bbox filter if provided
        bbox_filter = apply_collections_bbox_filter_shared(bbox)
        if bbox_filter:
            query_parts.append(bbox_filter)

        # Apply datetime filter if provided
        datetime_filter = apply_collections_datetime_filter_shared(datetime)
        if datetime_filter:
            query_parts.append(datetime_filter)

        # Exclude Catalogs to prevent them from appearing in the collections endpoint,
        # without strictly requiring "type": "Collection" to support legacy data.
        query_parts.append({"bool": {"must_not": [{"term": {"type": "Catalog"}}]}})

        # Combine all query parts with AND logic
        if query_parts:
            body["query"] = (
                query_parts[0]
                if len(query_parts) == 1
                else {"bool": {"must": query_parts}}
            )

        # Create async tasks for count
        count_task = asyncio.create_task(
            self.client.count(
                index=COLLECTIONS_INDEX,
                body={"query": body.get("query", {"match_all": {}})},
            )
        )

        # Wait for search task to complete
        try:
            response = await self.client.search(index=COLLECTIONS_INDEX, body=body)
        except RequestError as e:
            # Catch ES 400 errors and return HTTP 400 with the actual DB reason
            # e.g., "Fielddata is disabled on text fields in [title]"
            raise HTTPException(
                status_code=400,
                detail=f"Search request error: {e.info.get('error', {}).get('reason', str(e))}",
            )

        hits = response["hits"]["hits"]
        collections = [
            self.collection_serializer.db_to_stac(
                collection=hit["_source"], request=request, extensions=self.extensions
            )
            for hit in hits
        ]

        next_token = None
        if len(hits) == limit:
            next_token_values = hits[-1].get("sort")
            if next_token_values:
                # Join all sort values with '|' to create the token
                next_token = "|".join(str(val) for val in next_token_values)

        # Get the total count of collections
        matched = (
            response["hits"]["total"]["value"]
            if response["hits"]["total"]["relation"] == "eq"
            else None
        )

        # If count task is done, use its result
        if count_task.done():
            try:
                matched = count_task.result().get("count")
            except Exception as e:
                logger.error(f"Count task failed: {e}")

        return collections, next_token, matched

    @retry_on_connection_error
    async def get_one_item(self, collection_id: str, item_id: str) -> dict:
        """Retrieve a single item from the database.

        Args:
            collection_id (str): The id of the Collection that the Item belongs to.
            item_id (str): The id of the Item.

        Returns:
            item (dict): A dictionary containing the source data for the Item.

        Raises:
            NotFoundError: If the specified Item does not exist in the Collection.

        Notes:
            The Item is retrieved from the Opensearch database using the `client.get` method,
            with the index for the Collection as the target index and the combined `mk_item_id` as the document id.
            Item is hidden if hide_item_path is configured via env var.
        """
        try:
            base_query = {"term": {"_id": mk_item_id(item_id, collection_id)}}

            HIDE_ITEM_PATH = os.getenv("HIDE_ITEM_PATH", None)

            if HIDE_ITEM_PATH:
                query = add_hidden_filter(base_query, HIDE_ITEM_PATH)
            else:
                query = base_query

            response = await self.client.search(
                index=index_alias_by_collection_id(collection_id),
                body={
                    "query": query,
                    "size": 1,
                },
            )
            if response["hits"]["total"]["value"] == 0:
                raise NotFoundError(
                    f"Item {item_id} does not exist inside Collection {collection_id}"
                )

            return response["hits"]["hits"][0]["_source"]
        except exceptions.NotFoundError:
            raise NotFoundError(
                f"Item {item_id} does not exist inside Collection {collection_id}"
            )

    async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
        """Retrieve mapping of Queryables for search.

        Args:
            collection_id (str, optional): The id of the Collection the Queryables
            belongs to. Defaults to "*".

        Returns:
            dict: A dictionary containing the Queryables mappings.
        """
        mappings = await self.client.indices.get_mapping(
            index=f"{ITEMS_INDEX_PREFIX}{collection_id}",
        )
        return await get_queryables_mapping_shared(
            collection_id=collection_id, mappings=mappings
        )

    @staticmethod
    def make_search():
        """Database logic to create a Search instance."""
        return Search().sort(*DEFAULT_SORT)

    @staticmethod
    def apply_ids_filter(search: Search, item_ids: list[str]):
        """Database logic to search a list of STAC item ids."""
        return search.filter("terms", id=item_ids)

    @staticmethod
    def apply_collections_filter(search: Search, collection_ids: list[str]):
        """Database logic to search a list of STAC collection ids."""
        collection_nested_field = DatabaseLogic.__nested_field__(
            DatabaseLogic.COLLECTION_FIELD
        )
        return search.filter("terms", **{collection_nested_field: collection_ids})

    @staticmethod
    def apply_free_text_filter(search: Search, free_text_queries: list[str] | None):
        """Create a free text query for OpenSearch queries.

        This method delegates to the shared implementation in apply_free_text_filter_shared.

        Args:
            search (Search): The search object to apply the query to.
            free_text_queries (list[str] | None): A list of text strings to search for in the properties.

        Returns:
            Search: The search object with the free text query applied, or the original search
                object if no free_text_queries were provided.
        """
        return apply_free_text_filter_shared(
            search=search, free_text_queries=free_text_queries
        )

    @staticmethod
    def apply_datetime_filter(
        search: Search, datetime: str | None
    ) -> tuple[Search, dict[str, str | None]]:
        """Apply a filter to search on datetime, start_datetime, and end_datetime fields.

        Args:
            search: The search object to filter.
            datetime: Optional[str]

        Returns:
            The filtered search object.
        """
        datetime_search = return_date(datetime)

        if not datetime_search:
            return search, datetime_search

        # USE_DATETIME env var
        # True: Search by datetime, if null search by start/end datetime
        # False: Always search only by start/end datetime
        USE_DATETIME = get_bool_env("USE_DATETIME", default=True)

        nested_datetime_field = DatabaseLogic.__nested_field__(
            DatabaseLogic.PROPERTIES_DATETIME_FIELD
        )
        nested_start_datetime_field = DatabaseLogic.__nested_field__(
            DatabaseLogic.PROPERTIES_START_DATETIME_FIELD
        )
        nested_end_datetime_field = DatabaseLogic.__nested_field__(
            DatabaseLogic.PROPERTIES_END_DATETIME_FIELD
        )

        if USE_DATETIME:
            if "eq" in datetime_search:
                # For exact matches, include:
                # 1. Items with matching exact datetime
                # 2. Items with datetime:null where the time falls within their range
                should = [
                    Q(
                        "bool",
                        filter=[
                            Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD),
                            Q(
                                "term",
                                **{nested_datetime_field: datetime_search["eq"]},
                            ),
                        ],
                    ),
                    Q(
                        "bool",
                        must_not=[
                            Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)
                        ],
                        filter=[
                            Q(
                                "exists",
                                field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
                            ),
                            Q(
                                "exists",
                                field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD,
                            ),
                            Q(
                                "range",
                                **{
                                    nested_start_datetime_field: {
                                        "lte": datetime_search["eq"]
                                    }
                                },
                            ),
                            Q(
                                "range",
                                **{
                                    nested_end_datetime_field: {
                                        "gte": datetime_search["eq"]
                                    }
                                },
                            ),
                        ],
                    ),
                ]
            else:
                # For date ranges, include:
                # 1. Items with datetime in the range
                # 2. Items with datetime:null that overlap the search range
                should = [
                    Q(
                        "bool",
                        filter=[
                            Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD),
                            Q(
                                "range",
                                **{
                                    nested_datetime_field: {
                                        "gte": datetime_search["gte"],
                                        "lte": datetime_search["lte"],
                                    }
                                },
                            ),
                        ],
                    ),
                    Q(
                        "bool",
                        must_not=[
                            Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)
                        ],
                        filter=[
                            Q(
                                "exists",
                                field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
                            ),
                            Q(
                                "exists",
                                field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD,
                            ),
                            Q(
                                "range",
                                **{
                                    nested_start_datetime_field: {
                                        "lte": datetime_search["lte"]
                                    }
                                },
                            ),
                            Q(
                                "range",
                                **{
                                    nested_end_datetime_field: {
                                        "gte": datetime_search["gte"]
                                    }
                                },
                            ),
                        ],
                    ),
                ]

            return (
                search.query(Q("bool", should=should, minimum_should_match=1)),
                datetime_search,
            )
        else:
            if "eq" in datetime_search:
                filter_query = Q(
                    "bool",
                    filter=[
                        Q(
                            "exists",
                            field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
                        ),
                        Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD),
                        Q(
                            "range",
                            **{
                                nested_start_datetime_field: {
                                    "lte": datetime_search["eq"]
                                }
                            },
                        ),
                        Q(
                            "range",
                            **{
                                nested_end_datetime_field: {
                                    "gte": datetime_search["eq"]
                                }
                            },
                        ),
                    ],
                )
            else:
                filter_query = Q(
                    "bool",
                    filter=[
                        Q(
                            "exists",
                            field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
                        ),
                        Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD),
                        Q(
                            "range",
                            **{
                                nested_start_datetime_field: {
                                    "lte": datetime_search["lte"]
                                }
                            },
                        ),
                        Q(
                            "range",
                            **{
                                nested_end_datetime_field: {
                                    "gte": datetime_search["gte"]
                                }
                            },
                        ),
                    ],
                )
        return search.query(filter_query), datetime_search

    @staticmethod
    def apply_bbox_filter(search: Search, bbox: list):
        """Filter search results based on bounding box.

        Args:
            search (Search): The search object to apply the filter to.
            bbox (List): The bounding box coordinates, represented as a list of four values [minx, miny, maxx, maxy].

        Returns:
            search (Search): The search object with the bounding box filter applied.

        Notes:
            The bounding box is transformed into a polygon using the `bbox2polygon` function and
            a geo_shape filter is added to the search object, set to intersect with the specified polygon.
        """
        return search.filter(
            Q(
                {
                    "geo_shape": {
                        DatabaseLogic.GEOMETRY_FIELD: {
                            "shape": {
                                "type": "polygon",
                                "coordinates": bbox2polygon(*bbox),
                            },
                            "relation": "intersects",
                        }
                    }
                }
            )
        )

    @staticmethod
    def apply_intersects_filter(
        search: Search,
        intersects: Geometry,
    ):
        """Filter search results based on intersecting geometry.

        Args:
            search (Search): The search object to apply the filter to.
            intersects (Geometry): The intersecting geometry, represented as a GeoJSON-like object.

        Returns:
            search (Search): The search object with the intersecting geometry filter applied.

        Notes:
            A geo_shape filter is added to the search object, set to intersect with the specified geometry.
        """
        filter = apply_intersects_filter_shared(intersects=intersects)
        return search.filter(Q(filter))

    @staticmethod
    def apply_stacql_filter(search: Search, op: str, field: str, value: float):
        """Filter search results based on a comparison between a field and a value.

        Args:
            search (Search): The search object to apply the filter to.
            op (str): The comparison operator to use. Can be 'eq' (equal), 'ne'/'neq' (not equal), 'gt' (greater than),
                'gte' (greater than or equal), 'lt' (less than), or 'lte' (less than or equal).
            field (str): The field to perform the comparison on.
            value (float): The value to compare the field against.

        Returns:
            search (Search): The search object with the specified filter applied.
        """
        if op == "eq":
            search = search.filter("term", **{field: value})
        elif op == "ne" or op == "neq":
            # For not equal, use a bool query with must_not
            search = search.exclude("term", **{field: value})
        elif op in ["gt", "gte", "lt", "lte"]:
            # For range operators
            key_filter = {field: {op: value}}
            search = search.filter(Q("range", **key_filter))
        elif op == "in":
            # For in operator (value should be a list)
            if isinstance(value, list):
                search = search.filter("terms", **{field: value})
            else:
                search = search.filter("term", **{field: value})
        elif op == "contains":
            # For contains operator (for arrays)
            search = search.filter("term", **{field: value})

        return search

    async def apply_cql2_filter(self, search: Search, _filter: dict[str, Any] | None):
        """
        Apply a CQL2 filter to an Opensearch Search object.

        This method transforms a dictionary representing a CQL2 filter into an Opensearch query
        and applies it to the provided Search object. If the filter is None, the original Search
        object is returned unmodified.

        Args:
            search (Search): The Opensearch Search object to which the filter will be applied.
            _filter (dict[str, Any] | None): The filter in dictionary form that needs to be applied
                                                to the search. The dictionary should follow the structure
                                                required by the `to_es` function which converts it
                                                to an Opensearch query.

        Returns:
            Search: The modified Search object with the filter applied if a filter is provided,
                    otherwise the original Search object.
        """
        if _filter is not None:
            es_query = filter_module.to_es(await self.get_queryables_mapping(), _filter)
            search = search.filter(es_query)

        return search

    @staticmethod
    def populate_sort(sortby: list) -> dict[str, dict[str, str]] | None:
        """Create a sort configuration for OpenSearch queries.

        This method delegates to the shared implementation in populate_sort_shared.

        Args:
            sortby (List): A list of sort specifications, each containing a field and direction.

        Returns:
            dict[str, dict[str, str]] | None: A dictionary mapping field names to sort direction
                configurations, or None if no sort was specified.
        """
        return populate_sort_shared(sortby=sortby)

    @retry_on_datetime_not_found
    @retry_on_connection_error
    async def execute_search(
        self,
        search: Search,
        limit: int,
        token: str | None,
        sort: dict[str, dict[str, str]] | None,
        collection_ids: list[str] | None,
        datetime_search: str,
        ignore_unavailable: bool = True,
    ) -> tuple[Iterable[dict[str, Any]], int | None, str | None]:
        """Execute a search query with limit and other optional parameters.

        Args:
            search (Search): The search query to be executed.
            limit (int): The maximum number of results to be returned.
            token (str | None): The token used to return the next set of results.
            sort (dict[str, dict[str, str]] | None): Specifies how the results should be sorted.
            collection_ids (list[str] | None): The collection ids to search.
            datetime_search (str): Datetime used for index selection.
            ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True.

        Returns:
            tuple[Iterable[dict[str, Any]], int | None, str | None]: A tuple containing:
                - An iterable of search results, where each result is a dictionary with keys and values representing the
                fields and values of each document.
                - The total number of results (if the count could be computed), or None if the count could not be
                computed.
                - The token to be used to retrieve the next set of results, or None if there are no more results.

        Raises:
            NotFoundError: If the collections specified in `collection_ids` do not exist.
        """
        search_body: dict[str, Any] = {}
        query = search.query.to_dict() if search.query else None

        index_param = await self.async_index_selector.select_indexes(
            collection_ids, datetime_search
        )
        if len(index_param) > ES_MAX_URL_LENGTH - 300:
            index_param = ITEM_INDICES
            query = add_collections_to_body(collection_ids, query)

        HIDE_ITEM_PATH = os.getenv("HIDE_ITEM_PATH", None)

        if HIDE_ITEM_PATH:
            search_body["query"] = add_hidden_filter(query, HIDE_ITEM_PATH)
        elif query:
            search_body["query"] = query

        search_after = None

        if token:
            search_after = orjson.loads(urlsafe_b64decode(token))
        if search_after:
            search_body["search_after"] = search_after

        search_body["sort"] = sort if sort else DEFAULT_SORT

        max_result_window = MAX_LIMIT

        size_limit = min(limit + 1, max_result_window)

        search_task = asyncio.create_task(
            self.client.search(
                index=index_param,
                ignore_unavailable=ignore_unavailable,
                body=search_body,
                size=size_limit,
            )
        )

        # Ensure hidden item is not counted
        count_query = search.to_dict(count=True)
        if HIDE_ITEM_PATH:
            q = count_query.get("query")
            count_query["query"] = add_hidden_filter(q, HIDE_ITEM_PATH)

        count_task = asyncio.create_task(
            self.client.count(
                index=index_param,
                ignore_unavailable=ignore_unavailable,
                body=count_query,
            )
        )

        try:
            es_response = await search_task
        except exceptions.NotFoundError:
            raise NotFoundError(f"Collections '{collection_ids}' do not exist")

        count_timeout = float(os.getenv("COUNT_TIMEOUT", 0.5))

        if count_timeout > 0 and not count_task.done():
            try:
                logger.debug("Waiting for count task to complete...")
                await asyncio.wait_for(count_task, timeout=count_timeout)
            except asyncio.TimeoutError:
                logger.warning("Count task timed out, returning results without count")

        hits = es_response["hits"]["hits"]
        items = (hit["_source"] for hit in hits[:limit])

        next_token = None
        if len(hits) > limit and limit < max_result_window:
            if hits and (sort_array := hits[limit - 1].get("sort")):
                next_token = urlsafe_b64encode(orjson.dumps(sort_array)).decode()

        matched = (
            es_response["hits"]["total"]["value"]
            if es_response["hits"]["total"]["relation"] == "eq"
            else None
        )
        if count_task.done():
            try:
                matched = count_task.result().get("count")
            except Exception as e:
                logger.error(f"Count task failed: {e}")

        return items, matched, next_token

    """ AGGREGATE LOGIC """

    @retry_on_connection_error
    async def aggregate(
        self,
        collection_ids: list[str] | None,
        aggregations: list[str],
        search: Search,
        centroid_geohash_grid_precision: int,
        centroid_geohex_grid_precision: int,
        centroid_geotile_grid_precision: int,
        geometry_geohash_grid_precision: int,
        geometry_geotile_grid_precision: int,
        datetime_frequency_interval: str,
        datetime_search: str,
        ignore_unavailable: bool | None = True,
    ):
        """Return aggregations of STAC Items."""
        search_body: dict[str, Any] = {}
        query = search.query.to_dict() if search.query else None
        if query:
            search_body["query"] = query

        def _fill_aggregation_parameters(name: str, agg: dict) -> dict:
            [key] = agg.keys()
            agg_precision = {
                "centroid_geohash_grid_frequency": centroid_geohash_grid_precision,
                "centroid_geohex_grid_frequency": centroid_geohex_grid_precision,
                "centroid_geotile_grid_frequency": centroid_geotile_grid_precision,
                "geometry_geohash_grid_frequency": geometry_geohash_grid_precision,
                "geometry_geotile_grid_frequency": geometry_geotile_grid_precision,
            }
            if name in agg_precision:
                agg[key]["precision"] = agg_precision[name]

            if key == "date_histogram":
                agg[key]["calendar_interval"] = datetime_frequency_interval

            return agg

        # include all aggregations specified
        # this will ignore aggregations with the wrong names
        search_body["aggregations"] = {
            k: _fill_aggregation_parameters(k, deepcopy(v))
            for k, v in self.aggregation_mapping.items()
            if k in aggregations
        }

        index_param = await self.async_index_selector.select_indexes(
            collection_ids, datetime_search
        )

        search_task = asyncio.create_task(
            self.client.search(
                index=index_param,
                ignore_unavailable=ignore_unavailable,
                body=search_body,
            )
        )

        try:
            db_response = await search_task
        except exceptions.NotFoundError:
            raise NotFoundError(f"Collections '{collection_ids}' do not exist")

        return db_response

    """ TRANSACTION LOGIC """

    async def check_collection_exists(self, collection_id: str):
        """Database logic to check if a collection exists."""
        if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id):
            raise NotFoundError(f"Collection {collection_id} does not exist")

    async def _check_item_exists_in_collection(
        self, collection_id: str, item_id: str
    ) -> bool:
        """Check if an item exists across all indexes for a collection.

        Args:
            collection_id (str): The collection identifier.
            item_id (str): The item identifier.

        Returns:
            bool: True if the item exists in any index, False otherwise.
        """
        alias = index_alias_by_collection_id(collection_id)
        doc_id = mk_item_id(item_id, collection_id)
        try:
            return await check_item_exists_in_alias(self.client, alias, doc_id)
        except Exception:
            return False

    def _check_item_exists_in_collection_sync(
        self, collection_id: str, item_id: str
    ) -> bool:
        """Check if an item exists across all indexes for a collection (sync version).

        Args:
            collection_id (str): The collection identifier.
            item_id (str): The item identifier.

        Returns:
            bool: True if the item exists in any index, False otherwise.
        """
        alias = index_alias_by_collection_id(collection_id)
        doc_id = mk_item_id(item_id, collection_id)
        try:
            return check_item_exists_in_alias_sync(self.sync_client, alias, doc_id)
        except Exception:
            return False

    async def async_prep_create_item(
        self, item: Item, base_url: str, exist_ok: bool = False
    ) -> Item:
        """
        Preps an item for insertion into the database.

        Args:
            item (Item): The item to be prepped for insertion.
            base_url (str): The base URL used to create the item's self URL.
            exist_ok (bool): Indicates whether the item can exist already.

        Returns:
            Item: The prepped item.

        Raises:
            ItemAlreadyExistsError: If the item already exists in the database.

        """
        await self.check_collection_exists(collection_id=item["collection"])

        if not exist_ok and await self._check_item_exists_in_collection(
            item["collection"], item["id"]
        ):
            raise ItemAlreadyExistsError(item["id"], item["collection"])

        return self.item_serializer.stac_to_db(item, base_url)

    async def bulk_async_prep_create_item(
        self, item: Item, base_url: str, exist_ok: bool = False
    ) -> Item | None:
        """
        Prepare an item for insertion into the database.

        This method performs pre-insertion preparation on the given `item`, such as:
        - Verifying that the collection the item belongs to exists.
        - Optionally checking if an item with the same ID already exists in the database.
        - Serializing the item into a database-compatible format.

        Args:
            item (Item): The item to be prepared for insertion.
            base_url (str): The base URL used to construct the item's self URL.
            exist_ok (bool): Indicates whether the item can already exist in the database.
                            If False, a `ConflictError` is raised if the item exists.

        Returns:
            Item: The prepared item, serialized into a database-compatible format.

        Raises:
            NotFoundError: If the collection that the item belongs to does not exist in the database.
            ConflictError: If an item with the same ID already exists in the collection and `exist_ok` is False,
                        and `RAISE_ON_BULK_ERROR` is set to `true`.
        """
        logger.debug(f"Preparing item {item['id']} in collection {item['collection']}.")

        # Check if the collection exists
        await self.check_collection_exists(collection_id=item["collection"])

        # Check if the item already exists in the database (across all datetime indexes)
        if not exist_ok and await self._check_item_exists_in_collection(
            item["collection"], item["id"]
        ):
            if self.async_settings.raise_on_bulk_error:
                raise ItemAlreadyExistsError(item["id"], item["collection"])
            else:
                logger.warning(
                    f"Item {item['id']} in collection {item['collection']} already exists. "
                    "Skipping as `RAISE_ON_BULK_ERROR` is set to false."
                )
                return None

        # Serialize the item into a database-compatible format
        prepped_item = self.item_serializer.stac_to_db(item, base_url)
        logger.debug(f"Item {item['id']} prepared successfully.")
        return prepped_item

    def bulk_sync_prep_create_item(
        self, item: Item, base_url: str, exist_ok: bool = False
    ) -> Item | None:
        """
        Prepare an item for insertion into the database.

        This method performs pre-insertion preparation on the given `item`, such as:
        - Verifying that the collection the item belongs to exists.
        - Optionally checking if an item with the same ID already exists in the database.
        - Serializing the item into a database-compatible format.

        Args:
            item (Item): The item to be prepared for insertion.
            base_url (str): The base URL used to construct the item's self URL.
            exist_ok (bool): Indicates whether the item can already exist in the database.
                            If False, a `ConflictError` is raised if the item exists.

        Returns:
            Item: The prepared item, serialized into a database-compatible format.

        Raises:
            NotFoundError: If the collection that the item belongs to does not exist in the database.
            ConflictError: If an item with the same ID already exists in the collection and `exist_ok` is False,
                        and `RAISE_ON_BULK_ERROR` is set to `true`.
        """
        logger.debug(f"Preparing item {item['id']} in collection {item['collection']}.")

        # Check if the collection exists
        if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=item["collection"]):
            raise NotFoundError(f"Collection {item['collection']} does not exist")

        # Check if the item already exists in the database (across all datetime indexes)
        if not exist_ok and self._check_item_exists_in_collection_sync(
            item["collection"], item["id"]
        ):
            if self.sync_settings.raise_on_bulk_error:
                raise ItemAlreadyExistsError(item["id"], item["collection"])
            else:
                logger.warning(
                    f"Item {item['id']} in collection {item['collection']} already exists. "
                    "Skipping as `RAISE_ON_BULK_ERROR` is set to false."
                )
                return None

        # Serialize the item into a database-compatible format
        prepped_item = self.item_serializer.stac_to_db(item, base_url)
        logger.debug(f"Item {item['id']} prepared successfully.")
        return prepped_item

    @retry_on_connection_error
    async def create_item(
        self,
        item: Item,
        base_url: str = "",
        exist_ok: bool = False,
        **kwargs: Any,
    ):
        """Database logic for creating one item.

        Args:
            item (Item): The item to be created.
            base_url (str, optional): The base URL for the item. Defaults to an empty string.
            exist_ok (bool, optional): Whether to allow the item to exist already. Defaults to False.
            **kwargs: Additional keyword arguments like refresh.

        Raises:
            ConflictError: If the item already exists in the database.

        Returns:
            None
        """
        # todo: check if collection exists, but cache
        item_id = item["id"]
        collection_id = item["collection"]

        # Ensure kwargs is a dictionary
        kwargs = kwargs or {}

        # Resolve the `refresh` parameter
        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
        refresh = validate_refresh(refresh)

        # Log the creation attempt
        logger.info(
            f"Creating item {item_id} in collection {collection_id} with refresh={refresh}"
        )

        if exist_ok and isinstance(self.async_index_inserter, DatetimeIndexInserter):
            existing_item = await self.get_one_item(collection_id, item_id)
            primary_datetime_name = self.async_index_inserter.primary_datetime_name

            existing_primary_datetime = existing_item.get("properties", {}).get(
                primary_datetime_name
            )
            new_primary_datetime = item.get("properties", {}).get(primary_datetime_name)

            if existing_primary_datetime != new_primary_datetime:
                self.async_index_inserter.validate_datetime_field_update(
                    f"properties/{primary_datetime_name}"
                )

            if primary_datetime_name == "start_datetime":
                existing_end_datetime = existing_item.get("properties", {}).get(
                    "end_datetime"
                )
                new_end_datetime = item.get("properties", {}).get("end_datetime")

                if existing_end_datetime != new_end_datetime:
                    self.async_index_inserter.validate_datetime_field_update(
                        "properties/end_datetime"
                    )

        item = await self.async_prep_create_item(
            item=item, base_url=base_url, exist_ok=exist_ok
        )

        target_index = await self.async_index_inserter.get_target_index(
            collection_id, item
        )

        await self.client.index(
            index=target_index,
            id=mk_item_id(item_id, collection_id),
            body=item,
            refresh=refresh,
        )

    @retry_on_connection_error
    async def merge_patch_item(
        self,
        collection_id: str,
        item_id: str,
        item: PartialItem,
        base_url: str,
        refresh: bool = True,
    ) -> Item:
        """Database logic for merge patching an item following RF7396.

        Args:
            collection_id(str): Collection that item belongs to.
            item_id(str): Id of item to be patched.
            item (PartialItem): The partial item to be updated.
            base_url: (str): The base URL used for constructing URLs for the item.
            refresh (bool, optional): Refresh the index after performing the operation. Defaults to True.

        Returns:
            patched item.
        """
        operations = merge_to_operations(item.model_dump())

        return await self.json_patch_item(
            collection_id=collection_id,
            item_id=item_id,
            operations=operations,
            base_url=base_url,
            create_nest=True,
            refresh=refresh,
        )

    @retry_on_connection_error
    async def json_patch_item(
        self,
        collection_id: str,
        item_id: str,
        operations: list[PatchOperation],
        base_url: str,
        create_nest: bool = False,
        refresh: bool = True,
    ) -> Item:
        """Database logic for json patching an item following RF6902.

        Args:
            collection_id(str): Collection that item belongs to.
            item_id(str): Id of item to be patched.
            operations (list): List of operations to run.
            base_url (str): The base URL used for constructing URLs for the item.
            refresh (bool, optional): Refresh the index after performing the operation. Defaults to True.

        Returns:
            patched item.
        """
        for operation in operations:
            if operation.op in ["add", "replace", "remove"]:
                self.async_index_inserter.validate_datetime_field_update(operation.path)

        new_item_id = None
        new_collection_id = None
        script_operations = []

        for operation in operations:
            if operation.path in ["collection", "id"] and operation.op in [
                "add",
                "replace",
            ]:
                if operation.path == "collection" and collection_id != operation.value:
                    await self.check_collection_exists(collection_id=operation.value)
                    new_collection_id = operation.value

                if operation.path == "id" and item_id != operation.value:
                    new_item_id = operation.value

            else:
                script_operations.append(operation)

        try:
            search_response = await self.client.search(
                index=index_alias_by_collection_id(collection_id),
                body={
                    "query": {"term": {"_id": mk_item_id(item_id, collection_id)}},
                    "size": 1,
                },
            )
            if search_response["hits"]["total"]["value"] == 0:
                raise NotFoundError(
                    f"Item {item_id} does not exist inside Collection {collection_id}"
                )

            if script_operations:
                script = operations_to_script(
                    script_operations, create_nest=create_nest
                )
                document_index = search_response["hits"]["hits"][0]["_index"]
                await self.client.update(
                    index=document_index,
                    id=mk_item_id(item_id, collection_id),
                    body={"script": script},
                    refresh=True,
                )
        except exceptions.NotFoundError:
            raise NotFoundError(
                f"Item {item_id} does not exist inside Collection {collection_id}"
            )
        except exceptions.RequestError as exc:
            raise HTTPException(
                status_code=400, detail=exc.info["error"]["caused_by"]
            ) from exc

        item = await self.get_one_item(collection_id, item_id)

        if new_collection_id:
            item["collection"] = new_collection_id
            item = await self.async_prep_create_item(item=item, base_url=base_url)
            await self.create_item(item=item, refresh=True)

            await self.delete_item(
                item_id=item_id,
                collection_id=collection_id,
                refresh=refresh,
            )

            collection_id = new_collection_id

        if new_item_id:
            item["id"] = new_item_id
            item = await self.async_prep_create_item(item=item, base_url=base_url)
            await self.create_item(item=item, refresh=True)

            await self.delete_item(
                item_id=item_id,
                collection_id=collection_id,
                refresh=refresh,
            )

        return item

    @retry_on_connection_error
    async def delete_item(self, item_id: str, collection_id: str, **kwargs: Any):
        """Delete a single item from the database.

        Args:
            item_id (str): The id of the Item to be deleted.
            collection_id (str): The id of the Collection that the Item belongs to.
            **kwargs: Additional keyword arguments like refresh.

        Raises:
            NotFoundError: If the Item does not exist in the database.
        """
        # Ensure kwargs is a dictionary
        kwargs = kwargs or {}

        # Resolve the `refresh` parameter
        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
        refresh = validate_refresh(refresh)

        # Log the deletion attempt
        logger.info(
            f"Deleting item {item_id} from collection {collection_id} with refresh={refresh}"
        )

        try:
            await self.client.delete_by_query(
                index=index_alias_by_collection_id(collection_id),
                body={"query": {"term": {"_id": mk_item_id(item_id, collection_id)}}},
                refresh=refresh,
            )
        except exceptions.NotFoundError:
            raise NotFoundError(
                f"Item {item_id} in collection {collection_id} not found"
            )

    async def get_items_mapping(self, collection_id: str) -> dict[str, Any]:
        """Get the mapping for the specified collection's items index.

        Args:
            collection_id (str): The ID of the collection to get items mapping for.

        Returns:
            dict[str, Any]: The mapping information.
        """
        index_name = index_alias_by_collection_id(collection_id)
        try:
            mapping = await self.client.indices.get_mapping(
                index=index_name, params={"allow_no_indices": "false"}
            )
            return mapping
        except exceptions.NotFoundError:
            raise NotFoundError(f"Mapping for index {index_name} not found")

    @retry_on_connection_error
    async def get_items_unique_values(
        self, collection_id: str, field_names: Iterable[str], *, limit: int = 100
    ) -> dict[str, list[str]]:
        """Get the unique values for the given fields in the collection."""
        limit_plus_one = limit + 1
        index_name = index_alias_by_collection_id(collection_id)

        query = await self.client.search(
            index=index_name,
            body={
                "size": 0,
                "aggs": {
                    field: {"terms": {"field": field, "size": limit_plus_one}}
                    for field in field_names
                },
            },
        )

        result: dict[str, list[str]] = {}
        for field, agg in query["aggregations"].items():
            if len(agg["buckets"]) > limit:
                logger.warning(
                    "Skipping enum field %s: exceeds limit of %d unique values. "
                    "Consider excluding this field from enumeration or increase the limit.",
                    field,
                    limit,
                )
                continue
            result[field] = [bucket["key"] for bucket in agg["buckets"]]
        return result

    @retry_on_connection_error
    async def create_collection(self, collection: Collection, **kwargs: Any):
        """Create a single collection in the database.

        Args:
            collection (Collection): The Collection object to be created.
            **kwargs: Additional keyword arguments like refresh.

        Raises:
            ConflictError: If a Collection with the same id already exists in the database.

        Notes:
            A new index is created for the items in the Collection if the index insertion strategy requires it.
        """
        collection_id = collection["id"]

        # Ensure kwargs is a dictionary
        kwargs = kwargs or {}

        # Resolve the `refresh` parameter
        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
        refresh = validate_refresh(refresh)

        # Log the creation attempt
        logger.info(f"Creating collection {collection_id} with refresh={refresh}")

        if await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id):
            raise ConflictError(f"Collection {collection_id} already exists")

        if get_bool_env("ENABLE_COLLECTIONS_SEARCH") or get_bool_env(
            "ENABLE_COLLECTIONS_SEARCH_ROUTE"
        ):
            # Convert bbox to bbox_shape for geospatial queries (ES/OS specific)
            add_bbox_shape_to_collection(collection)

        await self.client.index(
            index=COLLECTIONS_INDEX,
            id=collection_id,
            body=collection,
            refresh=refresh,
        )
        if self.async_index_inserter.should_create_collection_index():
            await self.async_index_inserter.create_simple_index(
                self.client, collection_id
            )

    @retry_on_connection_error
    async def find_collection(self, collection_id: str) -> Collection:
        """Find and return a collection from the database.

        Args:
            self: The instance of the object calling this function.
            collection_id (str): The ID of the collection to be found.

        Returns:
            Collection: The found collection, represented as a `Collection` object.

        Raises:
            NotFoundError: If the collection with the given `collection_id` is not found in the database.

        Notes:
            This function searches for a collection in the database using the specified `collection_id` and returns the found
            collection as a `Collection` object. If the collection is not found, a `NotFoundError` is raised.
        """
        try:
            collection = await self.client.get(
                index=COLLECTIONS_INDEX, id=collection_id
            )
        except exceptions.NotFoundError:
            raise NotFoundError(f"Collection {collection_id} not found")

        return collection["_source"]

    @retry_on_connection_error
    async def update_collection(
        self, collection_id: str, collection: Collection, **kwargs: Any
    ):
        """Update a collection from the database.

        Args:
            collection_id (str): The ID of the collection to be updated.
            collection (Collection): The Collection object to be used for the update.
            **kwargs: Additional keyword arguments like refresh.

        Raises:
            NotFoundError: If the collection with the given `collection_id` is not
            found in the database.

        Notes:
            This function updates the collection in the database using the specified
            `collection_id` and with the collection specified in the `Collection` object.
            If the collection is not found, a `NotFoundError` is raised.
        """
        # Ensure kwargs is a dictionary
        kwargs = kwargs or {}

        # Resolve the `refresh` parameter
        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
        refresh = validate_refresh(refresh)

        # Log the update attempt
        logger.info(f"Updating collection {collection_id} with refresh={refresh}")

        await self.find_collection(collection_id=collection_id)

        if collection_id != collection["id"]:
            logger.info(
                f"Collection ID change detected: {collection_id} -> {collection['id']}"
            )

            await self.create_collection(collection, refresh=refresh)

            await self.client.reindex(
                body={
                    "dest": {"index": f"{ITEMS_INDEX_PREFIX}{collection['id']}"},
                    "source": {"index": f"{ITEMS_INDEX_PREFIX}{collection_id}"},
                    "script": {
                        "lang": "painless",
                        "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""",
                    },
                },
                wait_for_completion=True,
                refresh=refresh,
            )

            await self.delete_collection(collection_id=collection_id, **kwargs)

        else:
            if get_bool_env("ENABLE_COLLECTIONS_SEARCH") or get_bool_env(
                "ENABLE_COLLECTIONS_SEARCH_ROUTE"
            ):
                # Convert bbox to bbox_shape for geospatial queries (ES/OS specific)
                add_bbox_shape_to_collection(collection)

            await self.client.index(
                index=COLLECTIONS_INDEX,
                id=collection_id,
                body=collection,
                refresh=refresh,
            )

    @retry_on_connection_error
    async def merge_patch_collection(
        self,
        collection_id: str,
        collection: PartialCollection,
        base_url: str,
        refresh: bool = True,
    ) -> Collection:
        """Database logic for merge patching a collection following RF7396.

        Args:
            collection_id(str): Id of collection to be patched.
            collection (PartialCollection): The partial collection to be updated.
            base_url: (str): The base URL used for constructing links.
            refresh (bool, optional): Refresh the index after performing the operation. Defaults to True.


        Returns:
            patched collection.
        """
        operations = merge_to_operations(collection.model_dump())

        return await self.json_patch_collection(
            collection_id=collection_id,
            operations=operations,
            base_url=base_url,
            create_nest=True,
            refresh=refresh,
        )

    @retry_on_connection_error
    async def json_patch_collection(
        self,
        collection_id: str,
        operations: list[PatchOperation],
        base_url: str,
        create_nest: bool = False,
        refresh: bool = True,
    ) -> Collection:
        """Database logic for json patching a collection following RF6902.

        Args:
            collection_id(str): Id of collection to be patched.
            operations (list): List of operations to run.
            base_url (str): The base URL used for constructing links.
            refresh (bool, optional): Refresh the index after performing the operation. Defaults to True.

        Returns:
            patched collection.
        """
        new_collection_id = None
        script_operations = []

        for operation in operations:
            if (
                operation.op in ["add", "replace"]
                and operation.path == "collection"
                and collection_id != operation.value
            ):
                new_collection_id = operation.value

            else:
                script_operations.append(operation)

        script = operations_to_script(script_operations, create_nest=create_nest)

        try:
            await self.client.update(
                index=COLLECTIONS_INDEX,
                id=collection_id,
                body={"script": script},
                refresh=True,
            )

        except exceptions.RequestError as exc:
            raise HTTPException(
                status_code=400, detail=exc.info["error"]["caused_by"]
            ) from exc

        collection = await self.find_collection(collection_id)

        if new_collection_id:
            collection["id"] = new_collection_id
            collection["links"] = resolve_links([], base_url)

            await self.update_collection(
                collection_id=collection_id,
                collection=collection,
                refresh=refresh,
            )

        return collection

    @retry_on_connection_error
    async def delete_collection(self, collection_id: str, **kwargs: Any):
        """Delete a collection from the database.

        Parameters:
            self: The instance of the object calling this function.
            collection_id (str): The ID of the collection to be deleted.
            **kwargs: Additional keyword arguments like refresh.

        Raises:
            NotFoundError: If the collection with the given `collection_id` is not found in the database.

        Notes:
            This function first verifies that the collection with the specified `collection_id` exists in the database, and then
            deletes the collection. If `refresh` is set to "true", "false", or "wait_for", the index is refreshed accordingly after
            the deletion. Additionally, this function also calls `delete_item_index` to delete the index for the items in the collection.
        """
        # Ensure kwargs is a dictionary
        kwargs = kwargs or {}

        await self.find_collection(collection_id=collection_id)

        # Resolve the `refresh` parameter
        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
        refresh = validate_refresh(refresh)

        # Log the deletion attempt
        logger.info(f"Deleting collection {collection_id} with refresh={refresh}")

        await self.client.delete(
            index=COLLECTIONS_INDEX, id=collection_id, refresh=refresh
        )
        # Delete the item index for the collection
        await delete_item_index(collection_id)
        await self.async_index_inserter.refresh_cache()

    @retry_on_connection_error
    async def bulk_async(
        self,
        collection_id: str,
        processed_items: list[Item],
        **kwargs: Any,
    ) -> tuple[int, list[dict[str, Any]]]:
        """
        Perform a bulk insert of items into the database asynchronously.

        Args:
            collection_id (str): The ID of the collection to which the items belong.
            processed_items (list[Item]): A list of `Item` objects to be inserted into the database.
            **kwargs (Any): Additional keyword arguments, including:
                - refresh (str, optional): Whether to refresh the index after the bulk insert.
                Can be "true", "false", or "wait_for". Defaults to the value of `self.sync_settings.database_refresh`.
                - refresh (bool, optional): Whether to refresh the index after the bulk insert.
                - raise_on_error (bool, optional): Whether to raise an error if any of the bulk operations fail.
                Defaults to the value of `self.async_settings.raise_on_bulk_error`.

        Returns:
            tuple[int, list[dict[str, Any]]]: A tuple containing:
                - The number of successfully processed actions (`success`).
                - A list of errors encountered during the bulk operation (`errors`).

        Notes:
            This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
            The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
            completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. The `refresh`
            parameter determines whether the index is refreshed after the bulk insert:
                - "true": Forces an immediate refresh of the index.
                - "false": Does not refresh the index immediately (default behavior).
                - "wait_for": Waits for the next refresh cycle to make the changes visible.
        """
        # Ensure kwargs is a dictionary
        kwargs = kwargs or {}

        # Resolve the `refresh` parameter
        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
        refresh = validate_refresh(refresh)

        # Log the bulk insert attempt
        logger.info(
            f"Performing bulk insert for collection {collection_id} with refresh={refresh}"
        )

        # Handle empty processed_items
        if not processed_items:
            logger.warning(f"No items to insert for collection {collection_id}")
            return 0, []

        raise_on_error = self.async_settings.raise_on_bulk_error
        actions = await self.async_index_inserter.prepare_bulk_actions(
            collection_id, processed_items
        )

        success, errors = await helpers.async_bulk(
            self.client,
            actions,
            refresh=refresh,
            raise_on_error=raise_on_error,
        )
        # Log the result
        logger.info(
            f"Bulk insert completed for collection {collection_id}: {success} successes, {len(errors)} errors"
        )
        return success, errors

    def bulk_sync(
        self,
        collection_id: str,
        processed_items: list[Item],
        **kwargs: Any,
    ) -> tuple[int, list[dict[str, Any]]]:
        """
        Perform a bulk insert of items into the database asynchronously.

        Args:
            collection_id (str): The ID of the collection to which the items belong.
            processed_items (list[Item]): A list of `Item` objects to be inserted into the database.
            **kwargs (Any): Additional keyword arguments, including:
                - refresh (str, optional): Whether to refresh the index after the bulk insert.
                Can be "true", "false", or "wait_for". Defaults to the value of `self.sync_settings.database_refresh`.
                - refresh (bool, optional): Whether to refresh the index after the bulk insert.
                - raise_on_error (bool, optional): Whether to raise an error if any of the bulk operations fail.
                Defaults to the value of `self.async_settings.raise_on_bulk_error`.

        Returns:
            tuple[int, list[dict[str, Any]]]: A tuple containing:
                - The number of successfully processed actions (`success`).
                - A list of errors encountered during the bulk operation (`errors`).

        Notes:
            This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
            The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
            completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. The `refresh`
            parameter determines whether the index is refreshed after the bulk insert:
                - "true": Forces an immediate refresh of the index.
                - "false": Does not refresh the index immediately (default behavior).
                - "wait_for": Waits for the next refresh cycle to make the changes visible.
        """
        # Ensure kwargs is a dictionary
        kwargs = kwargs or {}

        # Resolve the `refresh` parameter
        refresh = kwargs.get("refresh", self.async_settings.database_refresh)
        refresh = validate_refresh(refresh)

        # Log the bulk insert attempt
        logger.info(
            f"Performing bulk insert for collection {collection_id} with refresh={refresh}"
        )

        # Handle empty processed_items
        if not processed_items:
            logger.warning(f"No items to insert for collection {collection_id}")
            return 0, []

        raise_on_error = self.sync_settings.raise_on_bulk_error
        success, errors = helpers.bulk(
            self.sync_client,
            mk_actions(collection_id, processed_items),
            refresh=refresh,
            raise_on_error=raise_on_error,
        )
        return success, errors

    # DANGER
    async def delete_items(self) -> None:
        """Danger. this is only for tests."""
        await self.client.delete_by_query(
            index=ITEM_INDICES,
            body={"query": {"match_all": {}}},
            wait_for_completion=True,
        )

    # DANGER
    async def delete_collections(self) -> None:
        """Danger. this is only for tests."""
        await self.client.delete_by_query(
            index=COLLECTIONS_INDEX,
            body={"query": {"match_all": {}}},
            wait_for_completion=True,
        )

    """CATALOGS LOGIC"""

    @retry_on_connection_error
    async def get_all_catalogs(
        self,
        token: str | None,
        limit: int,
        request: Any = None,
        sort: list[dict[str, Any]] | None = None,
    ) -> tuple[list[dict[str, Any]], str | None, int | None]:
        """Retrieve a list of catalogs from OpenSearch, supporting pagination.

        Args:
            token (str | None): The pagination token.
            limit (int): The number of results to return.
            request (Any, optional): The FastAPI request object. Defaults to None.
            sort (list[dict[str, Any]] | None, optional): Optional sort parameter. Defaults to None.

        Returns:
            A tuple of (catalogs, next pagination token if any, optional count).
        """
        # Define sortable fields for catalogs
        sortable_fields = ["id"]

        # Format the sort parameter
        formatted_sort = []
        if sort:
            for item in sort:
                field = item.get("field")
                direction = item.get("direction", "asc")
                if field and field in sortable_fields:
                    formatted_sort.append({field: {"order": direction}})

        if not formatted_sort:
            formatted_sort = [{"id": {"order": "asc"}}]

        body = {
            "sort": formatted_sort,
            "size": limit,
            "query": {"term": {"type": "Catalog"}},
            "_source": True,  # Ensure all fields including parent_ids are returned
        }

        # Handle search_after token
        search_after = None
        if token:
            try:
                search_after = token.split("|")
                # Validate token format: must have correct number of values and be non-empty
                if len(search_after) != len(formatted_sort):
                    search_after = None
                else:
                    # Validate each value is non-empty (check for patterns like "id||date")
                    for val in search_after:
                        if not val:
                            search_after = None
                            break
            except Exception:
                search_after = None

            if search_after is not None:
                body["search_after"] = search_after

        # Search for catalogs in collections index
        response = await self.client.search(
            index=COLLECTIONS_INDEX,
            body=body,
        )

        hits = response["hits"]["hits"]
        catalogs = [hit["_source"] for hit in hits]

        next_token = None
        if len(hits) == limit:
            next_token_values = hits[-1].get("sort")
            if next_token_values:
                next_token = "|".join(str(val) for val in next_token_values)

        # Get the total count
        matched = (
            response["hits"]["total"]["value"]
            if response["hits"]["total"]["relation"] == "eq"
            else None
        )

        return catalogs, next_token, matched

    @retry_on_connection_error
    async def create_catalog(self, catalog: dict, refresh: bool = False) -> None:
        """Create a catalog in OpenSearch.

        Args:
            catalog (dict): The catalog document to create.
            refresh (bool): Whether to refresh the index after creation.
        """
        await self.client.index(
            index=COLLECTIONS_INDEX,
            id=catalog.get("id"),
            body=catalog,
            refresh=refresh,
        )

    @retry_on_connection_error
    async def find_catalog(self, catalog_id: str) -> dict:
        """Find a catalog in OpenSearch by ID.

        Args:
            catalog_id (str): The ID of the catalog to find.

        Returns:
            dict: The catalog document.

        Raises:
            NotFoundError: If the catalog is not found.
        """
        try:
            response = await self.client.get(
                index=COLLECTIONS_INDEX,
                id=catalog_id,
            )
            # Verify it's a catalog
            if response["_source"].get("type") != "Catalog":
                raise NotFoundError(f"Catalog {catalog_id} not found")
            return response["_source"]
        except exceptions.NotFoundError:
            raise NotFoundError(f"Catalog {catalog_id} not found")

    @retry_on_connection_error
    async def delete_catalog(self, catalog_id: str, refresh: bool = False) -> None:
        """Delete a catalog from OpenSearch.

        Args:
            catalog_id (str): The ID of the catalog to delete.
            refresh (bool): Whether to refresh the index after deletion.
        """
        await self.client.delete(
            index=COLLECTIONS_INDEX,
            id=catalog_id,
            refresh=refresh,
        )
