from __future__ import annotations

import contextlib
import datetime
import json
from abc import abstractmethod
from collections import ChainMap, defaultdict
from collections.abc import Iterator, Sequence
from dataclasses import InitVar, dataclass, fields
from enum import IntEnum
from functools import cached_property, partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Literal, NoReturn, TypeAlias, TypeVar, overload
from zoneinfo import ZoneInfo

from typing_extensions import Self

from cognite.client._constants import NUMPY_IS_AVAILABLE
from cognite.client.data_classes._base import (
    CogniteResource,
    CogniteResourceList,
    CogniteResourceListWithClientRef,
    IdTransformerMixin,
)
from cognite.client.data_classes.data_modeling import NodeId
from cognite.client.data_classes.datapoint_aggregates import (
    _INT_AGGREGATES_CAMEL,
    ALL_SORTED_DP_AGGS,
    Aggregate,
)
from cognite.client.utils import _json_extended as _json
from cognite.client.utils._auxiliary import find_duplicates
from cognite.client.utils._identifier import Identifier, InstanceId
from cognite.client.utils._importing import local_import
from cognite.client.utils._pandas_helpers import (
    concat_dps_dataframe_list,
    convert_dps_to_dataframe,
    convert_tz_for_pandas,
    notebook_display_with_fallback,
)
from cognite.client.utils._text import (
    convert_all_keys_to_camel_case,
    convert_all_keys_to_snake_case,
    iterable_to_case,
    to_camel_case,
    to_snake_case,
)
from cognite.client.utils._time import (
    convert_and_isoformat_timestamp,
    convert_timezone_to_str,
    datetime_to_ms,
    ms_to_datetime,
    parse_str_timezone,
)

if NUMPY_IS_AVAILABLE:
    import numpy as np

if TYPE_CHECKING:
    import numpy.typing as npt
    import pandas

    from cognite.client._api.datapoint_tasks import BaseTaskOrchestrator

    NumpyDatetime64NSArray: TypeAlias = npt.NDArray[np.datetime64]
    NumpyUInt32Array: TypeAlias = npt.NDArray[np.uint32]
    NumpyInt64Array: TypeAlias = npt.NDArray[np.int64]
    NumpyFloat64Array: TypeAlias = npt.NDArray[np.float64]
    NumpyObjArray: TypeAlias = npt.NDArray[np.object_]


_T_DPS = TypeVar("_T_DPS", "Datapoints", "DatapointsArray")


def numpy_dtype_fix(
    element: np.float64 | str | MaxOrMinDatapoint, camel_case: bool = False
) -> float | str | dict[str, int | float | str]:
    try:
        # Using .item() on numpy scalars gives us vanilla python types:
        return element.item()  # type: ignore [union-attr]
    except AttributeError:
        # Return no-op as array contains just references to vanilla python objects:
        if isinstance(element, str):
            return element
        elif isinstance(element, MaxOrMinDatapoint):
            return element.dump(camel_case=camel_case)
        raise


def build_id_mapping_with_duplicates(
    dps_objs: Sequence[_T_DPS], attr: Literal["id", "external_id", "instance_id"]
) -> dict[Any, _T_DPS | list[_T_DPS]]:
    identifiers = list(filter(lambda v: v is not None, (getattr(dps, attr) for dps in dps_objs)))
    duplicates = find_duplicates(identifiers)
    result: dict[Any, _T_DPS | list[_T_DPS]] = {}
    for dps in dps_objs:
        if (ident := getattr(dps, attr)) is not None:
            if ident in duplicates:
                result.setdefault(ident, []).append(dps)  # type: ignore [union-attr]
            else:
                result[ident] = dps
    return result


class StatusCode(IntEnum):
    """The three main categories of status codes"""

    Good = 0x0
    Uncertain = 0x40000000  # aka 1 << 30 aka 1073741824
    Bad = 0x80000000  # aka 1 << 31 aka 2147483648


@dataclass(slots=True, frozen=True)
class MaxOrMinDatapoint:
    @abstractmethod
    def dump(self, camel_case: bool = True) -> dict[str, Any]: ...

    @classmethod
    @abstractmethod
    def _load(cls, dct: dict[str, Any]) -> Self: ...


@dataclass(slots=True, frozen=True)
class MinDatapoint(MaxOrMinDatapoint):
    timestamp: int
    value: float

    @property
    def status_code(self) -> NoReturn:
        raise AttributeError(
            "'MinDatapoint' object has no attribute 'status_code'. Tip: fetch using `include_status=True`."
        )

    @property
    def status_symbol(self) -> NoReturn:
        raise AttributeError(
            "'MinDatapoint' object has no attribute 'status_symbol'. Tip: fetch using `include_status=True`."
        )

    @classmethod
    def _load(cls, dct: dict[str, Any]) -> Self:
        assert "statusCode" not in dct
        return cls(dct["timestamp"], dct["value"])

    def dump(self, camel_case: bool = True) -> dict[str, Any]:
        return {"timestamp": self.timestamp, "value": self.value}


@dataclass(slots=True, frozen=True)
class MinDatapointWithStatus(MinDatapoint):
    status_code: int  # type: ignore [assignment]
    status_symbol: str  # type: ignore [assignment]

    @classmethod
    def _load(cls, dct: dict[str, Any]) -> Self:
        return cls(dct["timestamp"], dct["value"], dct["statusCode"], dct["statusSymbol"])

    def dump(self, camel_case: bool = True) -> dict[str, Any]:
        return {
            "timestamp": self.timestamp,
            "value": self.value,
            "statusCode" if camel_case else "status_code": self.status_code,
            "statusSymbol" if camel_case else "status_symbol": self.status_symbol,
        }


@dataclass(slots=True, frozen=True)
class MaxDatapoint(MaxOrMinDatapoint):
    timestamp: int
    value: float

    @property
    def status_code(self) -> NoReturn:
        raise AttributeError(
            "'MaxDatapoint' object has no attribute 'status_code'. Tip: fetch using `include_status=True`."
        )

    @property
    def status_symbol(self) -> NoReturn:
        raise AttributeError(
            "'MaxDatapoint' object has no attribute 'status_symbol'. Tip: fetch using `include_status=True`."
        )

    @classmethod
    def _load(cls, dct: dict[str, Any]) -> Self:
        assert "statusCode" not in dct
        return cls(dct["timestamp"], dct["value"])

    def dump(self, camel_case: bool = True) -> dict[str, Any]:
        return {"timestamp": self.timestamp, "value": self.value}


@dataclass(slots=True, frozen=True)
class MaxDatapointWithStatus(MaxDatapoint):
    status_code: int  # type: ignore [assignment]
    status_symbol: str  # type: ignore [assignment]

    @classmethod
    def _load(cls, dct: dict[str, Any]) -> Self:
        return cls(dct["timestamp"], dct["value"], dct["statusCode"], dct["statusSymbol"])

    def dump(self, camel_case: bool = True) -> dict[str, Any]:
        return {
            "timestamp": self.timestamp,
            "value": self.value,
            "statusCode" if camel_case else "status_code": self.status_code,
            "statusSymbol" if camel_case else "status_symbol": self.status_symbol,
        }


def _min_dp_class(dct: dict[str, Any]) -> type[MinDatapoint | MinDatapointWithStatus]:
    return MinDatapointWithStatus if "statusCode" in dct else MinDatapoint


def _max_dp_class(dct: dict[str, Any]) -> type[MaxDatapoint | MaxDatapointWithStatus]:
    return MaxDatapointWithStatus if "statusCode" in dct else MaxDatapoint


@dataclass
class DatapointsQuery:
    """Represent a user request for datapoints for a single time series"""

    _NOT_SET = object()
    _API_DEFAULTS: ClassVar[MappingProxyType[str, Any]] = MappingProxyType(
        {
            "start": 0,
            "end": "now",
            "aggregates": None,
            "granularity": None,
            "timezone": None,
            "target_unit": None,
            "target_unit_system": None,
            "limit": None,
            "include_outside_points": False,
            "ignore_unknown_ids": False,
            "include_status": False,
            "ignore_bad_datapoints": True,
            "treat_uncertain_as_bad": True,
        }
    )
    id: InitVar[int | None] = None
    external_id: InitVar[str | None] = None
    instance_id: InitVar[NodeId | tuple[str, str] | None] = None
    start: int | str | datetime.datetime = _NOT_SET  # type: ignore [assignment]
    end: int | str | datetime.datetime = _NOT_SET  # type: ignore [assignment]
    aggregates: Aggregate | list[Aggregate] | None = _NOT_SET  # type: ignore [assignment]
    granularity: str | None = _NOT_SET  # type: ignore [assignment]
    timezone: str | datetime.timezone | ZoneInfo | None = _NOT_SET  # type: ignore [assignment]
    target_unit: str | None = _NOT_SET  # type: ignore [assignment]
    target_unit_system: str | None = _NOT_SET  # type: ignore [assignment]
    limit: int | None = _NOT_SET  # type: ignore [assignment]
    include_outside_points: bool = _NOT_SET  # type: ignore [assignment]
    ignore_unknown_ids: bool = _NOT_SET  # type: ignore [assignment]
    include_status: bool = _NOT_SET  # type: ignore [assignment]
    ignore_bad_datapoints: bool = _NOT_SET  # type: ignore [assignment]
    treat_uncertain_as_bad: bool = _NOT_SET  # type: ignore [assignment]

    def __post_init__(
        self, id: int | None, external_id: str | None, instance_id: NodeId | tuple[str, str] | None
    ) -> None:
        # Ensure user have just specified one of id/xid:
        self._identifier = Identifier.of_either(id, external_id, instance_id)
        if instance_id is not None:
            # dump/load is used in parsing and it loses info of this being a NodeId (loads back as InstanceId). We need
            # to lookup by identifier to sort (match user queries), and then InstanceId won't compare equal to NodeId:
            self._identifier = Identifier(NodeId(*self._identifier.as_primitive().as_tuple()))
        # Store the possibly custom granularity (we support more than the API and a translation is done)
        self._original_granularity = self.granularity

    def __eq__(self, other: object) -> bool:
        # Note: Instances representing identical queries should -not- compare equal as this would mean we
        #       would drop all-but-one of them - and the API support duplicated queries.
        if not isinstance(other, DatapointsQuery):
            return NotImplemented
        return self is other

    def __hash__(self) -> int:
        return hash(id(self))  # See note on __eq__

    @classmethod
    def valid_from_user_query(cls, query: Self, **settings: Any) -> Self:
        return cls(**ChainMap(query.dump(), settings, cls._API_DEFAULTS))  # type: ignore [arg-type]

    @property
    def identifier(self) -> Identifier:
        return self._identifier

    @property
    def original_granularity(self) -> str | None:
        return self._original_granularity

    @property
    def original_timezone(self) -> datetime.timezone | ZoneInfo | None:
        return self._original_timezone

    @original_timezone.setter
    def original_timezone(self, tz: datetime.timezone | ZoneInfo | None) -> None:
        self._original_timezone = tz

    @cached_property
    def aggs_camel_case(self) -> list[str]:
        return list(map(to_camel_case, self.aggregates or []))

    @property
    def start_ms(self) -> int:
        assert isinstance(self.start, int)
        return self.start

    @property
    def end_ms(self) -> int:
        assert isinstance(self.end, int)
        return self.end

    @property
    def is_raw_query(self) -> bool:
        return self._is_raw_query

    @is_raw_query.setter
    def is_raw_query(self, value: bool) -> None:
        assert isinstance(value, bool)
        self._is_raw_query = value

    @property
    def is_missing(self) -> bool:
        return self._is_missing

    @is_missing.setter
    def is_missing(self, value: bool) -> None:
        assert isinstance(value, bool)
        self._is_missing = value

    @property
    def is_calendar_query(self) -> bool:
        return self._is_calendar_query

    @is_calendar_query.setter
    def is_calendar_query(self, value: bool) -> None:
        assert isinstance(value, bool)
        self._is_calendar_query = value

    @cached_property
    def use_cursors(self) -> bool:
        return bool(self.timezone or self.is_calendar_query)

    @property
    def max_query_limit(self) -> int:
        return self._max_query_limit

    @max_query_limit.setter
    def max_query_limit(self, value: int) -> None:
        assert isinstance(value, int)
        self._max_query_limit = value

    @property
    def capped_limit(self) -> int:
        if self.limit is None:
            return self.max_query_limit
        return min(self.limit, self.max_query_limit)

    def __repr__(self) -> str:
        return json.dumps(self.dump(), indent=4)

    def dump(self) -> dict[str, Any]:
        # We need to dump only those fields specifically passed by the user:
        return {
            **self.identifier.as_dict(camel_case=False),
            **dict((fld.name, val) for fld in fields(self) if (val := getattr(self, fld.name)) is not self._NOT_SET),
        }

    @property
    def task_orchestrator(self) -> type[BaseTaskOrchestrator]:
        from cognite.client._api.datapoint_tasks import get_task_orchestrator

        return get_task_orchestrator(self)

    def to_payload_item(self) -> dict[str, Any]:
        payload = {
            **self.identifier.as_dict(),
            "start": self.start,
            "end": self.end,
            "limit": self.capped_limit,
        }
        if self.target_unit is not None:
            payload["targetUnit"] = self.target_unit
        elif self.target_unit_system is not None:
            payload["targetUnitSystem"] = self.target_unit_system

        if self.ignore_bad_datapoints is False:
            payload["ignoreBadDataPoints"] = self.ignore_bad_datapoints
        if self.treat_uncertain_as_bad is False:
            payload["treatUncertainAsBad"] = self.treat_uncertain_as_bad
        if self.timezone:
            payload["timeZone"] = self.timezone
        if self.is_raw_query:
            if self.include_outside_points is True:
                payload["includeOutsidePoints"] = self.include_outside_points
            if self.include_status is True:
                payload["includeStatus"] = self.include_status
        else:
            payload.update(aggregates=self.aggs_camel_case, granularity=self.granularity)
        return payload


@dataclass(frozen=True)
class LatestDatapointQuery:
    """Parameters describing a query for the latest datapoint from a time series.

    Note:
        Pass either ID, external ID or instance ID.

    Args:
        id (InitVar[int | None]): The internal ID of the time series to query.
        external_id (InitVar[str | None]): The external ID of the time series to query.
        instance_id (InitVar[NodeId | None]): The instance ID of the time series to query.
        before (None | int | str | datetime.datetime): Get latest datapoint before this time. None means 'now'.
        target_unit (str | None): The unit_external_id of the data points returned. If the time series does not have a unit_external_id that can be converted to the target_unit, an error will be returned. Cannot be used with target_unit_system.
        target_unit_system (str | None): The unit system of the data points returned. Cannot be used with target_unit.
        include_status (bool | None): Also return the status code, an integer, for each datapoint in the response.
        ignore_bad_datapoints (bool | None): Prevent data points with a bad status code to be returned. Default: True.
        treat_uncertain_as_bad (bool | None): Treat uncertain status codes as bad. If false, treat uncertain as good. Default: True.
    """

    id: InitVar[int | None] = None
    external_id: InitVar[str | None] = None
    instance_id: InitVar[NodeId | None] = None
    before: None | int | str | datetime.datetime = None
    target_unit: str | None = None
    target_unit_system: str | None = None
    include_status: bool | None = None
    ignore_bad_datapoints: bool | None = None
    treat_uncertain_as_bad: bool | None = None

    def __post_init__(self, id: int | None, external_id: str | None, instance_id: NodeId | None) -> None:
        # Ensure user have just specified one of id/external_id/instance_id:
        object.__setattr__(self, "_identifier", Identifier.of_either(id, external_id, instance_id))

    @property
    def identifier(self) -> Identifier:
        return self._identifier  # type: ignore [attr-defined]


class Datapoint(CogniteResource):
    """An object representing a datapoint.

    Args:
        timestamp (int): The data timestamp in milliseconds since the epoch (Jan 1, 1970). Can be negative to define a date before 1970. Minimum timestamp is 1900.01.01 00:00:00 UTC
        value (str | float | None): The raw data value. Can be string or numeric.
        average (float | None): The time-weighted average value in the aggregate interval.
        max (float | None): The maximum value in the aggregate interval.
        max_datapoint (MaxDatapoint | MaxDatapointWithStatus | None): Objects with the maximum values and their timestamps in the aggregate intervals, optionally including status codes and symbols.
        min (float | None): The minimum value in the aggregate interval.
        min_datapoint (MinDatapoint | MinDatapointWithStatus | None): Objects with the minimum values and their timestamps in the aggregate intervals, optionally including status codes and symbols.
        count (int | None): The number of raw datapoints in the aggregate interval.
        sum (float | None): The sum of the raw datapoints in the aggregate interval.
        interpolation (float | None): The interpolated value at the beginning of the aggregate interval.
        step_interpolation (float | None): The interpolated value at the beginning of the aggregate interval using stepwise interpretation.
        continuous_variance (float | None): The variance of the interpolated underlying function.
        discrete_variance (float | None): The variance of the datapoint values.
        total_variation (float | None): The total variation of the interpolated underlying function.
        count_bad (int | None): The number of raw datapoints with a bad status code, in the aggregate interval.
        count_good (int | None): The number of raw datapoints with a good status code, in the aggregate interval.
        count_uncertain (int | None): The number of raw datapoints with a uncertain status code, in the aggregate interval.
        duration_bad (int | None): The duration the aggregate is defined and marked as bad (measured in milliseconds).
        duration_good (int | None): The duration the aggregate is defined and marked as good (measured in milliseconds).
        duration_uncertain (int | None): The duration the aggregate is defined and marked as uncertain (measured in milliseconds).
        status_code (int | None): The status code for the raw datapoint.
        status_symbol (str | None): The status symbol for the raw datapoint.
        timezone (datetime.timezone | ZoneInfo | None): The timezone to use when displaying the datapoint.
    """

    def __init__(
        self,
        timestamp: int,
        value: str | float | None = None,
        average: float | None = None,
        max: float | None = None,
        max_datapoint: MaxDatapoint | MaxDatapointWithStatus | None = None,
        min: float | None = None,
        min_datapoint: MinDatapoint | MinDatapointWithStatus | None = None,
        count: int | None = None,
        sum: float | None = None,
        interpolation: float | None = None,
        step_interpolation: float | None = None,
        continuous_variance: float | None = None,
        discrete_variance: float | None = None,
        total_variation: float | None = None,
        count_bad: int | None = None,
        count_good: int | None = None,
        count_uncertain: int | None = None,
        duration_bad: int | None = None,
        duration_good: int | None = None,
        duration_uncertain: int | None = None,
        status_code: int | None = None,
        status_symbol: str | None = None,
        timezone: datetime.timezone | ZoneInfo | None = None,
    ) -> None:
        self.timestamp = timestamp
        self.value = value
        self.average = average
        self.max = max
        self.max_datapoint = max_datapoint
        self.min = min
        self.min_datapoint = min_datapoint
        self.count = count
        self.sum = sum
        self.interpolation = interpolation
        self.step_interpolation = step_interpolation
        self.continuous_variance = continuous_variance
        self.discrete_variance = discrete_variance
        self.total_variation = total_variation
        self.count_bad = count_bad
        self.count_good = count_good
        self.count_uncertain = count_uncertain
        self.duration_bad = duration_bad
        self.duration_good = duration_good
        self.duration_uncertain = duration_uncertain
        self.status_code = status_code
        self.status_symbol = status_symbol
        self.timezone = timezone

    def __str__(self) -> str:
        item = self.dump(camel_case=False)
        item["timestamp"] = convert_and_isoformat_timestamp(self.timestamp, self.timezone)
        return _json.dumps(item, indent=4)

    def to_pandas(self, camel_case: bool = False) -> pandas.DataFrame:  # type: ignore[override]
        """Convert the datapoint into a pandas DataFrame.

        Args:
            camel_case (bool): Convert column names to camel case (e.g. `stepInterpolation` instead of `step_interpolation`)

        Returns:
            pandas.DataFrame: The DataFrame representation of the datapoint.
        """
        pd = local_import("pandas")

        dumped = self.dump(camel_case=camel_case)
        for key in iterable_to_case(["min_datapoint", "max_datapoint"], camel_case):
            if dp := dumped.get(key):
                dumped[key] = [dp]  # make pandas treat this dict as a scalar value

        timestamp = dumped.pop("timestamp")
        tz = convert_tz_for_pandas(self.timezone)
        return pd.DataFrame(dumped, index=[pd.Timestamp(timestamp, unit="ms", tz=tz)])

    @classmethod
    def _load(cls, resource: dict[str, Any]) -> Self:
        match max_dp_raw := resource.get("maxDatapoint"):
            case dict():
                max_datapoint: MaxDatapoint | None = _max_dp_class(max_dp_raw)._load(max_dp_raw)
            case MaxDatapoint() | None:
                max_datapoint = max_dp_raw
            case _:
                raise TypeError(f"Expected dict or MaxDatapoint, got {type(max_dp_raw)}")

        match min_dp_raw := resource.get("minDatapoint"):
            case dict():
                min_datapoint: MinDatapoint | None = _min_dp_class(min_dp_raw)._load(min_dp_raw)
            case MinDatapoint() | None:
                min_datapoint = min_dp_raw
            case _:
                raise TypeError(f"Expected dict or MinDatapoint, got {type(min_dp_raw)}")

        timezone = None
        if (raw_timezone := resource.get("timezone")) is not None:
            if isinstance(raw_timezone, (datetime.timezone, ZoneInfo)):
                timezone = raw_timezone
            else:
                with contextlib.suppress(ValueError):
                    timezone = parse_str_timezone(raw_timezone)

        return cls(
            timestamp=resource["timestamp"],
            value=resource.get("value"),
            average=resource.get("average"),
            max=resource.get("max"),
            max_datapoint=max_datapoint,
            min=resource.get("min"),
            min_datapoint=min_datapoint,
            count=resource.get("count"),
            sum=resource.get("sum"),
            interpolation=resource.get("interpolation"),
            step_interpolation=resource.get("stepInterpolation"),
            continuous_variance=resource.get("continuousVariance"),
            discrete_variance=resource.get("discreteVariance"),
            total_variation=resource.get("totalVariation"),
            count_bad=resource.get("countBad"),
            count_good=resource.get("countGood"),
            count_uncertain=resource.get("countUncertain"),
            duration_bad=resource.get("durationBad"),
            duration_good=resource.get("durationGood"),
            duration_uncertain=resource.get("durationUncertain"),
            status_code=resource.get("statusCode"),
            status_symbol=resource.get("statusSymbol"),
            timezone=timezone,
        )

    def dump(self, camel_case: bool = True, include_timezone: bool = True) -> dict[str, Any]:
        dumped = super().dump(camel_case=camel_case)
        # Keep value even if None (bad status codes support missing):
        dumped["value"] = self.value  # TODO: What if Datapoint represents one or more aggregates?
        if self.max_datapoint:
            dumped["maxDatapoint" if camel_case else "max_datapoint"] = self.max_datapoint.dump(camel_case)
        if self.min_datapoint:
            dumped["minDatapoint" if camel_case else "min_datapoint"] = self.min_datapoint.dump(camel_case)
        if include_timezone:
            if self.timezone is not None:
                dumped["timezone"] = convert_timezone_to_str(self.timezone)
        else:
            dumped.pop("timezone", None)
        return dumped


class DatapointsArray(CogniteResource):
    """An object representing datapoints using numpy arrays."""

    def __init__(
        self,
        id: int,
        is_string: bool,
        is_step: bool,
        type: Literal["numeric", "string", "state"],
        external_id: str | None = None,
        instance_id: NodeId | None = None,
        unit: str | None = None,
        unit_external_id: str | None = None,
        granularity: str | None = None,
        timestamp: NumpyDatetime64NSArray | None = None,
        value: NumpyFloat64Array | NumpyObjArray | None = None,
        average: NumpyFloat64Array | None = None,
        max: NumpyFloat64Array | None = None,
        max_datapoint: NumpyObjArray | None = None,
        min: NumpyFloat64Array | None = None,
        min_datapoint: NumpyObjArray | None = None,
        count: NumpyInt64Array | None = None,
        sum: NumpyFloat64Array | None = None,
        interpolation: NumpyFloat64Array | None = None,
        step_interpolation: NumpyFloat64Array | None = None,
        continuous_variance: NumpyFloat64Array | None = None,
        discrete_variance: NumpyFloat64Array | None = None,
        total_variation: NumpyFloat64Array | None = None,
        count_bad: NumpyInt64Array | None = None,
        count_good: NumpyInt64Array | None = None,
        count_uncertain: NumpyInt64Array | None = None,
        duration_bad: NumpyInt64Array | None = None,
        duration_good: NumpyInt64Array | None = None,
        duration_uncertain: NumpyInt64Array | None = None,
        status_code: NumpyUInt32Array | None = None,
        status_symbol: NumpyObjArray | None = None,
        null_timestamps: set[int] | None = None,
        timezone: datetime.timezone | ZoneInfo | None = None,
    ) -> None:
        self.id = id
        self.external_id = external_id
        self.instance_id = instance_id
        self.is_string = is_string
        self.is_step = is_step
        self.type = type
        self.unit = unit
        self.unit_external_id = unit_external_id
        self.granularity = granularity
        self.timestamp: NumpyDatetime64NSArray = (
            timestamp if timestamp is not None else np.array([], dtype="datetime64[ns]")
        )
        self.value = value
        self.average = average
        self.max = max
        self.max_datapoint = max_datapoint
        self.min = min
        self.min_datapoint = min_datapoint
        self.count = count
        self.sum = sum
        self.interpolation = interpolation
        self.step_interpolation = step_interpolation
        self.continuous_variance = continuous_variance
        self.discrete_variance = discrete_variance
        self.total_variation = total_variation
        self.count_bad = count_bad
        self.count_good = count_good
        self.count_uncertain = count_uncertain
        self.duration_bad = duration_bad
        self.duration_good = duration_good
        self.duration_uncertain = duration_uncertain
        self.status_code = status_code
        self.status_symbol = status_symbol
        self.null_timestamps = null_timestamps
        self.timezone = timezone

    @property
    def _ts_info(self) -> dict[str, Any]:
        return {
            "id": self.id,
            "external_id": self.external_id,
            "instance_id": self.instance_id,
            "is_string": self.is_string,
            "is_step": self.is_step,
            "type": self.type,
            "unit": self.unit,
            "unit_external_id": self.unit_external_id,
            "granularity": self.granularity,
            "timezone": None if self.timezone is None else convert_timezone_to_str(self.timezone),
        }

    @classmethod
    def _load_from_arrays(
        cls,
        dps_dct: dict[str, Any],
    ) -> DatapointsArray:
        assert isinstance(dps_dct["timestamp"], np.ndarray)  # mypy love
        # We store datetime using nanosecond resolution to future-proof the SDK in case it is ever added:
        dps_dct["timestamp"] = dps_dct["timestamp"].astype("datetime64[ms]").astype("datetime64[ns]")
        return cls(**convert_all_keys_to_snake_case(dps_dct))

    @classmethod
    def _load(
        cls,
        dps_dct: dict[str, Any],
    ) -> DatapointsArray:
        array_by_attr = {}
        if "datapoints" in dps_dct:
            datapoints_by_attr = defaultdict(list)
            for row in dps_dct["datapoints"]:
                for attr, value in row.items():
                    datapoints_by_attr[attr].append(value)
            status = datapoints_by_attr.pop("status", None)
            for attr, values in datapoints_by_attr.items():
                if attr == "timestamp":
                    array_by_attr[attr] = np.array(values, dtype="datetime64[ms]").astype("datetime64[ns]")
                elif attr in _INT_AGGREGATES_CAMEL:
                    array_by_attr[attr] = np.array(values, dtype=np.int64)
                else:
                    try:
                        array_by_attr[attr] = np.array(values, dtype=np.float64)
                    except ValueError:
                        array_by_attr[attr] = np.array(values, dtype=np.object_)
            if status is not None:
                array_by_attr["statusCode"] = np.array([s["code"] for s in status], dtype=np.uint32)
                array_by_attr["statusSymbol"] = np.array([s["symbol"] for s in status], dtype=np.object_)

        timezone = dps_dct.get("timezone")
        if isinstance(timezone, str):
            with contextlib.suppress(ValueError):  # Dont fail load if invalid (TODO: warn?)
                timezone = parse_str_timezone(timezone)
        return cls(
            id=dps_dct["id"],
            external_id=dps_dct.get("externalId"),
            instance_id=NodeId._load_if(dps_dct.get("instanceId")),
            is_step=dps_dct["isStep"],
            is_string=dps_dct["isString"],
            unit=dps_dct.get("unit"),
            type=dps_dct["type"],
            granularity=dps_dct.get("granularity"),
            unit_external_id=dps_dct.get("unitExternalId"),
            timestamp=array_by_attr.get("timestamp"),
            value=array_by_attr.get("value"),
            average=array_by_attr.get("average"),
            max=array_by_attr.get("max"),
            min=array_by_attr.get("min"),
            count=array_by_attr.get("count"),
            sum=array_by_attr.get("sum"),
            interpolation=array_by_attr.get("interpolation"),
            step_interpolation=array_by_attr.get("stepInterpolation"),
            continuous_variance=array_by_attr.get("continuousVariance"),
            discrete_variance=array_by_attr.get("discreteVariance"),
            total_variation=array_by_attr.get("totalVariation"),
            count_bad=array_by_attr.get("countBad"),
            count_good=array_by_attr.get("countGood"),
            count_uncertain=array_by_attr.get("countUncertain"),
            duration_bad=array_by_attr.get("durationBad"),
            duration_good=array_by_attr.get("durationGood"),
            duration_uncertain=array_by_attr.get("durationUncertain"),
            status_code=array_by_attr.get("statusCode"),
            status_symbol=array_by_attr.get("statusSymbol"),
            null_timestamps=set(dps_dct["nullTimestamps"]) if "nullTimestamps" in dps_dct else None,
            timezone=timezone,  # type: ignore [arg-type]
        )

    def __len__(self) -> int:
        return len(self.timestamp)

    def __eq__(self, other: Any) -> bool:
        # Override CogniteResource __eq__ which checks exact type & dump being equal. We do not want
        # this: comparing arrays with (mostly) floats is a very bad idea; also dump is exceedingly expensive.
        return id(self) == id(other)

    def __str__(self) -> str:
        return _json.dumps(self.dump(convert_timestamps=True), indent=4)

    @overload
    def __getitem__(self, item: int) -> Datapoint: ...

    @overload
    def __getitem__(self, item: slice) -> DatapointsArray: ...

    def __getitem__(self, item: int | slice) -> Datapoint | DatapointsArray:
        if isinstance(item, slice):
            return self._slice(item)
        attrs, arrays = self._data_fields()
        timestamp = arrays[0][item].item() // 1_000_000
        data: dict[str, float | str | dict | None] = {
            attr: numpy_dtype_fix(arr[item]) for attr, arr in zip(attrs[1:], arrays[1:])
        }
        for key in ("min_datapoint", "max_datapoint"):
            if key in data:
                data[key] = getattr(self, key)[item]
        if self.status_code is not None:
            data.update(status_code=self.status_code[item], status_symbol=self.status_symbol[item])  # type: ignore [index]
        if self.null_timestamps and timestamp in self.null_timestamps:
            data["value"] = None
        return Datapoint(timestamp=timestamp, **data, timezone=self.timezone)  # type: ignore [arg-type]

    def _slice(self, part: slice) -> DatapointsArray:
        data: dict[str, Any] = {attr: arr[part] for attr, arr in zip(*self._data_fields())}
        if self.status_code is not None:
            data.update(status_code=self.status_code[part], status_symbol=self.status_symbol[part])  # type: ignore [index]
        if self.null_timestamps is not None:
            data["null_timestamps"] = self.null_timestamps.intersection(
                data["timestamp"].astype("datetime64[ms]").astype(np.int64).tolist()
            )
        return DatapointsArray(**self._ts_info, **data)

    def __iter__(self) -> NoReturn:
        raise NotImplementedError(
            "Iterating through a DatapointsArray is very inefficient and support was dropped in major version 8. "
            "Tip: Access the arrays directly and use vectorised numpy ops on those. E.g. `dps.average` for the "
            "'average' aggregate, `dps.value` for the raw datapoints or `dps.timestamp` for the timestamps. You may "
            "also convert to a pandas DataFrame using `dps.to_pandas()`.",
        )

    def _data_fields(self) -> tuple[list[str], list[npt.NDArray]]:
        # Note: Does not return status-related fields
        data_field_tuples = [
            (attr, arr)
            for attr in ("timestamp", "value", *ALL_SORTED_DP_AGGS)  # ts must be first
            if (arr := getattr(self, attr)) is not None
        ]
        attrs, arrays = map(list, zip(*data_field_tuples))
        return attrs, arrays

    def dump(self, camel_case: bool = True, convert_timestamps: bool = False) -> dict[str, Any]:
        """Dump the DatapointsArray into a json serializable Python data type.

        Args:
            camel_case (bool): Use camelCase for attribute names. Defaults to True.
            convert_timestamps (bool): Convert timestamps to ISO 8601 formatted strings. Default: False (returns as integer, milliseconds since epoch)

        Returns:
            dict[str, Any]: A dictionary representing the instance.
        """
        attrs, arrays = self._data_fields()
        if not convert_timestamps:  # Eh.. so.. we still have to convert...
            arrays[0] = arrays[0].astype("datetime64[ms]").astype(np.int64)
        else:
            # Note: numpy does not have a strftime method to get the exact format we want (hence the datetime detour)
            #       and for some weird reason .astype(datetime) directly from dt64 returns native integer... whatwhyy
            if self.timezone is None:
                arrays[0] = arrays[0].astype("datetime64[ms]").astype(datetime.datetime).astype(str)
            else:
                arrays[0] = np.array(
                    [
                        convert_and_isoformat_timestamp(ts, self.timezone)
                        for ts in arrays[0].astype("datetime64[ms]").astype(np.int64).tolist()
                    ],
                    dtype=str,
                )

        if camel_case:
            attrs = list(map(to_camel_case, attrs))

        dumped = self._ts_info
        if self.timezone is not None:
            dumped["timezone"] = str(self.timezone)
        if self.instance_id:
            dumped["instance_id"] = self.instance_id.dump(camel_case=camel_case, include_instance_type=False)

        convert_fn = partial(numpy_dtype_fix, camel_case=camel_case)
        datapoints = [dict(zip(attrs, map(convert_fn, row))) for row in zip(*arrays)]

        if self.status_code is not None or self.status_symbol is not None:
            if (
                self.status_code is None
                or self.status_symbol is None
                or not len(self.status_symbol) == len(datapoints) == len(self.status_code)
            ):
                raise ValueError("The number of status codes/symbols does not match the number of datapoints")

            for dp, code, symbol in zip(datapoints, map(numpy_dtype_fix, self.status_code), self.status_symbol):
                dp["status"] = {"code": code, "symbol": symbol}  # type: ignore [dict-item]

        # When we're dealing with datapoints with bad status codes, NaN might be either one of [<missing>, nan]:
        if self.null_timestamps:
            for dp in datapoints:
                if dp["timestamp"] in self.null_timestamps:  # ...luckily, we know :3
                    dp["value"] = None  # type: ignore [assignment]
        dumped["datapoints"] = datapoints

        if camel_case:
            dumped = convert_all_keys_to_camel_case(dumped)
        return {k: v for k, v in dumped.items() if v is not None}

    def to_pandas(  # type: ignore [override]
        self,
        include_aggregate_name: bool = True,
        include_granularity_name: bool = False,
        include_unit: bool = True,
        include_status: bool = True,
    ) -> pandas.DataFrame:
        """Convert the DatapointsArray into a pandas DataFrame.

        Args:
            include_aggregate_name (bool): Include aggregate in the dataframe columns, if present (separate MultiIndex level)
            include_granularity_name (bool): Include granularity in the dataframe columns, if present (separate MultiIndex level)
            include_unit (bool): Include the unit_external_id in the dataframe columns, if present (separate MultiIndex level)
            include_status (bool): Include status code and status symbol as separate columns, if available. Also adds the status info
                as a separate level in the columns (MultiIndex).

        Returns:
            pandas.DataFrame: The datapoints as a pandas DataFrame.
        """
        local_import("pandas")  # throw nice import error early

        return convert_dps_to_dataframe(
            self,
            include_aggregate_name=include_aggregate_name,
            include_granularity_name=include_granularity_name,
            include_status=include_status,
            include_unit=include_unit,
        )


class Datapoints(CogniteResource):
    """An object representing a list of datapoints.

    Args:
        id (int): Id of the time series the datapoints belong to
        is_string (bool): Whether the time series contains numerical or string data.
        is_step (bool): Whether the time series is stepwise or continuous.
        type (Literal['numeric', 'string', 'state']): The type of the time series.
        external_id (str | None): External id of the time series the datapoints belong to
        instance_id (NodeId | None): The instance id of the time series the datapoints belong to
        unit (str | None): The physical unit of the time series (free-text field). Omitted if the datapoints were converted to another unit.
        unit_external_id (str | None): The unit_external_id (as defined in the unit catalog) of the returned data points. If the datapoints were converted to a compatible unit, this will equal the converted unit, not the one defined on the time series.
        granularity (str | None): The granularity of the aggregate datapoints (does not apply to raw data)
        timestamp (list[int] | None): The data timestamps in milliseconds since the epoch (Jan 1, 1970). Can be negative to define a date before 1970. Minimum timestamp is 1900.01.01 00:00:00 UTC
        value (list[str] | list[float] | None): The raw data values. Can be string or numeric.
        average (list[float] | None): The time-weighted average values per aggregate interval.
        max (list[float] | None): The maximum values per aggregate interval.
        max_datapoint (list[MaxDatapoint] | list[MaxDatapointWithStatus] | None): Objects with the maximum values and their timestamps in the aggregate intervals, optionally including status codes and symbols.
        min (list[float] | None): The minimum values per aggregate interval.
        min_datapoint (list[MinDatapoint] | list[MinDatapointWithStatus] | None): Objects with the minimum values and their timestamps in the aggregate intervals, optionally including status codes and symbols.
        count (list[int] | None): The number of raw datapoints per aggregate interval.
        sum (list[float] | None): The sum of the raw datapoints per aggregate interval.
        interpolation (list[float] | None): The interpolated values at the beginning of each the aggregate interval.
        step_interpolation (list[float] | None): The interpolated values at the beginning of each the aggregate interval using stepwise interpretation.
        continuous_variance (list[float] | None): The variance of the interpolated underlying function.
        discrete_variance (list[float] | None): The variance of the datapoint values.
        total_variation (list[float] | None): The total variation of the interpolated underlying function.
        count_bad (list[int] | None): The number of raw datapoints with a bad status code, per aggregate interval.
        count_good (list[int] | None): The number of raw datapoints with a good status code, per aggregate interval.
        count_uncertain (list[int] | None): The number of raw datapoints with a uncertain status code, per aggregate interval.
        duration_bad (list[int] | None): The duration the aggregate is defined and marked as bad (measured in milliseconds).
        duration_good (list[int] | None): The duration the aggregate is defined and marked as good (measured in milliseconds).
        duration_uncertain (list[int] | None): The duration the aggregate is defined and marked as uncertain (measured in milliseconds).
        status_code (list[int] | None): The status codes for the raw datapoints.
        status_symbol (list[str] | None): The status symbols for the raw datapoints.
        timezone (datetime.timezone | ZoneInfo | None): The timezone to use when displaying the datapoints.
    """

    def __init__(
        self,
        id: int,
        is_string: bool,
        is_step: bool,
        type: Literal["numeric", "string", "state"],
        external_id: str | None = None,
        instance_id: NodeId | None = None,
        unit: str | None = None,
        unit_external_id: str | None = None,
        granularity: str | None = None,
        timestamp: list[int] | None = None,
        value: list[str] | list[float] | None = None,
        average: list[float] | None = None,
        max: list[float] | None = None,
        max_datapoint: list[MaxDatapoint] | list[MaxDatapointWithStatus] | None = None,
        min: list[float] | None = None,
        min_datapoint: list[MinDatapoint] | list[MinDatapointWithStatus] | None = None,
        count: list[int] | None = None,
        sum: list[float] | None = None,
        interpolation: list[float] | None = None,
        step_interpolation: list[float] | None = None,
        continuous_variance: list[float] | None = None,
        discrete_variance: list[float] | None = None,
        total_variation: list[float] | None = None,
        count_bad: list[int] | None = None,
        count_good: list[int] | None = None,
        count_uncertain: list[int] | None = None,
        duration_bad: list[int] | None = None,
        duration_good: list[int] | None = None,
        duration_uncertain: list[int] | None = None,
        status_code: list[int] | None = None,
        status_symbol: list[str] | None = None,
        timezone: datetime.timezone | ZoneInfo | None = None,
    ) -> None:
        self.id = id
        self.external_id = external_id
        self.instance_id = instance_id
        self.is_string = is_string
        self.is_step = is_step
        self.type = type
        self.unit = unit
        self.unit_external_id = unit_external_id
        self.granularity = granularity
        self.timestamp: list[int] = timestamp or []
        self.value = value
        self.average = average
        self.max = max
        self.max_datapoint = max_datapoint
        self.min = min
        self.min_datapoint = min_datapoint
        self.count = count
        self.sum = sum
        self.interpolation = interpolation
        self.step_interpolation = step_interpolation
        self.continuous_variance = continuous_variance
        self.discrete_variance = discrete_variance
        self.total_variation = total_variation
        self.count_bad = count_bad
        self.count_good = count_good
        self.count_uncertain = count_uncertain
        self.duration_bad = duration_bad
        self.duration_good = duration_good
        self.duration_uncertain = duration_uncertain
        self.status_code = status_code
        self.status_symbol = status_symbol
        self.timezone = timezone

        self.__datapoint_objects: list[Datapoint] | None = None

    def __str__(self) -> str:
        dumped = self.dump()
        for dct in dumped["datapoints"]:
            dct["timestamp"] = convert_and_isoformat_timestamp(dct["timestamp"], self.timezone)
        return _json.dumps(dumped, indent=4)

    def __len__(self) -> int:
        return len(self.timestamp)

    def __eq__(self, other: Any) -> bool:
        return (
            type(self) is type(other)
            and self.id == other.id
            and self.external_id == other.external_id
            and list(self._get_non_empty_data_fields()) == list(other._get_non_empty_data_fields())
        )

    @overload
    def __getitem__(self, item: int) -> Datapoint: ...

    @overload
    def __getitem__(self, item: slice) -> Datapoints: ...

    def __getitem__(self, item: int | slice) -> Datapoint | Datapoints:
        if isinstance(item, slice):
            return self._slice(item)
        dp_args: dict[str, Any] = {"timezone": self.timezone}
        for attr, values in self._get_non_empty_data_fields():
            dp_args[attr] = values[item]

        if self.status_code is not None:
            dp_args.update(status_code=self.status_code[item], status_symbol=self.status_symbol[item])  # type: ignore [index]

        return Datapoint(**dp_args)

    def __iter__(self) -> Iterator[Datapoint]:
        yield from self.__get_datapoint_objects()

    def dump(self, camel_case: bool = True) -> dict[str, Any]:
        """Dump the datapoints into a json serializable Python data type.

        Args:
            camel_case (bool): Use camelCase for attribute names. Defaults to True.

        Returns:
            dict[str, Any]: A dictionary representing the instance.
        """
        dumped: dict[str, Any] = {
            "id": self.id,
            "external_id": self.external_id,
            "is_string": self.is_string,
            "is_step": self.is_step,
            "type": self.type,
            "unit": self.unit,
            "unit_external_id": self.unit_external_id,
        }
        if self.instance_id is not None:
            dumped["instance_id"] = self.instance_id.dump(camel_case=camel_case, include_instance_type=False)
        if self.timezone is not None:
            dumped["timezone"] = convert_timezone_to_str(self.timezone)
        datapoints = [dp.dump(camel_case=camel_case, include_timezone=False) for dp in self.__get_datapoint_objects()]
        if self.status_code is not None or self.status_symbol is not None:
            if (
                self.status_code is None
                or self.status_symbol is None
                or not len(self.status_symbol) == len(datapoints) == len(self.status_code)
            ):
                raise ValueError("The number of status codes/symbols does not match the number of datapoints")
            for dp in datapoints:
                dp["status"] = {"code": dp.pop("statusCode"), "symbol": dp.pop("statusSymbol")}
        dumped["datapoints"] = datapoints

        if camel_case:
            dumped = convert_all_keys_to_camel_case(dumped)
        return {key: value for key, value in dumped.items() if value is not None}

    def to_pandas(  # type: ignore [override]
        self,
        include_aggregate_name: bool = True,
        include_granularity_name: bool = False,
        include_unit: bool = True,
        include_status: bool = True,
    ) -> pandas.DataFrame:
        """Convert the datapoints into a pandas DataFrame.

        Args:
            include_aggregate_name (bool): Include aggregate in the dataframe columns, if present (separate MultiIndex level)
            include_granularity_name (bool): Include granularity in the dataframe columns, if present (separate MultiIndex level)
            include_unit (bool): Include the unit_external_id in the dataframe columns, if present (separate MultiIndex level)
            include_status (bool): Include status code and status symbol as separate columns, if available. Also adds the status info
                as a separate level in the columns (MultiIndex).

        Returns:
            pandas.DataFrame: The dataframe.
        """
        local_import("pandas")  # throw nice import error early

        return convert_dps_to_dataframe(
            self,
            include_aggregate_name=include_aggregate_name,
            include_granularity_name=include_granularity_name,
            include_status=include_status,
            include_unit=include_unit,
        )

    @classmethod
    def _load(
        cls,
        dps_object: dict[str, Any],
    ) -> Datapoints:
        instance = cls(
            id=dps_object["id"],
            external_id=dps_object.get("externalId"),
            instance_id=NodeId._load_if(dps_object.get("instanceId")),
            is_string=dps_object["isString"],
            is_step=dps_object["isStep"],
            type=dps_object["type"],
            unit=dps_object.get("unit"),
            unit_external_id=dps_object.get("unitExternalId"),
        )
        if len(dps_object["datapoints"]) == 0:
            instance.value = []
            instance.timestamp = []
            return instance

        data_lists = defaultdict(list)
        for dp in dps_object["datapoints"]:
            for attr, value in dp.items():
                data_lists[attr].append(value)
        if (timezone := dps_object.get("timezone")) is not None:
            instance.timezone = parse_str_timezone(timezone)
        if (status := data_lists.pop("status", None)) is not None:
            data_lists["status_code"] = [s["code"] for s in status]
            data_lists["status_symbol"] = [s["symbol"] for s in status]
        if min_dp := data_lists.get("minDatapoint"):
            data_lists["minDatapoint"] = list(map(_min_dp_class(min_dp[0])._load, min_dp))
        if max_dp := data_lists.get("maxDatapoint"):
            data_lists["maxDatapoint"] = list(map(_max_dp_class(max_dp[0])._load, max_dp))

        for key, data in data_lists.items():
            snake_key = to_snake_case(key)
            setattr(instance, snake_key, data)
        return instance

    def _extend(self, other_dps: Datapoints) -> None:
        raise NotImplementedError("Extending Datapoints is not supported.")

    def _get_non_empty_data_fields(self, get_empty_lists: bool = False) -> list[tuple[str, Any]]:
        non_empty_data_fields = []
        skip_attrs = {
            "id",
            "external_id",
            "instance_id",
            "is_string",
            "is_step",
            "type",
            "unit",
            "unit_external_id",
            "granularity",
            "status_code",
            "status_symbol",
            "timezone",
        }
        for attr, value in self.__dict__.copy().items():
            if attr not in skip_attrs and attr[0] != "_":
                if value is not None or attr == "timestamp":
                    if len(value) > 0 or get_empty_lists or attr == "timestamp":
                        non_empty_data_fields.append((attr, value))
        return non_empty_data_fields

    def __get_datapoint_objects(self) -> list[Datapoint]:
        if self.__datapoint_objects is not None:
            return self.__datapoint_objects
        fields = self._get_non_empty_data_fields()
        new_dps_objects = []
        for i in range(len(self)):
            dp_args: dict[str, Any] = {"timezone": self.timezone}
            for attr, value in fields:
                dp_args[to_camel_case(attr)] = value[i]
            if self.status_code is not None:
                dp_args.update(
                    statusCode=self.status_code[i],
                    statusSymbol=self.status_symbol[i],  # type: ignore [index]
                )
            new_dps_objects.append(Datapoint._load(dp_args))
        self.__datapoint_objects = new_dps_objects
        return self.__datapoint_objects

    def _slice(self, slice: slice) -> Datapoints:
        truncated_datapoints = Datapoints(
            id=self.id,
            external_id=self.external_id,
            instance_id=self.instance_id,
            is_string=self.is_string,
            is_step=self.is_step,
            type=self.type,
            unit=self.unit,
            unit_external_id=self.unit_external_id,
            granularity=self.granularity,
            timezone=self.timezone,
        )
        for attr, value in self._get_non_empty_data_fields():
            setattr(truncated_datapoints, attr, value[slice])
        if self.status_code is not None:
            truncated_datapoints.status_code = self.status_code[slice]
            truncated_datapoints.status_symbol = self.status_symbol[slice]  # type: ignore [index]
        return truncated_datapoints

    def _repr_html_(self) -> str:
        return notebook_display_with_fallback(self)


class SyntheticDatapoints(CogniteResource):
    def __init__(
        self,
        expression: str,
        timestamp: list[int],
        value: list[float | None],
        error: list[str | None],
        is_string: bool,
        timezone: datetime.timezone | ZoneInfo | None = None,
    ) -> None:
        self.expression = expression
        self.timestamp = timestamp
        self.value = value
        self.error = error
        self.is_string = is_string
        self.timezone = timezone

    def __len__(self) -> int:
        return len(self.timestamp)

    def __eq__(self, other: Any) -> bool:
        raise NotImplementedError(
            "Equality comparison is not implemented for SyntheticDatapoints because it most likely involves "
            "float comparisons."
        )

    @classmethod
    def _load(cls, dps_object: dict[str, Any]) -> Self:
        ts, value, errors = [], [], []
        for dp in dps_object["datapoints"]:
            ts.append(dp["timestamp"])
            # Each element is either a "value" or an "error", never both:
            value.append(dp.get("value"))
            errors.append(dp.get("error"))

        if (tz := dps_object.get("timezone")) is not None:
            tz = parse_str_timezone(tz)
        return cls(
            # Expression is not actually part of the API response for synthetic datapoints, so we add it in manually
            # to e.g. have a column name when converting to pandas
            expression=dps_object["expression"],
            timestamp=ts,
            value=value,
            error=errors,
            is_string=dps_object["isString"],
            timezone=tz,
        )

    def dump(self, camel_case: bool = True) -> dict[str, Any]:
        """Dump the synthetic datapoints into a json serializable Python data type.

        Args:
            camel_case (bool): Use camelCase for attribute names. Defaults to True.

        Returns:
            dict[str, Any]: A dictionary representing the instance.
        """
        dumped = {
            "expression": self.expression,
            "isString" if camel_case else "is_string": self.is_string,
            # Since SyntheticDatapoints is a pure read-class, we don't care that we don't exactly follow
            # the API response format (which has exactly one of "value" or "error" per datapoint):
            "datapoints": [
                {"timestamp": ts, "value": val, "error": err}
                for ts, val, err in zip(self.timestamp, self.value, self.error)
            ],
        }
        if self.timezone is not None:
            dumped["timezone"] = convert_timezone_to_str(self.timezone)
        return dumped

    def to_pandas(self, include_errors: bool = True) -> pandas.DataFrame:  # type: ignore[override]
        """Convert the synthetic datapoints into a pandas DataFrame.

        Note:
            The error column will only be included if ``include_errors=True`` AND there is at least
            one error exists.

        Args:
            include_errors (bool): Whether to include the error column. Defaults to True, but will be skipped if there are no errors.

        Returns:
            pandas.DataFrame: A DataFrame with timestamp as index and columns for the expression value and optionally error.
        """
        pd = local_import("pandas")

        tz = convert_tz_for_pandas(self.timezone)
        index = pd.to_datetime(self.timestamp, unit="ms", utc=True)
        if tz is not None:
            index = index.tz_convert(tz)

        data: dict[str, Any] = {self.expression: self.value}
        # Only include error column if requested AND there's at least one non-null error
        if include_errors and any(self.error):
            data["error"] = [e or "" for e in self.error]

        df = pd.DataFrame(data, index=index)
        df.index.name = "timestamp"
        return df


class SyntheticDatapointsList(CogniteResourceList[SyntheticDatapoints]):
    """A list of SyntheticDatapoints objects representing multiple expressions.

    Each SyntheticDatapoints in the list represents the result of evaluating one expression.
    """

    _RESOURCE = SyntheticDatapoints

    def get(self, *args: Any, **kwargs: Any) -> NoReturn:
        raise NotImplementedError

    def to_pandas(self, include_errors: bool = True) -> pandas.DataFrame:  # type: ignore[override]
        """Convert the list of synthetic datapoints into a single pandas DataFrame.

        Each expression becomes a column in the resulting DataFrame, with timestamps as the index.
        Error columns are only included for expressions that have at least one error.

        Args:
            include_errors (bool): Whether to include error columns. Defaults to True.

        Returns:
            pandas.DataFrame: A DataFrame with timestamp as index and columns for each expression and optionally errors.
        """
        pd = local_import("pandas")

        if not self.data:
            # Return empty DataFrame with proper structure
            return pd.DataFrame()

        # Convert each SyntheticDatapoints to a DataFrame and concat horizontally
        dfs = [item.to_pandas(include_errors=include_errors) for item in self.data]

        # Concatenate along columns (axis=1), aligning on timestamp index
        return pd.concat(dfs, axis=1)

    def dump(self, camel_case: bool = True) -> NoReturn:
        raise NotImplementedError


class DatapointsArrayList(CogniteResourceListWithClientRef[DatapointsArray]):
    _RESOURCE = DatapointsArray

    @cached_property
    def _id_to_item(self) -> dict[int, DatapointsArray | list[DatapointsArray]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "id")

    @cached_property
    def _external_id_to_item(self) -> dict[str, DatapointsArray | list[DatapointsArray]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "external_id")

    @cached_property
    def _instance_id_to_item(self) -> dict[InstanceId, DatapointsArray | list[DatapointsArray]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "instance_id")

    def get(  # type: ignore [override]
        self,
        id: int | None = None,
        external_id: str | None = None,
        instance_id: NodeId | tuple[str, str] | None = None,
    ) -> DatapointsArray | list[DatapointsArray] | None:
        """Get a specific DatapointsArray from this list by id or external_id.

        Note:
            For duplicated time series, returns a list of DatapointsArray.

        Args:
            id (int | None): The id of the item(s) to get.
            external_id (str | None): The external_id of the item(s) to get.
            instance_id (NodeId | tuple[str, str] | None): The instance_id of the item(s) to get.

        Returns:
            DatapointsArray | list[DatapointsArray] | None: The requested item(s)
        """
        # TODO: Question, can we type annotate without specifying the function?
        return super().get(id, external_id, instance_id)

    def __str__(self) -> str:
        return _json.dumps(self.dump(convert_timestamps=True), indent=4)

    def to_pandas(  # type: ignore [override]
        self,
        include_aggregate_name: bool = True,
        include_granularity_name: bool = False,
        include_unit: bool = True,
        include_status: bool = True,
    ) -> pandas.DataFrame:
        """Convert the DatapointsArrayList into a pandas DataFrame.

        Args:
            include_aggregate_name (bool): Include aggregate in the dataframe columns, if present (separate MultiIndex level)
            include_granularity_name (bool): Include granularity in the dataframe columns, if present (separate MultiIndex level)
            include_unit (bool): Include the unit_external_id in the dataframe columns, if present (separate MultiIndex level)
            include_status (bool): Include status code and status symbol as separate columns, if available. Also adds the status info
                as a separate level in the columns (MultiIndex).

        Returns:
            pandas.DataFrame: The datapoints as a pandas DataFrame.
        """
        return concat_dps_dataframe_list(
            self,
            include_aggregate_name=include_aggregate_name,
            include_granularity_name=include_granularity_name,
            include_status=include_status,
            include_unit=include_unit,
        )

    def dump(self, camel_case: bool = True, convert_timestamps: bool = False) -> list[dict[str, Any]]:
        """Dump the instance into a json serializable Python data type.

        Args:
            camel_case (bool): Use camelCase for attribute names. Defaults to True.
            convert_timestamps (bool): Convert timestamps to ISO 8601 formatted strings. Default: False (returns as integer, milliseconds since epoch)

        Returns:
            list[dict[str, Any]]: A list of dicts representing the instance.
        """
        return [dps.dump(camel_case, convert_timestamps) for dps in self]


class DatapointsList(CogniteResourceListWithClientRef[Datapoints]):
    _RESOURCE = Datapoints

    @cached_property
    def _id_to_item(self) -> dict[int, Datapoints | list[Datapoints]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "id")

    @cached_property
    def _external_id_to_item(self) -> dict[str, Datapoints | list[Datapoints]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "external_id")

    @cached_property
    def _instance_id_to_item(self) -> dict[InstanceId, Datapoints | list[Datapoints]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "instance_id")

    def get(  # type: ignore [override]
        self,
        id: int | None = None,
        external_id: str | None = None,
        instance_id: InstanceId | tuple[str, str] | None = None,
    ) -> Datapoints | list[Datapoints] | None:
        """Get a specific Datapoints from this list by id, external_id or instance_id.

        Note:
            For duplicated time series, returns a list of Datapoints.

        Args:
            id (int | None): The id of the item(s) to get.
            external_id (str | None): The external_id of the item(s) to get.
            instance_id (InstanceId | tuple[str, str] | None): The instance_id of the item(s) to get.

        Returns:
            Datapoints | list[Datapoints] | None: The requested item(s)
        """
        # TODO: Question, can we type annotate without specifying the function?
        return super().get(id, external_id, instance_id)

    def __str__(self) -> str:
        dumped = self.dump()
        for dps, item in zip(self, dumped):
            for dct in item["datapoints"]:
                dct["timestamp"] = convert_and_isoformat_timestamp(dct["timestamp"], dps.timezone)
        return _json.dumps(dumped, indent=4)

    def to_pandas(  # type: ignore [override]
        self,
        include_aggregate_name: bool = True,
        include_granularity_name: bool = False,
        include_unit: bool = True,
        include_status: bool = True,
    ) -> pandas.DataFrame:
        """Convert the datapoints list into a pandas DataFrame.

        Args:
            include_aggregate_name (bool): Include aggregate in the dataframe columns, if present (separate MultiIndex level)
            include_granularity_name (bool): Include granularity in the dataframe columns, if present (separate MultiIndex level)
            include_unit (bool): Include the unit_external_id in the dataframe columns, if present (separate MultiIndex level)
            include_status (bool): Include status code and status symbol as separate columns, if available. Also adds the status info
                as a separate level in the columns (MultiIndex).

        Returns:
            pandas.DataFrame: The datapoints list as a pandas DataFrame.
        """
        return concat_dps_dataframe_list(
            self,
            include_aggregate_name=include_aggregate_name,
            include_granularity_name=include_granularity_name,
            include_status=include_status,
            include_unit=include_unit,
        )


class LatestDatapoint(CogniteResource):
    """An object representing the latest datapoint for a time series.

    This class combines time series metadata with at most one datapoint, optimized for the
    ``retrieve_latest`` method response.

    Args:
        id (int): Id of the time series the datapoint belongs to
        timestamp (datetime.datetime | None): The data timestamp. None if no datapoint exists.
        value (str | float | None): The data value. Can be string or numeric, or None if no datapoint exists or value is missing.
        is_string (bool): Whether the time series contains numerical or string data.
        type (Literal['numeric', 'string', 'state']): The type of the time series.
        before (datetime.datetime | None): The timestamp used as the 'before' parameter in the query that retrieved this datapoint.
        is_step (bool | None): Whether the time series is stepwise or continuous.
        external_id (str | None): External id of the time series the datapoint belongs to
        instance_id (NodeId | None): The instance id of the time series the datapoint belongs to
        unit (str | None): The physical unit of the time series (free-text field).
        unit_external_id (str | None): The unit_external_id of the returned data points.
        status_code (int | None): The status code for the datapoint.
        status_symbol (str | None): The status symbol for the datapoint.
    """

    def __init__(
        self,
        id: int,
        timestamp: datetime.datetime | None,
        value: str | float | None,
        is_string: bool,
        type: Literal["numeric", "string", "state"],
        before: datetime.datetime | None,
        is_step: bool | None = None,
        external_id: str | None = None,
        instance_id: NodeId | None = None,
        unit: str | None = None,
        unit_external_id: str | None = None,
        status_code: int | None = None,
        status_symbol: str | None = None,
    ) -> None:
        self.id = id
        self.external_id = external_id
        self.instance_id = instance_id
        self.is_string = is_string
        self.is_step = is_step
        self.type = type
        self.before = before
        self.unit = unit
        self.unit_external_id = unit_external_id
        self.timestamp = timestamp
        self.value = value
        self.status_code = status_code
        self.status_symbol = status_symbol

    def __str__(self) -> str:
        dumped = self.dump(camel_case=False)
        if self.timestamp is not None:
            dumped["timestamp"] = self.timestamp.isoformat(sep=" ", timespec="milliseconds")
        return _json.dumps(dumped, indent=4)

    def __bool__(self) -> bool:
        """Returns True if the latest datapoint exists (has a timestamp)."""
        return self.timestamp is not None

    def __eq__(self, other: Any) -> bool:
        raise NotImplementedError(
            "Equality comparison is not implemented for LatestDatapoint because it most likely involves "
            "a float comparison."
        )

    @property
    def has_datapoint(self) -> bool:
        """Whether a datapoint exists for this time series."""
        return bool(self)

    def dump(self, camel_case: bool = True) -> dict[str, Any]:
        """Dump the latest datapoint into a json serializable Python data type.

        Args:
            camel_case (bool): Use camelCase for attribute names. Defaults to True.

        Returns:
            dict[str, Any]: A dictionary representing the instance.
        """
        dumped: dict[str, Any] = {
            "id": self.id,
            "is_string": self.is_string,
            "is_step": self.is_step,
            "type": self.type,
            "before": datetime_to_ms(self.before) if self.before is not None else None,
            "unit": self.unit,
            "unit_external_id": self.unit_external_id,
        }
        if self.instance_id is not None:
            dumped["instance_id"] = self.instance_id.dump(camel_case=camel_case, include_instance_type=False)
        if self.external_id is not None:
            dumped["external_id"] = self.external_id

        if self.timestamp is None:
            dumped["datapoints"] = []
        else:
            dp: dict[str, Any] = {"timestamp": datetime_to_ms(self.timestamp), "value": self.value}
            if self.status_code is not None:
                dp["status"] = {"code": self.status_code, "symbol": self.status_symbol}
            dumped["datapoints"] = [dp]

        if camel_case:
            dumped = convert_all_keys_to_camel_case(dumped)
        return dumped

    def to_pandas(self, camel_case: bool = False) -> pandas.DataFrame:  # type: ignore[override]
        """Convert the latest datapoint into a pandas DataFrame.

        Args:
            camel_case (bool): Convert column names to camel case. Defaults to False.

        Returns:
            pandas.DataFrame: The DataFrame representation of the latest datapoint.
        """
        pd = local_import("pandas")
        # Some of these may be None (and dump will remove them), but we want them always present:
        dumped = {"value": self.value, "timestamp": self.timestamp, "before": self.before}
        for k, v in self.dump(camel_case=camel_case).items():
            if k not in dumped:
                dumped[k] = v
        return pd.Series(dumped).to_frame(name="value")

    @classmethod
    def _load(cls, resource: dict[str, Any]) -> Self:
        status_code = None
        status_symbol = None

        match resource["datapoints"]:
            case []:
                timestamp: datetime.datetime | None = None
                value: str | float | None = None
            case [dict() as dp]:
                timestamp = ms_to_datetime(dp["timestamp"])
                value = dp.get("value")
                if status := dp.get("status"):
                    status_code = status.get("code")
                    status_symbol = status.get("symbol")
            case _ as dps:
                raise ValueError(f"Expected at most one datapoint for LatestDatapoint, got: {dps!r}")

        # Before is not part of the API payload, and only a convenience property we add:
        if (before := resource.get("before")) is not None:
            before = ms_to_datetime(before)
        return cls(
            id=resource["id"],
            timestamp=timestamp,
            value=value,
            external_id=resource.get("externalId"),
            instance_id=NodeId._load_if(resource.get("instanceId")),
            type=resource["type"],
            is_string=resource["isString"],
            is_step=resource["isStep"],
            unit=resource.get("unit"),
            unit_external_id=resource.get("unitExternalId"),
            status_code=status_code,
            status_symbol=status_symbol,
            before=before,
        )


class LatestDatapointList(CogniteResourceListWithClientRef[LatestDatapoint], IdTransformerMixin):
    """A list of LatestDatapoint objects.

    This list is optimized for the ``retrieve_latest`` method, providing a ``to_pandas()``
    method that creates a DataFrame with time series identifiers as index and timestamp/value
    as columns, avoiding sparse DataFrames when timestamps differ across time series.
    """

    _RESOURCE = LatestDatapoint

    @cached_property
    def _id_to_item(self) -> dict[int, LatestDatapoint | list[LatestDatapoint]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "id")

    @cached_property
    def _external_id_to_item(self) -> dict[str, LatestDatapoint | list[LatestDatapoint]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "external_id")

    @cached_property
    def _instance_id_to_item(self) -> dict[InstanceId, LatestDatapoint | list[LatestDatapoint]]:  # type: ignore [override]
        return build_id_mapping_with_duplicates(self.data, "instance_id")

    def get(  # type: ignore [override]
        self,
        id: int | None = None,
        external_id: str | None = None,
        instance_id: InstanceId | tuple[str, str] | None = None,
    ) -> LatestDatapoint | list[LatestDatapoint] | None:
        """Get a specific LatestDatapoint from this list by id, external_id or instance_id.

        Note:
            For duplicated time series, returns a list of LatestDatapoint.

        Args:
            id (int | None): The id of the item(s) to get.
            external_id (str | None): The external_id of the item(s) to get.
            instance_id (InstanceId | tuple[str, str] | None): The instance_id of the item(s) to get.

        Returns:
            LatestDatapoint | list[LatestDatapoint] | None: The requested item(s)
        """
        return super().get(id, external_id, instance_id)

    def __str__(self) -> str:
        dumped = self.dump(camel_case=False)
        for item in dumped:
            if item.get("timestamp") is not None:
                item["timestamp"] = convert_and_isoformat_timestamp(item["timestamp"], None)
        return _json.dumps(dumped, indent=4)

    def to_pandas(  # type: ignore [override]
        self,
        include_status: bool = True,
    ) -> pandas.DataFrame:
        """Convert the latest datapoints list into a pandas DataFrame.

        Creates a DataFrame with time series identifiers (preferring external_id, then id)
        as the index, and timestamp/value as columns. This format avoids sparse DataFrames
        when timestamps differ across time series.

        Args:
            include_status (bool): Include status_code and status_symbol columns if available. Default: True

        Returns:
            pandas.DataFrame: A DataFrame with columns 'timestamp', 'value' (and optionally
                'status_code', 'status_symbol') with time series identifiers as the index.

        Examples:

            Get the latest datapoint for multiple time series and convert to DataFrame:

                >>> from cognite.client import CogniteClient
                >>> client = CogniteClient()
                >>> latest = client.time_series.data.retrieve_latest(external_id=["ts1", "ts2", "ts3"])
                >>> df = latest.to_pandas()
        """
        pd = local_import("pandas")

        rows = []
        index_values: list[int | str | InstanceId] = []

        for item in self:
            # Prefer instance_id, then external_id, then id for the index
            if item.instance_id is not None:
                index_values.append(item.instance_id)
            elif item.external_id is not None:
                index_values.append(item.external_id)
            else:
                index_values.append(item.id)

            row: dict[str, Any] = {
                "value": item.value,
                "timestamp": item.timestamp if item.timestamp is not None else pd.NaT,
                "before": item.before,
            }
            if item.unit_external_id is not None:
                row["unit_external_id"] = item.unit_external_id
            if include_status:
                row["status_code"] = item.status_code
                row["status_symbol"] = item.status_symbol
            rows.append(row)

        df = pd.DataFrame(rows, index=index_values)
        df.index.name = "identifier"

        # Drop status columns if they are all null
        if include_status:
            if df["status_code"].isna().all():  # symbol column will also be null
                df = df.drop(columns=["status_code", "status_symbol"])

        return df
