"""A handler for remote datasets.

DAP handlers convert from different data formats (NetCDF, eg) to the internal
pydap model. The pydap client is just a handler that converts from a remote
dataset to the internal model.

"""

import copy
import gzip
import io

# handlers should be set by the application
# http://docs.python.org/2/howto/logging.html#configuring-logging-for-a-library
import logging
import pprint
import re
import sys
import tempfile
import warnings
from io import BufferedReader, BytesIO
from itertools import chain
from pathlib import Path
from xml.etree import ElementTree as ET

import numpy
import requests
import requests_cache
from requests.utils import urlparse, urlunparse
from webob.response import Response as webob_Response

from pydap.handlers.lib import BaseHandler, ConstraintExpression, IterData
from pydap.lib import (
    DAP2_ARRAY_LENGTH_NUMPY_TYPE,
    DEFAULT_TIMEOUT,
    START_OF_SEQUENCE,
    BytesReader,
    StreamReader,
    _quote,
    combine_slices,
    encode,
    fix_slice,
    hyperslab,
    old_BytesReader,
    unquote,
    walk,
)
from pydap.model import (
    BaseType,
    DapDecodedArray,
    DatasetType,
    GridType,
    GroupType,
    SequenceType,
    StructureType,
)
from pydap.net import GET
from pydap.parsers import parse_ce
from pydap.parsers.das import add_attributes, parse_das
from pydap.parsers.dds import dds_to_dataset
from pydap.parsers.dmr import DMRPPParser, dmr_to_dataset
from pydap.responses.dods import DAP2_response_dtypemap

try:
    import httpx
except ImportError:
    httpx = None

try:
    from netCDF4 import Dataset

    HAVE_NETCDF4 = True
except Exception:
    Dataset = None
    HAVE_NETCDF4 = False

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

BLOCKSIZE = 512


class DAPHandler(BaseHandler):
    """Build a dataset from a DAP base URL."""

    def __init__(
        self,
        url,
        application=None,
        session=None,
        output_grid=True,
        flat=True,
        timeout=DEFAULT_TIMEOUT,
        verify=True,
        checksums=True,
        user_charset="ascii",
        protocol=None,
        get_kwargs=None,
    ):

        self.application = application
        self.session = session
        self.output_grid = output_grid
        self.flat = flat
        self.timeout = timeout
        self.verify = verify
        self.checksums = checksums
        self.user_charset = user_charset
        self.get_kwargs = get_kwargs or {}

        if url.endswith(".dmrpp") and not url.startswith("dap4"):
            self.protocol = "dap4"
            if url.startswith("http://") or url.startswith("https://"):
                r = session.get(url, stream=True)
                dmrpp = io.BytesIO(r.content)
                dmrpp_instance = DMRPPParser(root=ET.parse(dmrpp).getroot())
            else:
                dmrpp = open(url).read()
                dmrpp_instance = DMRPPParser(root=ET.fromstring(dmrpp))
            self.projection = []
            self.dataset = dmrpp_instance.to_dataset()
            self.base_url = dmrpp_instance.opendap_url

            if not self.base_url:
                # some dmrpps do not have the opendap url embedded in their
                if url.endswith(".dap.dmrpp"):
                    self.base_url = url.removesuffix(".dap.dmrpp")
                else:
                    self.base_url = url.removesuffix(".dmrpp")
        else:
            self.url = url
            # urlparse returns an additional var compared to
            # urlsplit: `param`. Will toss it.
            scheme, netloc, path, _, query, fragment = urlparse(self.url)
            self.scheme = scheme
            self.netloc = netloc
            self.path = path
            self.query = query
            self.fragment = fragment

            if protocol:
                if protocol not in ["dap2", "dap4"]:
                    raise TypeError("protocol must be one of `dap2` or `dap4")
                self.protocol = protocol
                if self.scheme == protocol:
                    # the other alternative occurs during testing
                    # the server - only when protocol and scheme match,
                    # should pydap change the scheme provided by user
                    self.scheme = "https"
            else:
                self.protocol = self.determine_protocol()

            self.projection, self.selection = parse_ce(self.query, self.protocol)
            arg = (
                self.scheme,
                self.netloc,
                self.path,
                "",
                "&".join(self.selection),
                self.fragment,
            )
            self.base_url = urlunparse(arg)
            self.make_dataset()
        self.add_proxies()

    def determine_protocol(self):
        # determines the opendap protocol from url scheme, query, or extension
        if self.application:
            # server - only dap2 for now.
            return "dap2"
        if self.scheme not in ["http", "https"]:
            if self.scheme in ["dap4", "dap2"]:
                protocol = self.scheme
                self.scheme = "https"
                return protocol
            else:
                raise TypeError(
                    "Invalid URL scheme - acceptable options are"
                    "`dap2`, `dap4`. `https` and `http`",
                )
        if self.query[:4] == "dap4":
            return "dap4"
        else:
            warnings.warn(
                "PyDAP was unable to determine the DAP protocol defaulting "
                "to DAP2. DAP2 is consider legacy and may result in slower "
                "responses. \nConsider replacing `http` in your `url` with "
                "either `dap2` or `dap4` to specify the DAP protocol (e.g. "
                "`dap2://<data_url>` or `dap4://<data_url>`).  For more \n"
                "information, go to https://www.opendap.org/faq-page."
            )
            return "dap2"

    def make_dataset(
        self,
    ):
        if self.protocol == "dap4":
            self.dataset_from_dap4()
        else:
            self.dataset_from_dap2()
            self.attach_das()

    def dataset_from_dap4(self):
        if not self.path.endswith(".dmr"):
            path = self.path + ".dmr"
        else:
            path = self.path
        dmr_url = urlunparse(
            (
                self.scheme,
                self.netloc,
                path,
                "",
                _quote(self.query),
                self.fragment,
            )
        )
        r = GET(
            dmr_url,
            self.application,
            self.session,
            timeout=self.timeout,
            verify=self.verify,
            get_kwargs=self.get_kwargs,
        )
        dmr = safe_charset_text(r, self.user_charset)
        self.dataset = dmr_to_dataset(dmr, self.flat)

    def dataset_from_dap2(self):
        # escape for certain characters
        dds_url = urlunparse(
            (
                self.scheme,
                self.netloc,
                self.path + ".dds",
                "",
                _quote(self.query),
                self.fragment,
            )
        )
        r = GET(
            dds_url,
            self.application,
            self.session,
            timeout=self.timeout,
            verify=self.verify,
            get_kwargs=self.get_kwargs,
        )

        dds = safe_charset_text(r, self.user_charset)
        self.dataset = dds_to_dataset(dds)

    def attach_das(self):
        # Also pull the DAS and add additional attributes
        das_url = urlunparse(
            (
                self.scheme,
                self.netloc,
                self.path + ".das",
                "",
                self.query,
                self.fragment,
            )
        )
        r = GET(
            das_url,
            self.application,
            self.session,
            timeout=self.timeout,
            verify=self.verify,
            get_kwargs=self.get_kwargs,
        )
        das = safe_charset_text(r, self.user_charset)
        add_attributes(self.dataset, parse_das(das))

    def add_proxies(self):
        if self.protocol == "dap4":
            self.add_dap4_proxies()
        else:
            self.add_dap2_proxies()

    def add_dap4_proxies(self):
        # remove any projection from the base_url, leaving selections
        for var in walk(self.dataset, BaseType):
            if hasattr(var, "parent") and isinstance(var.parent, StructureType):
                if var.parent.type == "Group":
                    var_name = var.parent.id + "/" + var.name
                elif isinstance(var.parent, DatasetType):
                    var_name = var.name
                elif var.parent.type == "Structure" or isinstance(
                    var.parent, SequenceType
                ):
                    var_name = var.parent.id + "." + var.name
            else:
                if var.path is not None:
                    var_name = (
                        var.path + "/" + var.name
                        if var.path[-1] != "/"
                        else var.path + var.name
                    )
                else:
                    var_name = var.name
            var.data = BaseProxyDap4(
                self.base_url,
                var_name,
                var.dtype,
                var.shape,
                application=self.application,
                session=self.session,
                timeout=self.timeout,
                verify=self.verify,
                checksums=self.checksums,
                get_kwargs={**self.get_kwargs, "stream": True},
            )

        for var in walk(self.dataset, SequenceType):
            warnings.warn(
                f"The remote file contains Sequence `{var.name}`"
                ". Sequences in DAP4 are not fully supported and their"
                " use may lead to unexpected results."
            )

        self.dataset.assign_dataset_recursive(self.dataset)
        # self.dataset.enable_batch_mode()

        # apply projections to BaseType only
        # CE for sequences and structs
        # are not ready (see https://github.com/pydap/pydap/issues/314)
        for var in self.projection:
            target = self.dataset
            while var:
                token, index = var.pop(0)
                target = target[token]
                if isinstance(target, BaseType):
                    target.data.slice = fix_slice(index, target.shape, projection=True)

    def add_dap2_proxies(self):
        # now add data proxies
        for var in walk(self.dataset, BaseType):
            var.data = BaseProxyDap2(
                self.base_url,
                var.id,
                var.dtype,
                var.shape,
                application=self.application,
                session=self.session,
                timeout=self.timeout,
                verify=self.verify,
                get_kwargs={**self.get_kwargs, "stream": True},
            )
        for var in walk(self.dataset, SequenceType):
            template = copy.copy(var)
            var.data = SequenceProxy(
                self.base_url,
                template,
                selection=self.selection,
                application=self.application,
                session=self.session,
                timeout=self.timeout,
                verify=self.verify,
                get_kwargs={**self.get_kwargs, "stream": True},
            )

        # apply projections
        for var in self.projection:
            target = self.dataset
            while var:
                token, index = var.pop(0)
                target = target[token]
                if isinstance(target, BaseType):
                    target.data.slice = fix_slice(index, target.shape, projection=True)
                elif isinstance(target, GridType):
                    index = fix_slice(index, target.array.shape, projection=True)
                    target.array.data.slice = index
                    for s, child in zip(index, target.maps):
                        target[child].data.slice = (s,)
                elif isinstance(target, SequenceType):
                    target.data.slice = index

        # retrieve only main variable for grid types:
        for var in walk(self.dataset, GridType):
            var.set_output_grid(self.output_grid)


def get_charset(r, user_charset):
    charset = r.charset
    if not charset:
        charset = user_charset
    return charset


def safe_charset_text(r, user_charset):
    if isinstance(r, webob_Response):
        if r.content_encoding == "gzip":
            return (
                gzip.GzipFile(fileobj=BytesIO(r.body))
                .read()
                .decode(get_charset(r, user_charset))
            )
        else:
            r.charset = get_charset(r, user_charset)
    return r.text


def safe_dds_and_data(r, user_charset):
    """
    Takes the raw response of a dap2 request and splits it into the dds and data.
    If the response is gzipped, it is decompressed first.
    """
    dds, data = None, None  # initialize
    if isinstance(r, webob_Response):
        if r.content_encoding == "gzip":
            raw = gzip.GzipFile(fileobj=BytesIO(r.body)).read()
        else:
            raw = r.body
        _dds, data = raw.split(b"\nData:\n", 1)
        dds = _dds.decode(get_charset(r, user_charset))
    elif isinstance(r, requests.Response):
        raw = r.content
        _dds, data = raw.split(b"\nData:\n", 1)
        dds = _dds.decode(user_charset)
    return dds, data


class BaseProxyDap2(object):
    """A proxy for remote base types.

    This class behaves like a Numpy array, proxying the data from a base type
    on a remote dataset.
    """

    def __init__(
        self,
        baseurl,
        id,
        dtype,
        shape,
        slice_=None,
        application=None,
        session=None,
        timeout=DEFAULT_TIMEOUT,
        verify=True,
        user_charset="ascii",
        get_kwargs=None,
    ):
        self.baseurl = baseurl
        self.id = id
        self.dtype = dtype
        self.shape = shape
        self.slice = slice_ or tuple(slice(None) for s in self.shape)
        self.application = application
        self.session = session
        self.timeout = timeout
        self.verify = verify
        self.user_charset = user_charset
        self.get_kwargs = get_kwargs or {}

    def __repr__(self):
        return "BaseProxy(%s)" % ", ".join(
            map(repr, [self.baseurl, self.id, self.dtype, self.shape, self.slice])
        )

    def __getitem__(self, index):
        # build download url

        index = combine_slices(self.slice, fix_slice(index, self.shape))
        scheme, netloc, path, params, query, fragment = urlparse(self.baseurl)
        url = urlunparse(
            (
                scheme,
                netloc,
                path + ".dods",
                "",
                self.id + hyperslab(index) + "&" + _quote(query),
                fragment,
            )
        ).rstrip("&")

        # download and unpack data
        logger.info("Fetching URL: %s" % url)
        r = GET(
            url,
            self.application,
            self.session,
            timeout=self.timeout,
            verify=self.verify,
            get_kwargs=self.get_kwargs,
        )

        dds, data = safe_dds_and_data(r, self.user_charset)

        # Parse received dataset:
        dataset = dds_to_dataset(dds)
        dataset.data = unpack_dap2_data(old_BytesReader(data), dataset)
        return dataset[self.id].data

    def __len__(self):
        return self.shape[0]

    def __iter__(self):
        return iter(self[:])

    # Comparisons return a boolean array
    def __eq__(self, other):
        return self[:] == other

    def __ne__(self, other):
        return self[:] != other

    def __ge__(self, other):
        return self[:] >= other

    def __le__(self, other):
        return self[:] <= other

    def __gt__(self, other):
        return self[:] > other

    def __lt__(self, other):
        return self[:] < other


class BaseProxyDap4(BaseProxyDap2):

    def __init__(
        self,
        baseurl,
        id,
        dtype,
        shape,
        slice_=None,
        application=None,
        session=None,
        timeout=DEFAULT_TIMEOUT,
        verify=True,
        checksums=False,
        user_charset="ascii",
        get_kwargs=None,
    ):
        self.baseurl = baseurl
        self.id = id
        self.dtype = dtype
        self.shape = shape
        self.slice = slice_ or tuple(slice(None) for s in self.shape)
        self.application = application
        self.session = session
        self.timeout = timeout
        self.verify = verify
        self.checksums = checksums
        self.user_charset = user_charset
        self.get_kwargs = get_kwargs or {}
        self.ce = None

    def __repr__(self):
        return "Dap4BaseProxy(%s)" % ", ".join(
            map(
                repr,
                [
                    self.baseurl,
                    self.id,
                    self.dtype,
                    self.shape,
                    self.ce if self.ce is not None else self.slice,
                ],
            )
        )

    def __getitem__(self, index, build_only=False):
        # build download url
        index = combine_slices(self.slice, fix_slice(index, self.shape))

        scheme, netloc, path, _, query, fragment = urlparse(self.baseurl)
        if self.id.startswith("/"):
            self.ce = "dap4.ce=" + self.id + hyperslab(index)
        else:
            self.ce = "dap4.ce=/" + self.id + hyperslab(index)

        if build_only:
            # In batch mode: just store CE, don't fetch
            return self
        url = urlunparse((scheme, netloc, path + ".dap", "", self.ce, fragment)).rstrip(
            "&"
        )
        if self.checksums:
            url += "&dap4.checksum=true"
        else:
            url += "&dap4.checksum=false"
        # download and unpack data
        logger.info("Fetching URL: %s" % url)

        cache_kwargs = {}
        if isinstance(self.session, requests_cache.CachedSession):
            _vars = getattr(self.session.settings.key_fn, "_collapse_vars", None)
            concat_dim = getattr(self.session.settings.key_fn, "_concat_dim", None)
            _vars = [] if _vars is None else list(_vars)
            _vars += [] if concat_dim is None else concat_dim
            if self.id not in _vars and "debug" not in self.session.cache.cache_name:
                cache_kwargs = {"skip": True}
        r = GET(
            url,
            self.application,
            self.session,
            timeout=self.timeout,
            verify=self.verify,
            get_kwargs=self.get_kwargs,
            cache_kwargs=cache_kwargs,
        )

        dataset = UNPACKDAP4DATA(r, self.checksums, self.user_charset).dataset
        self._data = dataset[self.id]._data
        if self.checksums:
            self.checksums = dataset[self.id].attributes["_DAP4_Checksum_CRC32"]

        return self._data


class SequenceProxy(object):
    """A proxy for remote sequences.

    This class behaves like a Numpy structured array, proxying the data from a
    sequence on a remote dataset. The data is streamed from the dataset,
    meaning it can be treated one record at a time before the whole data is
    downloaded.

    """

    shape = ()

    def __init__(
        self,
        baseurl,
        template,
        selection=None,
        slice_=None,
        application=None,
        session=None,
        timeout=DEFAULT_TIMEOUT,
        verify=True,
        get_kwargs=None,
    ):
        self.baseurl = baseurl
        self.template = template
        self.selection = selection or []
        self.slice = slice_ or (slice(None),)
        self.application = application
        self.session = session
        self.timeout = timeout
        self.verify = verify

        # this variable is true when only a subset of the children are selected
        self.sub_children = False
        self.get_kwargs = get_kwargs or {}

    @property
    def dtype(self):
        return self.template.dtype

    def __repr__(self):
        return "SequenceProxy(%s)" % ", ".join(
            map(repr, [self.baseurl, self.template, self.selection, self.slice])
        )

    def __copy__(self):
        """Return a lightweight copy of the object."""
        return self.__class__(
            self.baseurl,
            self.template,
            self.selection[:],
            self.slice[:],
            self.application,
        )

    def __getitem__(self, key):
        """Return a new object representing a subset of the data."""
        out = copy.copy(self)

        # return the data for a children
        if isinstance(key, str):
            out.template = out.template[key]

        # return a new object with requested columns
        elif isinstance(key, list):
            out.sub_children = True
            out.template._visible_keys = key

        # return a copy with the added constraints
        elif isinstance(key, ConstraintExpression):
            out.selection.extend(str(key).split("&"))

        # slice data
        else:
            if isinstance(key, int):
                key = slice(key, key + 1)
            out.slice = combine_slices(self.slice, (key,))

        return out

    @property
    def url(self):
        """Return url from where data is fetched."""
        scheme, netloc, path, params, query, fragment = urlparse(self.baseurl)
        url = urlunparse(
            (
                scheme,
                netloc,
                path + ".dods",
                "",
                self.id + hyperslab(self.slice) + "&" + "&".join(self.selection),
                fragment,
            )
        ).rstrip("&")

        return url

    @property
    def id(self):
        """Return the id of this sequence."""
        if self.sub_children:
            id_ = ",".join(child.id for child in self.template.children())
        else:
            id_ = self.template.id
        return id_

    def __iter__(self):
        # download and unpack data
        r = GET(
            self.url,
            self.application,
            self.session,
            timeout=self.timeout,
            verify=self.verify,
            get_kwargs=self.get_kwargs,
        )

        if isinstance(r, webob_Response):
            i = r.app_iter
            if not hasattr(i, "__next__"):
                i = iter(i)
        elif isinstance(r, requests.Response):
            i = r.iter_content()

        # Fast forward past the DDS header
        # the pattern could span chunk boundaries though so make sure to check
        pattern = b"Data:\n"
        last_chunk = find_pattern_in_string_iter(pattern, i)

        if last_chunk is None:
            raise ValueError(
                "Could not find data segment in response from {}".format(self.url)
            )

        # Then construct a stream consisting of everything from
        # 'Data:\n' to the end of the chunk + the rest of the stream
        def stream_start():
            yield last_chunk

        stream = StreamReader(chain(stream_start(), i))

        return unpack_sequence(stream, self.template)

    def __eq__(self, other):
        return ConstraintExpression("%s=%s" % (self.id, encode(other)))

    def __ne__(self, other):
        return ConstraintExpression("%s!=%s" % (self.id, encode(other)))

    def __ge__(self, other):
        return ConstraintExpression("%s>=%s" % (self.id, encode(other)))

    def __le__(self, other):
        return ConstraintExpression("%s<=%s" % (self.id, encode(other)))

    def __gt__(self, other):
        return ConstraintExpression("%s>%s" % (self.id, encode(other)))

    def __lt__(self, other):
        return ConstraintExpression("%s<%s" % (self.id, encode(other)))


def unpack_sequence(stream, template):
    """Unpack data from a sequence, yielding records."""
    # is this a sequence or a base type?
    sequence = isinstance(template, SequenceType)

    # if there are no children, we use the template as the only column
    cols = list(template.children()) or [template]

    # if there are no strings and no nested sequences we can unpack record by
    # record easily
    simple = all(isinstance(c, BaseType) and c.dtype.char not in "SU" for c in cols)

    if simple:
        dtype = numpy.dtype([("", c.dtype, c.shape) for c in cols])
        marker = stream.read(4)
        while marker == START_OF_SEQUENCE:
            rec = numpy.frombuffer(stream.read(dtype.itemsize), dtype=dtype)[0]
            if not sequence:
                rec = rec[0]
            yield rec
            marker = stream.read(4)
    else:
        marker = stream.read(4)
        while marker == START_OF_SEQUENCE:
            rec = unpack_children(stream, template)
            if not sequence:
                rec = rec[0]
            else:
                rec = tuple(rec)
            yield rec
            marker = stream.read(4)


def unpack_children(stream, template):
    """Unpack children from a structure, returning their data."""
    cols = list(template.children()) or [template]

    out = []
    for col in cols:
        # sequences and other structures
        if isinstance(col, SequenceType):
            out.append(IterData(list(unpack_sequence(stream, col)), col))
        elif isinstance(col, StructureType):
            out.append(tuple(unpack_children(stream, col)))

        # unpack arrays
        else:
            out.extend(convert_stream_to_list(stream, col.dtype, col.shape, col.id))
    return out


def convert_stream_to_list(stream, parser_dtype, shape, id):
    out = []
    response_dtype = DAP2_response_dtypemap(parser_dtype)
    if shape:
        n = numpy.frombuffer(stream.read(4), DAP2_ARRAY_LENGTH_NUMPY_TYPE)[0]
        count = response_dtype.itemsize * n
        if response_dtype.char in "S":
            # Consider on 'S' and not 'SU' because
            # response_dtype.char should never be
            data = []
            for _ in range(n):
                k = numpy.frombuffer(stream.read(4), DAP2_ARRAY_LENGTH_NUMPY_TYPE)[0]
                data.append(stream.read(k))
                stream.read(-k % 4)
            out.append(
                numpy.array([str(x.decode("ascii")) for x in data], "S").reshape(shape)
            )
        else:
            stream.read(4)  # read additional length
            try:
                out.append(
                    numpy.frombuffer(stream.read(count), response_dtype)
                    .astype(parser_dtype)
                    .reshape(shape)
                )
            except ValueError as e:
                if str(e) == "total size of new array must be unchanged":
                    # server-side failure.
                    # it is expected that the user should be mindful of this:
                    raise RuntimeError(
                        (
                            "variable {0} could not be properly "
                            "retrieved. To avoid this "
                            "error consider using open_url(..., "
                            "output_grid=False)."
                        ).format(id)
                    )
                else:
                    raise
            if response_dtype.char == "B":
                # Unsigned Byte type is packed to multiples of 4 bytes:
                stream.read(-n % 4)

    # special types: strings and bytes
    elif response_dtype.char in "S":
        # Consider on 'S' and not 'SU' because
        # response_dtype.char should never be
        # 'U'
        k = numpy.frombuffer(stream.read(4), DAP2_ARRAY_LENGTH_NUMPY_TYPE)[0]
        out.append(str(stream.read(k).decode("ascii")))
        stream.read(-k % 4)
    # usual data
    else:
        out.append(
            numpy.frombuffer(
                stream.read(response_dtype.itemsize), response_dtype
            ).astype(parser_dtype)[0]
        )
        if response_dtype.char == "B":
            # Unsigned Byte type is packed to multiples of 4 bytes:
            stream.read(3)
    return out


def unpack_dap2_data(xdr_stream, dataset):
    """Unpack a string of encoded data, returning data as lists."""
    return unpack_children(xdr_stream, dataset)


def find_pattern_in_string_iter(pattern, i):
    last_chunk = b""
    length = len(pattern)
    for this_chunk in i:
        last_chunk += this_chunk
        m = re.search(pattern, last_chunk)
        if m:
            return last_chunk[m.end() :]
        last_chunk = last_chunk[-length:]


def dump():  # pragma: no cover
    """Unpack dods response into lists.

    Return pretty-printed data.

    """
    dods = sys.stdin.read()
    dds, xdrdata = dods.split(b"\nData:\n", 1)
    dataset = dds_to_dataset(dds)
    xdr_stream = io.BytesIO(xdrdata)
    data = unpack_dap2_data(xdr_stream, dataset)
    pprint.pprint(data)


def decode_chunktype(chunk_type):
    """
    Takes the chunk_type from chunk header embeded in the dap response,
    and returns a tuple of
        last_chunk: bool
        error: bool
        endian
    """
    encoding = "{0:03b}".format(chunk_type)
    if sys.byteorder == "little":
        # If our machine's byteorder is little,
        # we need to swap since the chunk_type is always big endian
        encoding = encoding[::-1]
    last_chunk = bool(int(encoding[0]))
    error = bool(int(encoding[1]))
    endian = {"0": ">", "1": "<"}[encoding[2]]
    return last_chunk, error, endian


def get_count(variable):
    count = int(numpy.array(variable.shape).prod())
    item_size = numpy.dtype(variable.dtype).itemsize
    count = count * item_size
    return count


def decode_variable(buffer, start, variable, endian):
    dtype = variable.dtype
    if dtype.kind == "S":
        DATA = []
        stop = start
        for i in range(int(numpy.prod(variable.shape))):
            string, stop = decode_utf8_string_array(buffer, start=stop)
            DATA.append(numpy.array(string).astype(dtype.kind))
        data = numpy.array(DATA).reshape(variable.shape)
        return data, stop
    else:
        stop = get_count(variable) + start
        dtype = dtype.newbyteorder(endian)
        data = numpy.frombuffer(buffer[start:stop], dtype=dtype)
        data = data.reshape(variable.shape)
        return DapDecodedArray(data), stop


def decode_utf8_string_array(buffer, start=0):
    offset = start
    strings = []

    # 1. Read 8-byte little-endian length
    length_bytes = buffer[offset : offset + 8]
    strlen = int.from_bytes(length_bytes, byteorder="little")
    offset += 8

    # 2. Read the UTF-8 string
    str_bytes = buffer[offset : offset + strlen]
    offset += strlen

    # 3. Decode the bytes to Python string (UTF-8)
    decoded_str = str_bytes.decode("utf-8")
    strings.append(decoded_str)

    return strings, offset


def stream2bytearray(data):
    """
    Computes the buffer size of the (binary) data form dap response.
    data is sent in chunks, with encoding in between. The encoding is
    sent in packs of 4 bytes, which tells info about endianness, chunk type
    and chunk size, and whether it is last chunk or not. Data inbetween chunks
    is numeric (array) data in binary form that needs to be turn into native
    numpy array data of size =  len(buffer). That last bit is done outside the
    scope of this function.
    """

    # Precompute chunk positions
    chunk_positions = []
    offset = data.data.tell()  # current position in the stream
    last = False
    while not last:
        # Read the chunk header
        chunk_header = numpy.frombuffer(data.slice(offset=offset, n=4), dtype=">u4")[0]
        chunk_size = chunk_header & 0x00FFFFFF
        chunk_type = (chunk_header >> 24) & 0xFF
        last, _, _ = decode_chunktype(chunk_type)
        chunk_positions.append((offset + 4, chunk_size))
        offset += 4 + chunk_size
        if last:
            break
    # Process chunks serially (used to be parallelized- no penalty when serialized).
    results = []
    for offset, length in chunk_positions:
        data.data.seek(offset)
        results.append(data.data.read(length))

    buffer = bytearray()
    # Combine results
    for chunk_data in results:
        buffer.extend(chunk_data)
    return buffer


def get_endianness(chunk_header):
    chunk_header = numpy.frombuffer(chunk_header, dtype=">u4")[0]
    chunk_type = (chunk_header >> 24) & 0xFF
    _, _, endian = decode_chunktype(chunk_type)
    return endian


class UNPACKDAP4DATA(object):
    """
    Unpacks DAP4 response, remote or local, which is split into chunks. The
    first chunk contains the DMR response, and the endianness is defined in the first
    4 bytes before the DMR. Once the endianness is set, it remains unchanged. This
    makes the assumption that the all variables within dataset have the same endianness.

    Parameters:
    -----------
        r: contains the dap response.
            May be a Webob.response.Response or requests.response created from
            pydap.net.GET if the dataset is remote (from a url), or a
            `io.BufferedReader` if the data is local within a filesystem.
            See `pydap.net.get.open_dap_file`
    """

    def __init__(
        self,
        r,
        checksums=True,
        user_charset="ascii",
        output_path: str | None = None,
        dmrVersion: str | None = None,
    ):
        self.user_charset = user_charset
        self.checksums = checksums
        self.r = r
        self.output_path = Path(output_path) if output_path else output_path
        self.nc = None
        self._dims_cache: dict[tuple[str, tuple[int, ...]], list[str]] = {}
        self.dmrVersion = dmrVersion

        try:
            iterator = self.iter_body()
            CHUNK_SIZE = 1048576
            # remote dataset
            with tempfile.TemporaryFile() as tmp:
                # write the response to a temporary file
                # so that we can read it in chunks
                for chunk in iterator(chunk_size=CHUNK_SIZE):
                    if chunk:  # filter out keep-alive chunks
                        tmp.write(chunk)
                tmp.seek(0)
                self.raw = BytesReader(tmp)
                self.dmr, self.endianness = self.safe_dmr_and_data()
                dataset = dmr_to_dataset(self.dmr, dmrVersion=self.dmrVersion)
                if self.output_path is not None:
                    if not HAVE_NETCDF4:
                        raise ImportError(
                            "NetCDF4 is required for streaming output. "
                            "Install with: pip install netCDF4"
                        )
                    self._init_netcdf_from_dmr(dataset)
                self.dataset = self.unpack_dap4_data(dataset)
        except TypeError:
            if isinstance(r, webob_Response):
                self.r = r
                if self.r.content_encoding == "gzip":
                    self.raw = BytesReader(
                        gzip.GzipFile(fileobj=BytesIO(self.r.body)).read()
                    )
                else:
                    self.raw = BytesReader(r.body)
            elif isinstance(r, BufferedReader):
                # r comes from reading a local file
                self.r = webob_Response()  # make empty response
                self.raw = BytesReader(r.read())
            else:
                raise TypeError("""
                    Unrecognized file type object for unpacking dap4 binary data.
                    Acceptable formats are `webob.response.Response` and
                    `io.BufferedReader`
                    """)
            self.dmr, self.endianness = self.safe_dmr_and_data()
            # need to split dmr from data
            dataset = dmr_to_dataset(self.dmr)
            self.dataset = self.unpack_dap4_data(dataset)

    def iter_body(self):
        """
        enables iterate over a response, whether the response
        if a requests.Response, requests_cache.Response, or
        httpx.Respose
        """

        if isinstance(self.r, requests.Response):
            return self.r.iter_content
        elif httpx is not None and isinstance(self.r, httpx.Response):
            return self.r.iter_bytes
        else:
            raise TypeError("Unsupported response type")

    def safe_dmr_and_data(self):
        """
        Splits the dap response (.dap) into the dmr (metadata), and the raw
        (binary) data. It also computes the endianness of the data.

        Returns:
            dmr, data, endianness
        """
        # decode the first 4 bytes are CRLF
        chunk_header = numpy.frombuffer(self.raw.read(4), dtype=">u4")[0]
        dmr_length = chunk_header & 0x00FFFFFF
        chunk_type = (chunk_header >> 24) & 0xFF
        if isinstance(self.r, webob_Response):
            dmr = self.raw.read(dmr_length).decode(
                get_charset(self.r, self.user_charset)
            )
        else:
            dmr = self.raw.read(dmr_length)
            # figure out encoding defined in the xml header
            match = re.search(rb'encoding=["\']([^"\']+)["\']', dmr)
            if match:
                encoding = match.group(1).decode("ascii")
            else:
                encoding = self.user_charset
            dmr = dmr.decode(encoding)
        # get endianness from first chunk
        _, _, endianness = decode_chunktype(chunk_type)
        return dmr, endianness

    def _init_netcdf_from_dmr(self, dataset):
        """Create an empty netCDF4 file from a DMR description.

        Requires the optional dependency `netCDF4`.
        """

        if not HAVE_NETCDF4:
            raise ImportError(
                "The 'netCDF4' package is required to initialize a netCDF file "
                "from a DMR response. Currently it is not installed."
            )

        FILL_KEYS = {"missing_value", "fmissing_value"}  # keep _FillValue

        filename = unquote(dataset.name)
        if "nc4" in filename.split("."):
            # filename may have nc4.h5 or something
            # retain only the first extension in name
            # to avoid nc4.nc4
            filename = ".".join(filename.split(".nc4")[:-1] + ["nc4"])
        if not filename.endswith(".nc4"):
            filename = str(Path(filename).with_suffix("")) + ".nc4"

        self.nc = Dataset(self.output_path / filename, "w")
        # start at root
        # create dimensions
        for name, size in dataset.dimensions.items():
            self.nc.createDimension(name, size)
        # create variables
        for var in dataset.variables():
            _FillValue = dataset[var].attributes.pop("_FillValue", None)
            dtype = dataset[var].dtype
            _dims = [dim.split("/")[1] for dim in dataset[var].dims]
            args = {
                "varname": var,
                "datatype": dtype,
                "fill_value": _FillValue,
            }
            if len(_dims) != len(dataset[var].shape):
                _dims = self._get_or_create_dims_for_var(var, dataset[var])
            ncvar = self.nc.createVariable(**args, dimensions=_dims)

            # copy attributes
            for k, v in dataset[var].attributes.items():
                if k in {"Maps", "path"}:
                    continue
                if k in FILL_KEYS:  # avoid writing multiple fill values
                    continue
                ncvar.setncattr(k, v)

        # now identify groups and begin to populate
        variables = [
            var for var in walk(dataset, BaseType) if isinstance(var.parent, GroupType)
        ]
        groups = list(set([var.parent.id for var in variables]))

        for gr in groups:
            self.nc.createGroup(unquote(gr[1:]))
            for dim, size in dataset[gr[1:]].dimensions.items():
                self.nc[unquote(gr[1:])].createDimension(dim, size)

        for var in variables:
            parent = unquote(var.parent.id[1:])
            dtype = var.dtype
            _FillValue = var.attributes.pop("_FillValue", None)
            args = {
                "varname": var.name,
                "datatype": dtype,
                "fill_value": _FillValue,
            }

            _dims = [dim.split("/")[1] for dim in var.dims]
            # copy attributes
            if len(_dims) != len(dataset[var.id].shape):
                _dims = self._get_or_create_dims_for_var(var, dataset[var.id])
            ncvar = self.nc[parent].createVariable(**args, dimensions=_dims)

            # copy attributes
            for k, v in dataset[var.id].attributes.items():
                if k in {"Maps", "path"}:
                    continue
                if k in FILL_KEYS:  # avoid writing multiple fill values
                    continue
                ncvar.setncattr(k, v)

    def _next_phony_dim_name(self) -> str:
        if self.nc is None:
            raise RuntimeError(
                "self.nc is not initialized yet; cannot create dimensions."
            )
        i = 0
        while f"dim{i}" in self.nc.dimensions:
            i += 1
        return f"dim{i}"

    def _ensure_nc_dim(self, dim_name: str, size: int) -> str:
        if dim_name not in self.nc.dimensions:
            self.nc.createDimension(dim_name, size)
        return dim_name

    def _infer_dims_from_parent(
        self, parent_dims: dict, shape: tuple[int, ...]
    ) -> list[str] | None:
        if not parent_dims:
            return None

        items = list(parent_dims.items())  # preserve parent ordering if any
        chosen: list[str] = []
        used: set[str] = set()

        for axis_size in shape:
            match = None
            for name, dim_size in items:
                if name in used:
                    continue
                if int(dim_size) == int(axis_size):
                    match = name
                    break
            if match is None:
                return None
            chosen.append(match)
            used.add(match)

        return chosen

    def _get_or_create_dims_for_var(self, varname: str, data_obj) -> list[str]:
        """
        Decide dims for this variable and ensure they exist in the output netCDF.
        Reuses group-scoped dims when possible.
        """
        shape = tuple(int(s) for s in data_obj.shape)
        rank = len(shape)

        group_path = data_obj.parent.id

        # 1) Honor usable variable dims if present

        var_dims = data_obj.dims
        if len(var_dims) == rank:
            parent = getattr(data_obj, "parent", None)
            parent_dims = getattr(parent, "dimensions", {}) or {}
            for d, axis_size in zip(var_dims, shape):
                self._ensure_nc_dim(d, int(parent_dims.get(d, axis_size)))
            return var_dims

        # 2) Cache hit: same group + same shape
        key = (group_path, shape)
        cached = self._dims_cache.get(key)
        if cached is not None:
            for d, axis_size in zip(cached, shape):
                self._ensure_nc_dim(d, axis_size)
            return cached

        # 3) Infer from parent group dimensions
        parent = getattr(data_obj, "parent", None)
        parent_dims = getattr(parent, "dimensions", {}) or {}
        inferred = self._infer_dims_from_parent(parent_dims, shape)
        if inferred is not None:
            for d, axis_size in zip(inferred, shape):
                self._ensure_nc_dim(d, int(parent_dims.get(d, axis_size)))
            self._dims_cache[key] = inferred
            return inferred

        # 4) Fallback: create phony dims, but cache for reuse
        phony: list[str] = []
        for axis_size in shape:
            d = self._next_phony_dim_name()
            self._ensure_nc_dim(d, axis_size)
            phony.append(d)

        self._dims_cache[key] = phony
        return phony

    def unpack_dap4_data(self, dataset):
        """
        Takes a pydap.DatasetType previously created, and populates its variables
        (BaseType only) with data that is currently in binary form (within a dap
        response).
        """
        checksum_dtype = numpy.dtype(self.endianness + "u4")
        buffer = stream2bytearray(self.raw)
        start = 0
        for variable in walk(dataset, BaseType):
            data, stop = decode_variable(
                buffer,
                start=start,
                variable=variable,
                endian=self.endianness,
            )
            if self.nc is not None:
                name = variable.id.split("/")[-1]
                if isinstance(variable.parent, DatasetType):
                    ncvar = self.nc.variables[name]
                else:
                    parent = unquote(variable.parent.id[1:])
                    ncvar = self.nc[parent].variables[name]
                ncvar[...] = data
                variable._set_data(None)
            else:
                variable._set_data(data)
            if self.checksums:
                checksum = numpy.frombuffer(
                    buffer[stop : stop + 4], dtype=checksum_dtype
                )
                variable.attributes["_DAP4_Checksum_CRC32"] = checksum[0]

            # Jump over the 4 byte chunk_header
            start = stop + 4
        if self.nc is not None:
            self.nc.close()
        return dataset
