#!python3
# -*- coding: utf-8 -*-
"""

"""
from typing import List, Tuple, Generator, Any, Union, OrderedDict
import io
import json
from warnings import warn

import numpy as np
import requests
from requests.auth import AuthBase
import pandas as pd

from ckanapi_harvesters.auxiliary.error_level_message import ContextErrorLevelMessage, ErrorLevel
from ckanapi_harvesters.auxiliary.list_records import ListRecords, records_to_df
from ckanapi_harvesters.auxiliary.proxy_config import ProxyConfig
from ckanapi_harvesters.auxiliary.ckan_model import CkanResourceInfo, CkanAliasInfo
from ckanapi_harvesters.auxiliary.ckan_map import CkanMap
from ckanapi_harvesters.auxiliary.ckan_auxiliary import bytes_to_megabytes
from ckanapi_harvesters.auxiliary.ckan_auxiliary import assert_or_raise, CkanIdFieldTreatment
from ckanapi_harvesters.auxiliary.ckan_auxiliary import datastore_id_col
from ckanapi_harvesters.auxiliary.ckan_auxiliary import RequestType
from ckanapi_harvesters.auxiliary.ckan_action import CkanActionResponse, CkanNotFoundError, CkanSqlCapabilityError
from ckanapi_harvesters.auxiliary.ckan_errors import (IntegrityError, CkanServerError, CkanArgumentError, SearchAllNoCountsError,
                                                      DataStoreNotFoundError, RequestError)
from ckanapi_harvesters.auxiliary.ckan_progress_callbacks import CkanProgressCallbackABC
from ckanapi_harvesters.ckan_api.ckan_api_params import CkanApiParamsBasic
from ckanapi_harvesters.auxiliary.ckan_api_key import CkanApiKey
from ckanapi_harvesters.ckan_api.ckan_api_0_base import ckan_request_proxy_default_auth_if_ckan

from ckanapi_harvesters.ckan_api.ckan_api_1_map import CkanApiMap

df_download_read_csv_kwargs = dict(keep_default_na=False)

ckan_dtype_mapper = {
    "text": "str",
    "numeric": "float",
    "timestamp": "datetime64",
    "int": "int",
    "name": "str",
    "oid": "str",  # to confirm
    "bool": "object",  # enable None values but if they are present, booleans are converted to str...
    "json": "object",
}

class CkanApiReadOnlyParams(CkanApiParamsBasic):
    map_all_aliases:bool = True
    default_df_download_id_field_treatment: CkanIdFieldTreatment = CkanIdFieldTreatment.SetIndex

    def __init__(self, *, proxies:Union[str,dict,ProxyConfig]=None,
                 ckan_headers:dict=None, http_headers:dict=None):
        super().__init__(proxies=proxies, ckan_headers=ckan_headers, http_headers=http_headers)
        self.df_download_id_field_treatment: CkanIdFieldTreatment = self.default_df_download_id_field_treatment

    def copy(self, new_identifier:str=None, *, dest=None):
        if dest is None:
            dest = CkanApiReadOnlyParams()
        super().copy(dest=dest)
        dest.df_download_id_field_treatment = self.df_download_id_field_treatment
        return dest


## Main class ------------------
class CkanApiReadOnly(CkanApiMap):
    """
    CKAN Database API interface to CKAN server with helper functions using pandas DataFrames.
    This class implements requests to read data from the CKAN server resources / DataStores.
    """

    def __init__(self, url:str=None, *, proxies:Union[str,dict,ProxyConfig]=None,
                 apikey:Union[str,CkanApiKey]=None, apikey_file:str=None,
                 owner_org:str=None, params:CkanApiReadOnlyParams=None,
                 map:CkanMap=None,
                 identifier=None):
        """
        CKAN Database API interface to CKAN server with helper functions using pandas DataFrames.

        :param url: url of the CKAN server
        :param proxies: proxies to use for requests
        :param apikey: way to provide the API key directly (optional)
        :param apikey_file: path to a file containing a valid API key in the first line of text (optional)
        :param owner_org: name of the organization to limit package_search (optional)
        :param params: other connection/behavior parameters
        :param map: map of known resources
        :param identifier: identifier of the ckan client
        """
        super().__init__(url=url, proxies=proxies, apikey=apikey, apikey_file=apikey_file,
                         owner_org=owner_org, map=map, identifier=identifier)
        if params is None:
            params = CkanApiReadOnlyParams()
        if proxies is not None:
            params.proxies = proxies
        self.params: CkanApiReadOnlyParams = params

    def _rx_records_df_clean(self, df: pd.DataFrame) -> None:
        """
        Auxiliary function for cleaning dataframe from DataStore requests

        :param df:
        :return:
        """
        if len(df) > 0 and datastore_id_col in df.columns:
            if self.params.df_download_id_field_treatment == CkanIdFieldTreatment.SetIndex:
                # use _id column as new index
                df.set_index(datastore_id_col, drop=False, inplace=True)
                assert(df.index.is_unique)  # verify integrity
            elif self.params.df_download_id_field_treatment == CkanIdFieldTreatment.Remove:
                # remove "_id" column
                df.pop(datastore_id_col)

    @staticmethod
    def read_fields_type_dict(fields_list_dict: List[dict]) -> OrderedDict:
        return OrderedDict([(field_dict["id"], field_dict["type"]) for field_dict in fields_list_dict])

    @staticmethod
    def read_fields_df_args(fields_type_dict: OrderedDict) -> dict:
        if fields_type_dict is None:
            return {}
        # fields_dtype_dict = fields_type_dict.copy()
        # for key, ckan_type in fields_type_dict.items():
        #     if ckan_type in ckan_dtype_mapper:
        #         fields_dtype_dict[key] = ckan_dtype_mapper[ckan_type]
        #     else:
        #         fields_dtype_dict[key] = "object"
        # return dict(names=list(fields_dtype_dict.keys()), dtype=fields_dtype_dict)
        return dict(names=list(fields_type_dict.keys()))

    @staticmethod
    def from_dict_df_args(fields_type_dict: OrderedDict) -> dict:
        df_args_dict = CkanApiReadOnly.read_fields_df_args(fields_type_dict)
        df_args_dict.pop("names")
        return df_args_dict

    @staticmethod
    def _get_default_bom_option(bom:bool=None, format:str=None, search_method:bool=False) -> Union[bool, None]:
        """
        API datastore_dump includes an option to return the BOM (Byte Order Mark) for requests in CSV/TSV format.
        The BOM helps text-processing tools and applications determine the encoding of the file e.g. to distinguish
        between UTF-8 and UTF-16.

        .. note :: To correctly handle BOM characters in pandas.read_csv, you should specify encoding=utf-8-sig parameter.
            This is taken into account in the decoding function.
        """
        if search_method:
            if bom is not None:
                msg = "Argument bom is not used with API datastore_search and will be ignored"
                warn(msg)
            return None  # argument bom not used with API datastore_search
        elif bom is not None:
            return bom
        elif format is not None:
            return format.lower() in {"csv", "tsv"}  # equivalent to format.lower() not in {"json", "xml"}
        else:
            return True  # default format is CSV

    ## Data queries ------------------
    ### Dump method ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    # NB: dump methods are not exposed to the user by default. Only datastore_search and resource_download methods are exposed.
    def _api_datastore_dump_raw(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                sort:str=None, limit:int=None, offset:int=0, format:str=None, bom:bool=None, params:dict=None,
                                compute_len:bool=False) -> requests.Response:
        """
        URL call to datastore/dump URL. Dumps successive lines in the DataStore.

        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param format: The return format in the returned response (default=csv, tsv, json, xml) (optional)
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :return: raw response
        """
        if compute_len:
            raise SearchAllNoCountsError("datastore_search", f"format={format}")
        if params is None:
            params = {}
        if offset is None:
            offset = 0
        params["offset"] = offset
        if limit is None:
            limit = self.params.default_limit_read
        if limit is not None:
            params["limit"] = limit
        if filters is not None:
            if isinstance(filters, str):
                # not recommended
                params["filters"] = filters
            else:
                params["filters"] = json.dumps(filters)
        if q is not None:
            params["q"] = q
        if fields is not None:
            params["fields"] = fields
        if sort is not None:
            params["sort"] = sort
        if format is not None:
            format = format.lower()
            params["format"] = format
        if bom is None and format is not None:
            bom = CkanApiReadOnly._get_default_bom_option(bom=bom, format=format, search_method=False)
        if bom is not None:
            params["bom"] = bom
        # params["bom"] = True  # useful?
        response = self._ckan_url_request(f"datastore/dump/{resource_id}", method=RequestType.Get, params=params)
        if response.status_code == 200:
            return response
        elif response.status_code == 404 and "DataStore resource not found" in response.text:
            raise DataStoreNotFoundError(resource_id, response.content.decode())
        else:
            raise CkanServerError(self, response, response.content.decode())

    def _api_datastore_dump_df(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                               sort:str=None, limit:int=None, offset:int=0, format:str=None, bom:bool=None, params:dict=None) -> pd.DataFrame:
        """
        Convert output of _api_datastore_dump_raw to pandas DataFrame.
        """
        response = self._api_datastore_dump_raw(resource_id=resource_id, filters=filters, q=q, fields=fields,
                                                sort=sort, limit=limit, offset=offset, format=format, bom=bom,
                                                params=params, compute_len=False)
        if format is not None:
            format = format.lower()
        bom = CkanApiReadOnly._get_default_bom_option(bom=bom, format=format, search_method=False)
        assert_or_raise(bom is not None, RuntimeError())
        with io.StringIO(response.content.decode()) as stream:
            read_csv_argument_bom = dict(encoding="utf-8-sig") if bom else {}  # dict(encoding="utf-8")
            if format is None or format == "csv":
                response_df = pd.read_csv(stream, **df_download_read_csv_kwargs, **read_csv_argument_bom)
            elif format == "tsv":
                response_df = pd.read_csv(stream, sep="\t", **df_download_read_csv_kwargs, **read_csv_argument_bom)  # not tested
            elif format == "json":
                response_dict = json.load(stream)
                fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response_dict["fields"])
                df_args = CkanApiReadOnly.read_fields_df_args(fields_type_dict)
                response_df = records_to_df(response_dict["records"], df_args)
                response_df.attrs["fields"] = fields_type_dict
            elif format == "xml":
                response_df = pd.read_xml(stream, parser="etree") # , xpath=".//row")  # partially tested  # otherwise, necessitates the installation of parser lxml
            else:
                raise NotImplementedError()
        self._rx_records_df_clean(response_df)
        return response_df

    def _api_datastore_dump_all(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                sort:str=None, limit:int=None, offset:int=0, format:str=None, bom:bool=None,
                                requests_limit:int=None, params:dict=None, search_all:bool=True, return_df:bool=True,
                                progress_callback:CkanProgressCallbackABC=None) \
            -> Union[pd.DataFrame, requests.Response]:
        """
        Successive calls to _api_datastore_dump_df until an empty list is received.

        :see: _api_datastore_dump()
        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param format: The return format in the returned response (default=csv, tsv, json, xml) (optional)
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: if False, only the first request is operated
        :return:
        """
        if return_df:
            return self._request_all_results_df(api_fun=self._api_datastore_dump_df, params=params,
                                                limit=limit, offset=offset,
                                                requests_limit=requests_limit, progress_callback=progress_callback,
                                                search_all=search_all, resource_id=resource_id,
                                                filters=filters, q=q, fields=fields, sort=sort, format=format, bom=bom)
        elif search_all:
            # cannot determine the number of records received if the response is not parsed with pandas in this mode
            # at least, the total number of rows should be known
            # concatenation of results requires parsing of the result
            # => this mode is useless => raise error
            raise SearchAllNoCountsError("datastore_dump")
        else:
            response = self._api_datastore_dump_raw(resource_id=resource_id, filters=filters, q=q, fields=fields,
                                                    sort=sort, limit=limit, offset=offset, format=format, bom=bom,
                                                    params=params, compute_len=search_all)
            return response

    def _api_datastore_dump_all_page_generator(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                               sort:str=None, limit:int=None, offset:int=0,
                                               requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                                               format:str=None, bom:bool=None,
                                               params:dict=None, search_all:bool=True, return_df:bool=True) \
            -> Union[Generator[pd.DataFrame, Any, None], Generator[requests.Response, Any, None]]:
        """
        Successive calls to _api_datastore_dump until an empty list is received.
        Generator implementation which yields one DataFrame per request.

        :see: _api_datastore_dump()
        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param format: The return format in the returned response (default=csv, tsv, json, xml) (optional)
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: if False, only the first request is operated
        :return:
        """
        if return_df:
            return self._request_all_results_page_generator(api_fun=self._api_datastore_dump_df, params=params,
                                                            limit=limit, offset=offset,
                                                            requests_limit=requests_limit, progress_callback=progress_callback,
                                                            search_all=search_all, resource_id=resource_id,
                                                            filters=filters, q=q, fields=fields, sort=sort, format=format, bom=bom)
        else:
            return self._request_all_results_page_generator(api_fun=self._api_datastore_dump_raw, params=params,
                                                            limit=limit, offset=offset,
                                                            requests_limit=requests_limit, progress_callback=progress_callback,
                                                            search_all=search_all, resource_id=resource_id,
                                                            filters=filters, q=q, fields=fields, sort=sort, format=format, bom=bom,
                                                            compute_len=search_all)


    ### Search method ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    def _api_datastore_search_raw(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                  distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, format:str=None,
                                  params:dict=None, compute_len:bool=False) -> CkanActionResponse:
        """
        API call to datastore_search. Performs queries on the DataStore.

        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param format: The return format in the returned response (default=objects, csv, tsv, lists) (optional)
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :return:
        """
        if params is None:
            params = {}
        if offset is None:
            offset = 0
        params["offset"] = offset
        if limit is None:
            limit = self.params.default_limit_read
        if limit is not None:
            params["limit"] = limit
        params["resource_id"] = resource_id
        if filters is not None:
            if isinstance(filters, str):
                # not recommended
                params["filters"] = filters
            else:
                params["filters"] = json.dumps(filters)
        if q is not None:
            params["q"] = q
        if fields is not None:
            params["fields"] = fields
        if distinct is not None:
            params["distinct"] = distinct
        if sort is not None:
            params["sort"] = sort
        if format is not None:
            format = format.lower()
            params["records_format"] = format
        response = self._api_action_request(f"datastore_search", method=RequestType.Get, params=params)
        if response.success:
            if response.dry_run:
                return response
            elif format is None or format in ["objects", "lists"]:
                response.len = len(response.result["records"])
                response.total_len = response.result["total"]
            elif compute_len:
                raise SearchAllNoCountsError("datastore_search", f"format={format}")
            return response
        elif response.status_code == 404 and response.error_message["__type"] == "Not Found Error":
            raise DataStoreNotFoundError(resource_id, response.error_message)
        else:
            raise response.default_error(self)

    def _api_datastore_search_df(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                 distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, format:str=None,
                                 params:dict=None, compute_len:bool=True) -> pd.DataFrame:
        """
        Convert output of _api_datastore_search_raw to pandas DataFrame.
        """
        response = self._api_datastore_search_raw(resource_id=resource_id, filters=filters, q=q, fields=fields, format=format,
                                                  distinct=distinct, sort=sort, limit=limit, offset=offset,
                                                  params=params, compute_len=False)
        if response.dry_run:
            return pd.DataFrame()
        if format is not None:
            format = format.lower()
        fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
        if format is None or format == "objects":
            df_args_dict = CkanApiReadOnly.from_dict_df_args(fields_type_dict)
            response_df = pd.DataFrame.from_dict(response.result["records"], **df_args_dict)
        else:
            df_args = CkanApiReadOnly.read_fields_df_args(fields_type_dict)
            if format == "lists":
                response_df = records_to_df(response.result["records"], df_args)
            else:
                with io.StringIO(response.result["records"]) as stream:
                    if format == "csv":
                        response_df = pd.read_csv(stream, **df_args, **df_download_read_csv_kwargs)
                    elif format == "tsv":
                        response_df = pd.read_csv(stream, sep='\t', **df_args, **df_download_read_csv_kwargs)
                    else:
                        raise NotImplementedError()
        self._rx_records_df_clean(response_df)
        response.result.pop("records")
        response_df.attrs["result"] = response.result
        response_df.attrs["fields"] = fields_type_dict
        response_df.attrs["total"] = response.result["total"]
        response_df.attrs["total_was_estimated"] = response.result["total_was_estimated"]
        response_df.attrs["limit"] = response.result["limit"]
        return response_df

    def _api_datastore_search_all(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                  distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, format:str=None,
                                  requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                                  search_all:bool=True, params:dict=None, return_df:bool=True, compute_len:bool=False) \
            -> Union[pd.DataFrame, ListRecords, Any]:
        """
        Successive calls to _api_datastore_search_df until an empty list is received.

        :see: _api_datastore_search()
        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param format: The return format in the returned response (default=objects, csv, tsv, lists) (optional)
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: if False, only the first request is operated
        :return:
        """
        if return_df:
            df = self._request_all_results_df(api_fun=self._api_datastore_search_df, params=params, limit=limit, offset=offset,
                                              requests_limit=requests_limit, progress_callback=progress_callback,
                                              search_all=search_all, resource_id=resource_id,
                                              filters=filters, q=q, fields=fields, distinct=distinct, sort=sort, format=format)
            if "fields" in df.attrs.keys():
                df.attrs["fields"] = df.attrs["fields"][0]
            if "total" in df.attrs.keys():
                assert_or_raise(np.all(np.array(df.attrs["total"]) == df.attrs["total"][0]), IntegrityError("total field varied in the responses"))
                df.attrs["total"] = df.attrs["total"][0]
            return df
        else:
            responses = self._request_all_results_list(api_fun=self._api_datastore_search_raw, params=params, limit=limit, offset=offset,
                                            requests_limit=requests_limit, progress_callback=progress_callback,
                                            search_all=search_all, resource_id=resource_id,
                                            filters=filters, q=q, fields=fields, distinct=distinct, sort=sort, format=format, compute_len=compute_len)
            # aggregate results, depending on the format
            if self.params.dry_run:
                return []
            if format is not None:
                format = format.lower()
            if len(responses) > 0:
                response = responses[0]
                fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
                df_args = CkanApiReadOnly.read_fields_df_args(fields_type_dict)
            else:
                fields_type_dict = None
                df_args = {}
            if format is None or format == "objects":
                return ListRecords(sum([response.result["records"] for response in responses], []))
            else:
                if format == "lists":
                    return sum([response.result["records"] for response in responses], [])
                else:
                    return "\n".join([response.result["records"] for response in responses])

    def _api_datastore_search_all_page_generator(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                                 distinct:bool=None, sort:str=None, limit:int=None, offset:int=0,
                                                 format:str=None, search_all:bool=True,
                                                 requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                                                 params:dict=None, return_df:bool=True) \
            -> Union[Generator[pd.DataFrame, Any, None], Generator[CkanActionResponse, Any, None]]:
        """
        Successive calls to _api_datastore_search_df until an empty list is received.
        Generator implementation which yields one DataFrame per request.

        :see: _api_datastore_search()
        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param format: The return format in the returned response (default=objects, csv, tsv, lists) (optional)
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: if False, only the first request is operated
        :return:
        """
        if return_df:
            return self._request_all_results_page_generator(api_fun=self._api_datastore_search_df, params=params, limit=limit, offset=offset,
                                                            requests_limit=requests_limit, progress_callback=progress_callback,
                                                            search_all=search_all, resource_id=resource_id,
                                                            filters=filters, q=q, fields=fields, distinct=distinct, sort=sort, format=format, compute_len=True)
        else:
            return self._request_all_results_page_generator(api_fun=self._api_datastore_search_raw, params=params,
                                                            limit=limit, offset=offset, search_all=search_all,
                                                            requests_limit=requests_limit, progress_callback=progress_callback,
                                                            resource_id=resource_id, filters=filters, q=q,
                                                            fields=fields, distinct=distinct, sort=sort,
                                                            format=format, compute_len=search_all)


    ### search_sql method ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    def _api_datastore_search_sql_raw(self, sql:str, *, params:dict=None, limit:int=None, offset:int=0) -> CkanActionResponse:
        """
        API call to datastore_search_sql. Performs SQL queries on the DataStore. These queries can be more complex than
        with datastore_search. The DataStores are referenced by their resource_id, surrounded by quotes. The field names
        are referred by their name in upper case, surrounded by quotes.
        __NB__: This action is not available when ckanapi_harvesters.datastore.sqlsearch.enabled is set to false

        :param sql: SQL query e.g. f'SELECT * IN "{resource_id}" WHERE "USER_ID" < 0'
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param params: N/A
        :return:
        """
        if params is None:
            params = {}
        params["sql"] = sql
        if offset is None:
            offset = 0
        params["offset"] = offset
        if limit is None:
            limit = self.params.default_limit_read
        if limit is not None:
            params["limit"] = limit
        response = self._api_action_request(f"datastore_search_sql", method=RequestType.Post, params=params)
        if response.success:
            if response.dry_run:
                return response
            else:
                response.len = len(response.result["records"])
            return response
        elif response.status_code == 400 and response.success_json_loads and response.response.text == '"Bad request - Action name not known: datastore_search_sql"':
            raise CkanSqlCapabilityError(self, response)
        elif response.status_code == 404 and response.success_json_loads and response.error_message["__type"] == "Not Found Error":
            raise CkanNotFoundError(self, "SQL", response)
        else:
            raise response.default_error(self)

    def _api_datastore_search_sql_df(self, sql:str, *, params:dict=None, limit:int=None, offset:int=0) -> pd.DataFrame:
        """
        Convert output of _api_datastore_search_sql_raw to pandas DataFrame.
        """
        response = self._api_datastore_search_sql_raw(sql=sql, params=params, limit=limit, offset=offset)
        fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
        df_args_dict = CkanApiReadOnly.from_dict_df_args(fields_type_dict)
        response_df = pd.DataFrame.from_dict(response.result["records"], **df_args_dict)
        response.result.pop("records")
        response_df.attrs["result"] = response.result
        response_df.attrs["fields"] = fields_type_dict
        # response_df.attrs["total"] = response.result["total"]
        # response_df.attrs["total_was_estimated"] = response.result["total_was_estimated"]
        response_df.attrs["limit"] = response.result["limit"]
        self._rx_records_df_clean(response_df)
        return response_df

    def _api_datastore_search_sql_all(self, sql:str, *, params:dict=None,
                                      search_all:bool=True, limit:int=None, offset:int=0,
                                      requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                                      return_df:bool=True) \
            -> Union[pd.DataFrame, ListRecords]:
        """
        Successive calls to _api_datastore_search_sql until an empty list is received.

        :see: _api_datastore_search_sql()
        :param sql: SQL query e.g. f'SELECT * IN "{resource_id}" WHERE "USER_ID" < 0'
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param params: N/A
        :param search_all: if False, only the first request is operated
        :return:
        """
        if return_df:
            df = self._request_all_results_df(api_fun=self._api_datastore_search_sql_df, params=params,
                                              limit=limit, offset=offset,
                                              requests_limit=requests_limit, progress_callback=progress_callback,
                                              search_all=search_all, sql=sql)
            if "fields" in df.attrs.keys():
                df.attrs["fields"] = df.attrs["fields"][0]
            # if "total" in df.attrs.keys():
            #     assert_or_raise(np.all(np.array(df.attrs["total"]) == df.attrs["total"][0]), IntegrityError("total field varied in the responses"))
            #     df.attrs["total"] = df.attrs["total"][0]
            return df
        else:
            responses = self._request_all_results_list(api_fun=self._api_datastore_search_sql_raw, params=params,
                                                       limit=limit, offset=offset, requests_limit=requests_limit,
                                                       progress_callback=progress_callback,
                                                       search_all=search_all, sql=sql)
            # TODO: test
            if len(responses) > 0:
                response = responses[0]
                fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
            else:
                fields_type_dict = None
            return ListRecords(sum([response.result["records"] for response in responses], []))

    def _api_datastore_search_sql_all_page_generator(self, sql:str, *, params:dict=None,
                                                     search_all:bool=True, limit:int=None, offset:int=0,
                                                     requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                                                     return_df:bool=True) \
            -> Union[Generator[pd.DataFrame, Any, None], Generator[CkanActionResponse, Any, None]]:
        """
        Successive calls to _api_datastore_search_sql until an empty list is received.
        Generator implementation which yields one DataFrame per request.

        :see: _api_datastore_search_sql()
        :param sql: SQL query e.g. f'SELECT * IN "{resource_id}" WHERE "USER_ID" < 0'
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param params: N/A
        :param search_all: if False, only the first request is operated
        :return:
        """
        if return_df:
            return self._request_all_results_page_generator(api_fun=self._api_datastore_search_sql_df, params=params,
                                                            limit=limit, offset=offset, search_all=search_all,
                                                            requests_limit=requests_limit, progress_callback=progress_callback,
                                                            sql=sql)
        else:
            return self._request_all_results_page_generator(api_fun=self._api_datastore_search_sql_raw, params=params,
                                                            limit=limit, offset=offset, search_all=search_all,
                                                            requests_limit=requests_limit, progress_callback=progress_callback,
                                                            sql=sql)


    ## Function aliases to limit the entry-points for the user  -------------------------------------------------------
    def datastore_search(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                         distinct:bool=None, sort:str=None, limit:int=None, offset:int=0, params:dict=None,
                         search_all:bool=False, search_method:bool=True,
                         requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                         format:str=None, bom:bool=None, return_df:bool=True) \
            -> Union[pd.DataFrame, ListRecords, Any, List[CkanActionResponse]]:
        """
        Preferred entry-point for a DataStore read request.
        Uses the API datastore_search

        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: Option to renew the request until there are no more records.
        :param search_method: API method selection (True=datastore_search, False=datastore_dump)
        :return:
        """
        if search_method:
            if return_df and format is None: format = "csv"
            return self._api_datastore_search_all(resource_id, filters=filters, q=q, fields=fields, distinct=distinct, sort=sort,
                                                  limit=limit, offset=offset,
                                                  requests_limit=requests_limit, progress_callback=progress_callback,
                                                  format=format, params=params, search_all=search_all, return_df=return_df)
        else:
            assert_or_raise(distinct is None, CkanArgumentError("DataStore dump", "distinct"))
            if return_df and format is None: format = "csv"
            bom = CkanApiReadOnly._get_default_bom_option(bom=bom, format=format, search_method=search_method)
            return self._api_datastore_dump_all(resource_id, filters=filters, q=q, fields=fields, sort=sort,
                                                limit=limit, offset=offset,
                                                requests_limit=requests_limit, progress_callback=progress_callback,
                                                format=format, bom=bom, params=params, search_all=search_all, return_df=return_df)

    def datastore_dump(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                       distinct:bool=None, sort:str=None, limit:int=None, offset:int=0,
                       requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                       params:dict=None, search_all:bool=True, search_method:bool=True, format:str=None,
                       return_df:bool=True) \
            -> Union[pd.DataFrame, ListRecords, Any, List[CkanActionResponse]]:
        """
        Alias of datastore_search with search_all=True by default.
        Uses the API datastore_search

        :see: datastore_search()
        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: Option to renew the request until there are no more records.
        :param search_method: API method selection (True=datastore_search, False=datastore_dump)
        :param return_df: Return pandas DataFrame (True) or dict (False)
        :return:
        """
        return self.datastore_search(resource_id, filters=filters, q=q, fields=fields,
                                     distinct=distinct, sort=sort, limit=limit, offset=offset, requests_limit=requests_limit,
                                     progress_callback=progress_callback, params=params, search_all=search_all,
                                     search_method=search_method, format=format,
                                     return_df=return_df)

    def datastore_search_page_generator(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                        distinct:bool=None, sort:str=None, limit:int=None, offset:int=0,
                                        requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None, params:dict=None,
                                        search_all:bool=True, search_method:bool=True, format:str=None, bom:bool=None, return_df:bool=True) \
            -> Union[Generator[pd.DataFrame, Any, None], Generator[CkanActionResponse, Any, None], Generator[requests.Response, Any, None]]:
        """
        Preferred entry-point for a DataStore read request.
        Uses the API datastore_search

        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: Option to renew the request until there are no more records.
        :param search_method: API method selection (True=datastore_search, False=datastore_dump)
        :param return_df: Return pandas DataFrame (True) or dict (False)
        :return:
        """
        if search_method:
            if return_df and format is None: format = "csv"
            return self._api_datastore_search_all_page_generator(resource_id, filters=filters, q=q, fields=fields, distinct=distinct, sort=sort,
                                                                 limit=limit, offset=offset, requests_limit=requests_limit, progress_callback=progress_callback,
                                                                 format=format, params=params, search_all=search_all, return_df=return_df)
        else:
            assert_or_raise(distinct is None, CkanArgumentError("DataStore dump", "distinct"))
            if return_df and format is None: format = "csv"
            bom = CkanApiReadOnly._get_default_bom_option(bom=bom, format=format, search_method=search_method)
            return self._api_datastore_dump_all_page_generator(resource_id, filters=filters, q=q, fields=fields, sort=sort,
                                                               limit=limit, offset=offset, requests_limit=requests_limit, progress_callback=progress_callback,
                                                               format=format, bom=bom, params=params, search_all=search_all, return_df=return_df)

    def datastore_search_cursor(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                distinct:bool=None, sort:str=None, limit:int=None, offset:int=0,
                                total_limit:int=None, requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None, params:dict=None,
                                search_all:bool=True, search_method:bool=True, format:str=None, bom:bool=None, return_df:bool=False) \
            -> Generator[Union[pd.Series, dict, list, str], Any, None]:
        """
        Cursor on rows of datastore_search

        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param total_limit: Limit the number of records to return
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: Option to renew the request until there are no more records.
        :param search_method: API method selection (True=datastore_search, False=datastore_dump)
        :param return_df: Return pandas Series (True) or dict (False)
        :param format: Format of the data requested through the API. This does not change the output if return_df is True.
        :return:
        """
        if total_limit is not None and total_limit <= 0:
            return
        row_index = 0
        page_generator = self.datastore_search_page_generator(resource_id, filters=filters, q=q, fields=fields,
                                                              distinct=distinct, sort=sort, limit=limit, offset=offset,
                                                              requests_limit=requests_limit, progress_callback=progress_callback, params=params,
                                                              search_all=search_all, search_method=search_method, format=format, bom=bom, return_df=return_df)
        if return_df:
            df: pd.DataFrame
            row: pd.Series
            for df in page_generator:
                for row_loc, row in df.iterrows():
                    yield row
                    row_index += 1
                    if total_limit is not None and row_index >= total_limit:
                        return
        elif search_method:
            response: CkanActionResponse
            # response.result: list
            if format is not None:
                format = format.lower()
            if format is None or format == "objects":
                for response in page_generator:
                    fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
                    for element in response.result["records"]:
                        yield element
                        row_index += 1
                        if total_limit is not None and row_index >= total_limit:
                            return
            else:
                for response in page_generator:
                    fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
                    for element in response.result["records"]:
                        yield element
                        row_index += 1
                        if total_limit is not None and row_index >= total_limit:
                            return
        else:
            raise TypeError("dumping datastore without parsing with a DataFrame does not return an iterable object")

    def datastore_dump_page_generator(self, resource_id:str, *, filters:dict=None, q:str=None, fields:List[str]=None,
                                      distinct:bool=None, sort:str=None, limit:int=None, offset:int=0,
                                      requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None, params:dict=None,
                                      search_all:bool=True, search_method:bool=True, format:str=None, bom:bool=None, return_df:bool=True) \
            -> Union[Generator[pd.DataFrame, Any, None], Generator[CkanActionResponse, Any, None]]:
        """
        Function alias to datastore_search_generator with search_all=True by default.
        Uses the API datastore_search

        :see: datastore_search_generator
        :param resource_id: resource id.
        :param filters: The base argument to filter values in a table (optional)
        :param q: Full text query (optional)
        :param fields: The base argument to filter columns (optional)
        :param distinct: return only distinct rows (optional, default: false) e.g. to return distinct ids: fields="id", distinct=True
        :param sort: Argument to sort results e.g. sort="index, quantity desc"  or  sort="index asc"
        :param limit: Limit the number of records per request
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param offset: Offset in the returned records
        :param params: Additional parameters such as filters, q, sort and fields can be given. See DataStore API documentation.
        :param search_all: Option to renew the request until there are no more records.
        :param search_method: API method selection (True=datastore_search, False=datastore_dump)
        :return:
        """
        return self.datastore_search_page_generator(resource_id, filters=filters, q=q, fields=fields,
                                                    distinct=distinct, sort=sort, limit=limit, offset=offset,
                                                    requests_limit=requests_limit, progress_callback=progress_callback, params=params,
                                                    search_all=search_all, search_method=search_method, format=format, bom=bom, return_df=return_df)

    def datastore_search_sql(self, sql:str, *, params:dict=None, search_all:bool=False,
                             limit:int=None, offset:int=0, requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None, return_df:bool=True) \
            -> Union[pd.DataFrame, Tuple[ListRecords, dict]]:
        """
        Preferred entry-point for a DataStore SQL request.
        :see: _api_datastore_search_sql()
        __NB__: This action is not available when ckanapi_harvesters.datastore.sqlsearch.enabled is set to false

        :param sql: SQL query e.g. f'SELECT * IN "{resource_id}" WHERE "USER_ID" < 0'
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param params: N/A
        :param search_all: Option to renew the request until there are no more records.
        :param return_df: Return pandas DataFrame (True) or dict (False)
        :return:
        """
        return self._api_datastore_search_sql_all(sql, params=params, limit=limit, offset=offset,
                                                  requests_limit=requests_limit, progress_callback=progress_callback,
                                                  search_all=search_all, return_df=return_df)

    def datastore_search_sql_page_generator(self, sql:str, *, params:dict=None, search_all:bool=True,
                                            limit:int=None, offset:int=0,
                                            requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                                            return_df:bool=True) \
            -> Union[Generator[pd.DataFrame, Any, None], Generator[CkanActionResponse, Any, None]]:
        """
        Preferred entry-point for a DataStore SQL request.
        :see: _api_datastore_search_sql()

        __NB__: This action is not available when ckanapi_harvesters.datastore.sqlsearch.enabled is set to false

        :param sql: SQL query e.g. f'SELECT * IN "{resource_id}" WHERE "USER_ID" < 0'
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param params: N/A
        :param search_all: Option to renew the request until there are no more records.
        :param return_df: Return pandas DataFrame (True) or dict (False)
        :return:
        """
        return self._api_datastore_search_sql_all_page_generator(sql, params=params, limit=limit, offset=offset,
                                                                 requests_limit=requests_limit, progress_callback=progress_callback,
                                                                 search_all=search_all, return_df=return_df)

    def datastore_search_sql_cursor(self, sql:str, *, params:dict=None, search_all:bool=True,
                                    limit:int=None, offset:int=0, total_limit:int=None,
                                    requests_limit:int=None, progress_callback:CkanProgressCallbackABC=None,
                                    return_df:bool=False) \
            -> Generator[Union[pd.Series,dict], Any, None]:
        """
        Preferred entry-point for a DataStore SQL request, to iterate over records.
        :see: _api_datastore_search_sql()

        __NB__: This action is not available when ckanapi_harvesters.datastore.sqlsearch.enabled is set to false

        :param sql: SQL query e.g. f'SELECT * IN "{resource_id}" WHERE "USER_ID" < 0'
        :param limit: Limit the number of records per request
        :param offset: Offset in the returned records
        :param total_limit: Limit the number of records to return
        :param requests_limit: Limit the number of requests
        :param progress_callback: Progress callback function
        :param params: N/A
        :param search_all: Option to renew the request until there are no more records.
        :param return_df: Return pandas Series (True) or dict (False)
        :return:
        """
        if total_limit is not None and total_limit <= 0:
            return
        row_index = 0
        page_generator = self.datastore_search_sql_page_generator(sql, params=params, search_all=search_all,
                                                                  limit=limit, offset=offset,
                                                                  requests_limit=requests_limit, progress_callback=progress_callback,
                                                                  return_df=return_df)
        if return_df:
            df: pd.DataFrame
            row: pd.Series
            for df in page_generator:
                for row_loc, row in df.iterrows():
                    yield row
                    row_index += 1
                    if total_limit is not None and row_index >= total_limit:
                        return
        else:
            response: CkanActionResponse
            # response.result: list
            element: Any
            for response in page_generator:
                fields_type_dict = CkanApiReadOnly.read_fields_type_dict(response.result["fields"])
                for element in response.result["records"]:
                    yield element
                    row_index += 1
                    if total_limit is not None and row_index >= total_limit:
                        return

    def datastore_search_sql_find_one(self, sql:str, *, params:dict=None,
                                      offset:int=0, return_df:bool=True) -> Union[pd.DataFrame, Tuple[ListRecords, dict]]:
        """
        First element of an SQL request
        """
        df_row = self.datastore_search_sql(sql, limit=1, search_all=False, offset=offset, params=params, return_df=return_df)
        return df_row

    def datastore_search_sql_fields_type_dict(self, sql:str, *, params:dict=None) -> OrderedDict:
        document, fields_dict = self.datastore_search_sql_find_one(sql, offset=0, params=params, return_df=False)
        return fields_dict

    def datastore_search_sql_row_count(self, sql:str, *, params:dict=None) -> int:
        df_row = self.datastore_search_sql_find_one(sql, offset=0, params=params, return_df=True)
        return df_row.attrs["total"]

    def datastore_search_find_one(self, resource_id:str, *, filters:dict=None, q:str=None, distinct:bool=None,
                                  fields:List[str]=None, offset:int=0, return_df:bool=True) \
            -> Union[pd.DataFrame, ListRecords, Any, List[CkanActionResponse]]:
        """
        Request first result of a query

        :param resource_id: resource id
        :return:
        """
        # resource_info = self.get_resource_info_or_request(resource_id)
        # return resource_info.datastore_info.row_count
        df_row = self.datastore_search(resource_id, limit=1, search_all=False, filters=filters, q=q, distinct=distinct,
                                       fields=fields, offset=offset, return_df=return_df)
        return df_row

    def datastore_search_fields_type_dict(self, resource_id:str, *,
                                          filters:dict=None, q:str=None, distinct:bool=None, fields:List[str]=None,
                                          request_missing:bool=True, error_not_mapped:bool=False,
                                          error_not_found:bool=True) -> OrderedDict:
        if fields is None:
            # if no field restriction was provided, refer to the fields of the DataStore
            fields_list = self.get_datastore_fields_or_request(resource_id, return_list=True,
                                                               request_missing=request_missing,
                                                               error_not_mapped=error_not_mapped,
                                                               error_not_found=error_not_found)
            return CkanApiReadOnly.read_fields_type_dict(fields_list)
        else:
            document, fields_dict = self.datastore_search_find_one(resource_id, filters=filters, q=q, distinct=distinct,
                                                                   fields=fields, return_df=False)
            return fields_dict

    def datastore_search_row_count(self, resource_id:str, *, filters:dict=None, q:str=None, distinct:bool=None,
                                   fields:List[str]=None) -> int:
        """
        Request the number of rows in a DataStore

        :param resource_id: resource id
        :return:
        """
        df_row = self.datastore_search_find_one(resource_id, filters=filters, q=q, distinct=distinct,
                                                fields=fields, return_df=True)
        return df_row.attrs["total"]

    def test_sql_capabilities(self, *, raise_error:bool=False) -> bool:
        """
        Test the availability of the API datastore_search_sql

        :return:
        """
        try:
            self.api_help_show("datastore_search_sql", print_output=False)
            return True
        except CkanNotFoundError:
            if raise_error:
                raise CkanSqlCapabilityError(self, CkanActionResponse(requests.Response()))
            return False


    ## Resource download by direct link (FileStore) -----------------------------------------------
    def resource_download(self, resource_id:str, *, method:str=None,
                          proxies:dict=None, headers:dict=None, auth: Union[AuthBase, Tuple[str,str]]=None,
                          verify:Union[bool,str,None]=None, stream:bool=False) \
            -> Tuple[CkanResourceInfo, Union[requests.Response,None]]:
        """
        Uses the link provided in resource_show to download a resource.

        :param resource_id: resource id
        :return:
        """
        resource_info = self.get_resource_info_or_request(resource_id)
        url = resource_info.download_url
        if len(url) == 0:
            return resource_info, None
        response = self.download_url_proxy(url, method=method, auth_if_ckan=ckan_request_proxy_default_auth_if_ckan,
                                           proxies=proxies, headers=headers, auth=auth, verify=verify, stream=stream)
        content_length = response.headers.get("content-length", None)
        if content_length is not None:
            resource_info.download_size_mb = bytes_to_megabytes(int(content_length))
        return resource_info, response

    def resource_download_test_head(self, resource_id:str, *, raise_error:bool=False,
                                    proxies:dict=None, headers:dict=None, auth: Union[AuthBase, Tuple[str,str]]=None, verify:Union[bool,str,None]=None) \
            -> Union[None,ContextErrorLevelMessage]:
        """
        This sends a HEAD request to the resource download url using the CKAN connexion parameters via resource_download.
        The resource is not downloaded but the headers indicate if the url is valid.

        :return: None if successful
        """
        resource_info = self.get_resource_info_or_request_of_id(resource_id)
        try:
            _, response = self.resource_download(resource_id, method="HEAD", proxies=proxies, headers=headers, auth=auth, verify=verify)
        except Exception as e:
            if raise_error:
                raise e from e
            return ContextErrorLevelMessage(f"Resource from URL {resource_info.name}", ErrorLevel.Error, f"Failed to query download url for resource id {resource_id}: {str(e)}")
        if response.ok and response.status_code == 200:
            return None
        else:
            if raise_error:
                raise RequestError(f"Failed to query download url for resource id {resource_id}: status {response.status_code} {response.reason}")
            return ContextErrorLevelMessage(f"Resource from URL {resource_info.name}", ErrorLevel.Error, f"Failed to query download url for resource id {resource_id}: status {response.status_code} {response.reason}")

    def resource_download_df(self, resource_id:str, *, method:str=None,
                          proxies:dict=None, headers:dict=None, auth: Union[AuthBase, Tuple[str,str]]=None, verify:Union[bool,str,None]=None) \
            -> Tuple[CkanResourceInfo, Union[pd.DataFrame,None]]:
        """
        Uses the link provided in resource_show to download a resource and interprets it as a DataFrame.

        :param resource_id: resource id
        :return:
        """
        resource_info, response = self.resource_download(resource_id, method=method, proxies=proxies, headers=headers, auth=auth, verify=verify)
        if response is None:
            return resource_info, None
        with io.StringIO(response.content.decode()) as stream:
            df = pd.read_csv(stream, **df_download_read_csv_kwargs)
        self._rx_records_df_clean(df)
        return resource_info, df

    def map_file_resource_sizes(self, cancel_if_present:bool=True) -> None:
        for resource_id, resource_info in self.map.resources.items():
            if resource_info.download_url:
                if not (cancel_if_present and resource_info.download_size_mb is not None):
                    try:
                        _, response = self.resource_download(resource_id, method="HEAD")
                        content_length_str = response.headers.get("content-length", None)
                    except Exception as e:
                        msg = f"Failed to query download url for resource id {resource_id}: {str(e)}"
                        warn(msg)
                        content_length_str = None
                    if content_length_str is not None:
                        content_length = int(content_length_str)  # raise error if not found or bad format
                        resource_info.download_size_mb = bytes_to_megabytes(content_length)
                    else:
                        resource_info.download_size_mb = None


    ## Mapping of resource aliases from table
    def list_datastore_aliases(self) -> List[CkanAliasInfo]:
        alias_resource_id = "_table_metadata"  # resource name of table containing CKAN aliases
        alias_list_dict = self.datastore_search(alias_resource_id, search_all=True, return_df=False, format="objects", search_method=True)
        alias_list = [CkanAliasInfo(alias_dict) for alias_dict in alias_list_dict]
        for alias_info in alias_list:
            if alias_info.alias_of is not None:
                self.map.resource_alias_index[alias_info.name] = alias_info.alias_of
        return alias_list

    def map_resources(self, package_list:Union[str, List[str]]=None, *, params:dict=None,
                      datastore_info:bool=None, resource_view_list:bool=None, organization_info:bool=None, license_list:bool=None,
                      only_missing:bool=True, error_not_found:bool=True,
                      owner_org:str=None) -> CkanMap:
        # overload including a call to list all aliases
        if len(self.map.resource_alias_index) == 0 and self.params.map_all_aliases:
            self.list_datastore_aliases()
        map = super().map_resources(package_list=package_list, params=params, datastore_info=datastore_info,
                              resource_view_list=resource_view_list, organization_info=organization_info,
                              license_list=license_list, only_missing=only_missing, error_not_found=error_not_found,
                              owner_org=owner_org)
        return map

