from dataclasses import dataclass
import itertools
from math import ceil
import numbers
from numpy import nan
import pandas as pd
import re
import requests
from time import sleep
from tqdm import tqdm
from typing import Optional
import warnings

from .data_loader import Data_Loader, str2json, _url_error_msg, get_legacy_session, _process_date, _default_limit, _use_gpd_force, _has_gpd
from ..datetime_parser import to_datetime
from ..exceptions import OPD_DataUnavailableError, OPD_arcgisAuthInfoError, OPD_TooManyRequestsError
from .. import log

if _has_gpd:
    import geopandas as gpd
    from shapely.geometry import Point

logger = log.get_logger()

# Flag to indicate if ArcGIS queries should be verified against the arcgis package. Used in testing
_verify_arcgis = False

class repeat_format(object):
    def __init__(self, string):
        self.string = string
        self.repeat = int(string.count('{}')/2)

    def __eq__(self, other) -> bool:
        return isinstance(other, repeat_format) and self.string == other.string

    def format(self, date_field, year):
        args = []
        for _ in range(self.repeat):
            args.extend([date_field, year])
        args = tuple(args)
        return self.string.format(*args)
      

class Arcgis(Data_Loader):
    """
    A class for accessing data from ArcGIS clients

    Parameters
    ----------
    url : str
        URL
    date_field : str
        Name of the column that contains the date
    max_record_count : int
        Maximum number of records that can be returned per request
    is_table : bool
        Indicates if dataset is a table. Tables do not have GPS data

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True)
        Load data for query
    get_count(year=None, where=None)
        Get number of records/rows generated by query
    get_years()
        Get years contained in data set
    """

    # Based on https://developers.arcgis.com/rest/services-reference/online/feature-layer.htm
    __max_maxRecordCount = 32000
    
    def __init__(self, url, date_field=None, query=None):
        '''Create Arcgis object

        Parameters
        ----------
        url : str
            URL for ArcGIS data
        date_field : str
            (Optional) Name of the column that contains the date
        query : str, dict
            (Optional) Additional query that will be added to each request
        '''

        self._date_type = None
        self._date_format = None
        self.date_field = date_field
        self.verify = False
        self.query = str2json(query)

        # Table vs. Layer: https://developers.arcgis.com/rest/services-reference/enterprise/layer-feature-service-.htm
        # The layer resource represents a single feature layer or a nonspatial table in a feature service. 
        # A feature layer is a table or view with at least one spatial column.
        # For tables, it provides basic information about the table such as its ID, name, fields, types, and templates. 
        # For feature layers, in addition to the table information, it provides information such as its geometry type, min and max scales, and spatial reference.

        p = re.search(r"(MapServer|FeatureServer)/\d+", url)
        self.url = url[:p.span()[1]]

        # Get metadata
        meta = self.__request()

        if 'type' not in meta and meta['status']=='error':
            raise OPD_DataUnavailableError(self.url, meta['messages'], _url_error_msg.format(self.url))

        if "maxRecordCount" in meta:
            self.max_record_count = meta["maxRecordCount"] if meta['maxRecordCount']<self.__max_maxRecordCount else self.__max_maxRecordCount
        else:
            self.max_record_count = None

        if meta["type"]=="Feature Layer":
            self.is_table = False
        elif meta["type"]=="Table":
            self.is_table = True
        else:
            raise ValueError("Unexpected ArcGIS layer type: {}".format(meta["type"]))

        self.__set_verify(_verify_arcgis)


    def isfile(self):
        '''Returns False to indicate that ArcGIS data is not file-based

        Returns
        -------
        False
        '''
        return False


    def __set_verify(self, verify):
        # Sets whether to validate OPD queries against ones using arcgis package
        if not verify:
            self.verify = verify
        else:
            # https://developers.arcgis.com/python/
            try:
                from arcgis.features import FeatureLayerCollection
                self.verify =verify
            except:
                self.verify = False
                return

            last_slash =self.url.rindex("/")
            layer_num = self.url[last_slash+1:]
            base_url = self.url[:last_slash]
            try:
                layer_collection = FeatureLayerCollection(base_url)
            except Exception as e:
                if len(e.args)>0:
                    if "Error Code: 500" in e.args[0]:
                        raise OPD_DataUnavailableError(self.url, e.args, _url_error_msg.format(self.url))
                    elif "A general error occurred: 'authInfo'" in e.args[0]:
                        raise OPD_arcgisAuthInfoError(self.url, e.args, _url_error_msg.format(self.url))
                raise e
            except: raise

            is_table = True
            self.__active_layer = None
            for layer in layer_collection.layers:
                layer_url = layer.url
                if layer_url[-1] == "/":
                    layer_url = layer_url[:-1]
                if layer_num == layer_url[last_slash+1:]:
                    self.__active_layer = layer
                    is_table = False
                    break

            if is_table != self.is_table:
                raise ValueError("is_table is not read in properly")

            if self.is_table:
                for layer in layer_collection.tables:
                    layer_url = layer.url
                    if layer_url[-1] == "/":
                        layer_url = layer_url[:-1]
                    if layer_num == layer_url[last_slash+1:]:
                        self.__active_layer = layer
                        break

            if self.__active_layer == None:
                raise ValueError("Unable to find layer")


    def get_count(self, year=None, *,  where=None, **kwargs):
        '''Get number of records for a Arcgis data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        where : str
            (Optional) SQL where query
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''
        
        if self._last_count is not None and self._last_count[0]==(year,where):
            logger.debug("Request matches previous count request. Returning saved count.")
            record_count = self._last_count[1]
            where_query = self._last_count[2]
        elif where==None:
            where_query, record_count = self.__construct_where(year)

            if not self.__accurate_count:
                raise ValueError(f"Count is not accurate for year input {self.__accurate_count}. "
                                 "Date field contains data in text format not date format "
                                 "and the text not formatted in a way that makes getting a count "
                                 "possible without loading in the data. Either adjust the input to "
                                 "get_count to get a range of years instead of a range of dates or "
                                 "load in the data for the current date range")
        else:
            where_query = where
            try:
                record_count = self.__request(where=where, return_count=True)["count"]
                if self.verify:
                    record_count_orig = self.__active_layer.query(where=where, return_count_only=True)
                    if record_count_orig!=record_count:
                        raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                else:
                    raise
            except:
                raise

        self._last_count = ((year,where), record_count, where_query)

        return record_count


    def __request(self, where=None, return_count=False, out_fields="*", out_type="json", offset=0, count=None, sp_ref=None):
        
        # Running with no inputs or just an out_type will return metadata only
        url = self.url + "/"
        params = {}
        if where != None:
            url+="query"
            params["where"] = where
            params["outFields"] = out_fields
            if return_count:
                params["returnCountOnly"] = True
            else:
                # Don't add offset for returning record count. The maximum value returned appears to be the maxRecordCount not the total count of records.
                # If it's ever desired to get the record with an offset, recommend getting the record count without the offset and then subtracting the offset.
                params["resultOffset"] = offset
                if sp_ref!=None:
                    params["outSR"] = sp_ref
                if self.date_field!=None:
                    params["orderByFields"] = self.date_field
                
            if count!=None:
                params["resultRecordCount"] = count

        params["f"] = out_type
        params["cacheHint"] = False

        logger.debug(f"Request data from {url}")
        for k,v in params.items():
            logger.debug(f"\t{k} = {v}")

        try:
            r = requests.get(url, params=params)
            r.raise_for_status()
        except requests.exceptions.SSLError as e:
            if "[SSL: UNSAFE_LEGACY_RENEGOTIATION_DISABLED] unsafe legacy renegotiation disabled" in str(e.args[0]):
                with get_legacy_session() as session:
                    r = session.get(url, params=params)
                    
                r.raise_for_status()
            else:
                raise e
        except requests.HTTPError as e:
            if len(e.args)>0:
                if "503 Server Error" in e.args[0]:
                    raise OPD_DataUnavailableError(self.url, e.args, _url_error_msg.format(self.url))

            else: raise e
        except requests.exceptions.ConnectTimeout as e:
            raise OPD_DataUnavailableError(self.url, e.args, _url_error_msg.format(self.url))
        except Exception as e: 
            raise e

        try:
            result = r.json()
        except requests.exceptions.JSONDecodeError:
            # Just raising for now. May add additional messaging later
            # This case is occurring in Charlotte OIS-Incidents data when the where query has OR keyword
            raise

        if isinstance(result, dict) and len(result.keys()) and "error" in result:
            args = ()
            for k,v in result['error'].items():
                if not hasattr(v,'__len__') or len(v)>0:
                    v = v[0] if isinstance(v,list) and len(v)==1 else v
                    args += (k,v)
            raise OPD_DataUnavailableError(url, 'Error returned by ArcGIS query', *args, _url_error_msg.format(self.url))
        
        return result


    def __construct_where(self, year=None, date_range_error=True):
        if self.query:
            where_query = " AND ".join([f"{k} = '{v}'" for k,v in self.query.items()])
        else:
            where_query = ""
        self.__accurate_count = True
        if self._last_count is not None and self._last_count[0]==(year,None):
            record_count = self._last_count[1]
            where_query = self._last_count[2]
        elif self.date_field!=None and year!=None:
            where_query, record_count = self._build_date_query(year, date_range_error)
        else:
            where_query = '1=1' if len(where_query)==0 else where_query
            try:
                record_count = self.__request(where=where_query, return_count=True)["count"]
                if self.verify:
                    record_count_orig = self.__active_layer.query(where=where_query, return_count_only=True)
                    if record_count_orig!=record_count:
                        raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                else:
                    raise
            except:
                raise

        if self.__accurate_count:
            # Count may not be accurate if date ranges are allowed and the date field was a string
            self._last_count = ((year,None), record_count, where_query)

        return where_query, record_count
    
    def _build_date_query(self, year, date_range_error):
        # Determine format by getting some data
        data = None
        if not self._date_type:
            data = self.__request(where='1=1', out_fields=self.date_field, count=1000)
            self._date_type = data['fields'][0]['type']

        if self._date_type=='esriFieldTypeDate':
            where_query, record_count = self._build_date_query_date_type(year)
        elif self._date_type=='esriFieldTypeString':
            where_query, record_count = self._build_date_query_string_type(year, data, date_range_error)
        elif self._date_type in ['esriFieldTypeInteger','esriFieldTypeDouble'] and \
            (self.date_field.lower()=='yr' or 'year' in self.date_field.lower()):
            where_query, record_count = self._build_date_query_date_type(year, is_numeric_year=True)
        else:
            raise NotImplementedError(f"Unknown field {self._date_type}")
        
        return where_query, record_count

    def _build_date_query_string_type(self, year, data, date_range_error):
        if data != None:
            if len(data['features'])==0:
                warnings.warn("No data found in dataset. Unable to determine date format in order to generate query")
                self._date_type = None  # Revert value so that data will not be set in the next call to this function since this function did not successfully run
                return "", 0
            
            dates = [x['attributes'][self.date_field] for x in data['features']]

            # [regex for pattern, Arcgis Pattern OR date delimiter (punctuation between numbers), whether to use inquality to do comparison,
                # OPTIONAL time delimiter]
            @dataclass
            class DateParseParams:
                regex_pattern: re.Pattern
                arcgis_pattern: Optional[str] = None
                ineq_comp: bool = False
                date_delim: str = ""
                full_date: bool = True

            matches = [
                DateParseParams(re.compile(r"^(19|20)\d{6}\b"), ineq_comp=True),  # YYYYMMDD
                DateParseParams(re.compile(r"^(19|20)\d{12}\b"), ineq_comp=True),  # YYYYMMDDHHMMSS
                DateParseParams(re.compile(r"^(19|20)\d{2}-\d{2}-\d{2}(\b|T)"), ineq_comp=True, date_delim="-"),  # YYYY-MM-DDThh:mm:ss
                DateParseParams(re.compile(r"^[A-Z][a-z]+ \d{1,2}, (19|20)\d{2}\b"), "{} LIKE '[A-Z]% [0-9][0-9], {}' OR {} LIKE '[A-Z]% [0-9], {}'"),  # Month DD, YYYY
                DateParseParams(re.compile(r"^\d{1,2}[-/]\d{1,2}[-/](19|20)\d{2}\b"), 
                    "{} LIKE '%[0-9][0-9][/-][0-9][0-9][/-]{}%' OR {} LIKE '%[0-9][/-][0-9][0-9][/-]{}%' OR " + 
                    "{} LIKE '%[0-9][/-][0-9][/-]{}%' OR {} LIKE '%[0-9][0-9][/-][0-9][/-]{}%'"),  # mm/dd/yyyy or mm-dd-yyyy
                DateParseParams(re.compile(r"^\d{4}[-/]\d{1,2}$"), "{} LIKE '{}[-/][0-9][0-9]' OR {} LIKE '{}[-/][0-9]'", full_date=False),  # YYYY-MM or YYYY/M
                DateParseParams(re.compile(r"^\d{4}$"), "{} = '{}'", full_date=False),  # YYYY
                DateParseParams(re.compile(r"^\d{1,2}[-/]\d{4}$"), "{} LIKE '[0-9][0-9][-/]{}' OR {} LIKE '[0-9][-/]{}'", full_date=False),  # MM-YYYY or MM/YYYY
            ]

            hi = 0.0
            idx = None
            for k, m in enumerate(matches):
                if (new:=sum([isinstance(x,str) and m.regex_pattern.search(x) != None for x in dates])/len(dates)) > hi:
                    hi = new
                    idx = k

            if hi < 0.9:
                raise ValueError("Unable to find date string pattern")
            
            self._ineq_comp = matches[idx].ineq_comp
            if self._ineq_comp:
                self._date_delim = matches[idx].date_delim
            else:
                self._date_format = repeat_format(matches[idx].arcgis_pattern)
                self._full_date = matches[idx].full_date

        if self._ineq_comp:
            where_query, record_count = self._build_date_query_date_type(year, self._date_delim, is_date_string=True)
        else:
            year = [year] if isinstance(year, numbers.Number) else year.copy()
            for k,y in enumerate(year):
                if isinstance(y,str) and re.search(r'^\d{4}-\d{2}-\d{2}', y):
                    year[k] = y[:4]
                    self.__accurate_count = False
                    
            if (not self._full_date or date_range_error) and any([isinstance(x,str) and len(x)!=4 for x in year]):
                # Currently can only handle years
                raise ValueError(f"Date column is a string data type at the source {self.url}. "+
                                "Currently only able to filter for a single year (2023) or a year range ([2022,2023]) "+
                                "not a date range ([2022-01-01, 2022-03-01]).")
            
            where_query = self._date_format.format(self.date_field, year[0])
            if len(year)>1:
                for x in range(int(year[0])+1,int(year[1])+1):
                    where_query = f"{where_query} or " + self._date_format.format(self.date_field, x)

            record_count = self.__request(where=where_query, return_count=True)["count"]

        return where_query, record_count
        
        

    def _build_date_query_date_type(self, year, date_delim='-', is_numeric_year=False, is_date_string=False):
        # List of error messages that can occur for bad queries as we search for the right query format
        query_err_msg = ["Unable to complete operation", "Failed to execute query", "Unable to perform query", "Database error has occurred", 
                         "'where' parameter is invalid", "Parsing error",'Query with count request failed']
        
        where_query = ""
        zero_found = False
        if self._date_format in [0,1] or self._date_format==None:
            start_date, stop_date = _process_date(year, force_year=is_numeric_year, is_date_string=is_date_string)

            if date_delim=='':
                start_date = start_date.replace('-','')
                stop_date = stop_date.replace('-','')
            elif date_delim=='/':
                start_date = start_date.replace('-','/')
                stop_date = stop_date.replace('-','/')
            elif date_delim!='-':
                raise NotImplementedError("Unable to handle this delimiter")
            
            for k in range(0,2):
                if self._date_format is not None and self._date_format!=k:
                    continue
                if k==0:
                    if is_numeric_year:
                        where_query = f"{self.date_field} >= {start_date} AND  {self.date_field} <= {stop_date}"
                    else:
                        where_query = f"{self.date_field} >= '{start_date}' AND  {self.date_field} <= '{stop_date}'"
                elif is_numeric_year:
                    break
                else:
                    # break
                    # Dataset (San Jose crash data) that required this does not function well so removing its functionality for now to speed up this function.
                    # This is the recommended way but it has been found to not work sometimes. One dataset was found that requires this.
                    # https://gis.stackexchange.com/questions/451107/arcgis-rest-api-unable-to-complete-operation-on-esrifieldtypedate-in-query
                    stop_date_tmp = stop_date.replace("T"," ")
                    where_query = f"{self.date_field} >= TIMESTAMP '{start_date}' AND  {self.date_field} < TIMESTAMP '{stop_date_tmp}'"
            
                try:
                    record_count = self.__request(where=where_query, return_count=True)["count"]

                    if self.verify:
                        record_count_orig = self.__active_layer.query(where=where_query, return_count_only=True)
                        if record_count_orig!=record_count:
                            raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
                    if self._date_format!=None or record_count>0:
                        self._date_format = k
                        return where_query, record_count
                    else:
                        zero_found = True
                except Exception as e:
                    if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                        raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                    elif any([x in y for x,y in itertools.product(query_err_msg, e.args) if isinstance(y,str)]):
                        # This query throws an error for this dataset. Try another one below
                        pass
                    else:
                        raise
                except:
                    raise

        if not zero_found:
            raise AttributeError(f"Unable to find date format for {self.url}")

        return "", 0
    
    def _build_date_query_old(self, year):

        # List of error messages that can occur for bad queries as we search for the right query format
        query_err_msg = ["Unable to complete operation", "Failed to execute query", "Unable to perform query", "Database error has occurred", 
                         "'where' parameter is invalid", "Parsing error",'Query with count request failed']
        
        where_query = ""
        zero_found = False
        if self._date_format in [0,1] or self._date_format==None:
            start_date, stop_date = _process_date(year)
            
            for k in range(0,2):
                if self._date_format is not None and self._date_format!=k:
                    continue
                if k==0:
                    where_query = f"{self.date_field} >= '{start_date}' AND  {self.date_field} < '{stop_date}'"
                else:
                    # break
                    # Dataset (San Jose crash data) that required this does not function well so removing its functionality for now to speed up this function.
                    # This is the recommended way but it has been found to not work sometimes. One dataset was found that requires this.
                    # https://gis.stackexchange.com/questions/451107/arcgis-rest-api-unable-to-complete-operation-on-esrifieldtypedate-in-query
                    stop_date_tmp = stop_date.replace("T"," ")
                    where_query = f"{self.date_field} >= TIMESTAMP '{start_date}' AND  {self.date_field} < TIMESTAMP '{stop_date_tmp}'"
            
                try:
                    record_count = self.__request(where=where_query, return_count=True)["count"]

                    if self.verify:
                        record_count_orig = self.__active_layer.query(where=where_query, return_count_only=True)
                        if record_count_orig!=record_count:
                            raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
                    if self._date_format!=None or record_count>0:
                        self._date_format = k
                        return where_query, record_count
                    else:
                        zero_found = True
                except Exception as e:
                    if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                        raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                    elif len(e.args)>0 and (any([x in e.args[0] for x in query_err_msg]) or any([x in e.args[-1] for x in query_err_msg])):
                        # This query throws an error for this dataset. Try another one below
                        pass
                    else:
                        raise
                except:
                    raise


        where_formats = [
            "{} LIKE '%[0-9][0-9][/-][0-9][0-9][/-]{}%' OR {} LIKE '%[0-9][/-][0-9][0-9][/-]{}%' OR " + 
                "{} LIKE '%[0-9][/-][0-9][/-]{}%' OR {} LIKE '%[0-9][0-9][/-][0-9][/-]{}%'",   # mm/dd/yyyy or mm-dd-yyyy
            "{} LIKE '{}/[0-9][0-9]' OR {} LIKE '{}/[0-9]'",                # yyyy/mm
            "{} = {}",                # yyyy
            "{} LIKE '[0-9][0-9]-{}' OR {} LIKE '[0-9]-{}'",   # mm-yyyy or m-yyyy
            "{} = '{}'",                # 'yyyy'
            "{}>='{}0101' AND {}<='{}1231'",            # yyyymmdd as float
            "{} LIKE '[A-Z]% [0-9][0-9], {}'"   # {Month Name} dd, yyyy
        ]
        where_formats = [repeat_format(x) for x in where_formats]

        # Make year iterable
        year = [year] if isinstance(year, numbers.Number) else year

        if self._date_format not in [None, 0, 1] and any([isinstance(x,str) and len(x)!=4 for x in year]):
            # Currently can only handle years
            raise ValueError("Currently unable to handle non-year inputs")

        for format in where_formats:
            if self._date_format==format or self._date_format==None:
                where_query = format.format(self.date_field, year[0])
                for x in year[1:]:
                    where_query = f"{where_query} or " + format.format(self.date_field, x)

                try:
                    record_count = self.__request(where=where_query, return_count=True)["count"]

                    if self.verify:
                        record_count_orig = self.__active_layer.query(where=where_query, return_count_only=True)
                        if record_count_orig!=record_count:
                            raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
                    if self._date_format!=None or record_count>0:
                        self._date_format = format
                        return where_query, record_count
                    else:
                        zero_found = True
                except Exception as e:
                    if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                        raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                    elif len(e.args)>0 and (any([x in e.args[0] for x in query_err_msg]) or any([x in e.args[-1] for x in query_err_msg])):
                        # This query throws an error for this dataset. Try another one below
                        pass
                    else:
                        raise
                except:
                    raise

        if not zero_found:
            raise AttributeError(f"Unable to find date format for {self.url}")

        return "", 0

    
    def load(self, year=None, nrows=None, offset=0, *, pbar=True, format_date=True, **kwargs):
        '''Download table from ArcGIS to pandas or geopandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the table
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
        format_date : bool, optional
            If True, known date columns (based on presence of date_field in datasets table or data type information provided by dataset owner) will be automatically formatted
            to be pandas datetimes (or pandas Period in rare cases), by default True
            
        Returns
        -------
        pandas or geopandas DataFrame
            DataFrame containing table imported from ArcGIS
        '''
        
        where_query, record_count = self.__construct_where(year, date_range_error=False)
        
        # Update record count for request record offset
        record_count-=offset
        if record_count<=0:
            return pd.DataFrame()

        batch_size = self.max_record_count or _default_limit
        nrows = nrows if nrows!=None and record_count>=nrows else record_count
        batch_size = nrows if nrows < batch_size else batch_size
        num_batches = ceil(nrows / batch_size)
            
        pbar = pbar and num_batches>1
        if pbar:
            bar = tqdm(desc=self.url, total=nrows, leave=False) 
            
        features = []
        for batch in range(num_batches):
            bs = batch_size if batch<num_batches-1 else nrows-batch*batch_size
            try:
                try:
                    data = self.__request(where=where_query, offset=offset+batch*batch_size, count=bs)
                except:
                    if batch>0:
                        # There may have been an error due to too many requests over a short time. Wait and try again
                        sleep(10)
                        data = self.__request(where=where_query, offset=offset+batch*batch_size, count=bs)
                    else:
                        raise

                features.extend(data["features"])
                if self.verify:
                    layer_query_result_old = self.__active_layer.query(where=where_query, result_offset=batch*batch_size, 
                        result_record_count=batch_size, return_all_records=False)
                    sdf = layer_query_result_old.sdf

                    attributes = pd.DataFrame.from_records([x["attributes"] for x in data["features"]])
                    for col in [x["name"] for x in data["fields"] if x["type"]=='esriFieldTypeDate']:
                        attributes[col] = to_datetime(attributes[col], unit="ms")
                    
                    if not self.is_table:
                        geom_old = sdf.pop("SHAPE")
                        has_point_geometry = any("geometry" in x and "x" in x["geometry"] for x in data["features"])
                        if not has_point_geometry and geom_old.apply(lambda x: x is not None).any():
                            raise KeyError("Geometry not found")
                        if _has_gpd and has_point_geometry:
                            x = [x["geometry"]["x"] if "geometry" in x else None for x in data["features"]]
                            y = [x["geometry"]["y"] if "geometry" in x else None for x in data["features"]]
                            x_old = [x["x"] if x!=None else None for x in geom_old]
                            y_old = [x["y"] if x!=None else None for x in geom_old]
                            if x!=x_old and any([x!=None for x in x_old]):
                                raise ValueError(f"X coordinates do not match for {self.url}")
                            if y!=y_old and any([x!=None for x in y_old]):
                                raise ValueError(f"Y coordinates do not match for {self.url}")

                    if not sdf.columns.equals(attributes.columns):
                        # A case was found where data from arcgis package had extra OBJECT_ID column (OBJECT_ID1 and OBJECT_ID)
                        # These columns are not used anyway so just remove them
                        missing_cols = [x for x in sdf.columns if x not in attributes.columns]
                        for col in missing_cols:
                            if col in ["OBJECTID"]:
                                sdf.pop(col)
                            else:
                                raise ValueError(f"Column '{col}' exists in arcgis query but not opd query")

                    if not sdf.equals(attributes):
                        raise ValueError(f"DataFrames do not match for {self.url}")

                if batch==0:
                    date_cols = [x["name"] for x in data["fields"] if x["type"]=='esriFieldTypeDate' and x['name'].lower()!='time']
                    if not self.is_table:
                        wkid = data["spatialReference"]["wkid"]
                    if len(data["features"]) not in [batch_size, nrows]:
                        num_rows = len(data["features"])
                        raise ValueError(f"Number of rows is {num_rows} but is expected to be max rows to read {batch_size} or total number of rows {nrows}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                else:
                    raise
            except:
                raise

            if pbar:
                bar.update(len(data["features"]))

        if pbar:
            bar.close()

        df = pd.DataFrame.from_records([x["attributes"] for x in features])
        if format_date:
            for col in date_cols:
                if col in df:
                    logger.debug(f"Column {col} had a data type of esriFieldTypeDate. Converting values to datetime objects.")
                    df[col] = to_datetime(df[col], unit="ms", errors='coerce')

        if not self.__accurate_count:
            if not format_date:
                raise ValueError("Dates cannot be filtered if format_date is False for this dataset due to the date column not being a "+
                                 "esriFieldTypeDate type at the Arcgis source. Note: most other Arcgis datasets will work fine if format_date is False")
            logger.debug(f"User requested filtering by a date range but this was NOT done in the Arcgis query "+
                         f"due to the date field not being in a date format. Converting {self.date_field} column to "
                         f"a datetime in order to filter for requested date range {year}")
            df[self.date_field] = to_datetime(df[self.date_field], errors='coerce')
            date_range = [str(x) for x in year]
            if len(date_range[0])==4:
                date_range[0] = date_range[0]+'-01-01'
            if len(date_range[1])==4:
                date_range[1] = date_range[1] + "-12-31T23:59:59.999"
            else:
                date_range[1] = date_range[1] + "T23:59:59.999"

            df = df[ (df[self.date_field] >= date_range[0]) & (df[self.date_field] <= date_range[1]) ]

        if len(df) > 0:
            has_point_geometry = any("geometry" in x and "x" in x["geometry"] for x in features)
            if not self.is_table and has_point_geometry:
                if _use_gpd_force is not None:
                    if not _has_gpd and _use_gpd_force:
                        raise ValueError("User cannot force GeoPandas usage when it is not installed")
                    use_gpd = _use_gpd_force
                else:
                    use_gpd = _has_gpd

                if use_gpd:
                    # pyproj installs with geopandas
                    from pyproj.exceptions import CRSError
                    from pyproj import CRS

                    geometry = []
                    for feat in features:
                        if "geometry" not in feat:
                            geometry.append(None)
                        elif feat["geometry"]["x"]=="NaN":
                            geometry.append(Point(nan, nan))
                        else:
                            geometry.append(Point(feat["geometry"]["x"], feat["geometry"]["y"]))

                    logger.debug("Geometry found. Contructing geopandas GeoDataFrame")
                    try:
                        df = gpd.GeoDataFrame(df, crs=wkid, geometry=geometry)
                    except CRSError:
                        # CRS input method recommended by pyproj team to deal with CRSError for wkid = 102685
                        crs = CRS.from_authority("ESRI", wkid)
                        df = gpd.GeoDataFrame(df, crs=crs, geometry=geometry)
                    except Exception as e:
                        raise e
                else:
                    geometry = [feat["geometry"] if "geometry" in feat else None for feat in features]

                    if "geolocation" not in df:
                        logger.debug("Adding geometry column generated from spatial data provided by request.")
                        df["geolocation"] = geometry

            return df
        else:
            return pd.DataFrame()
