from io import BytesIO
import numbers
import os
import tempfile
from datetime import date
import pandas as pd
from pandas.api.types import is_numeric_dtype
from pandas.api.types import is_datetime64_any_dtype as is_datetime
from numpy import nan
import requests
import urllib
import urllib3
import ssl
from abc import ABC, abstractmethod
from sodapy import Socrata as SocrataClient
from pyproj.exceptions import CRSError
from pyproj import CRS
import warnings
from time import sleep
from tqdm import tqdm
from math import ceil
import re
from xlrd.biffh import XLRDError

try:
    import geopandas as gpd
    from shapely.geometry import Point
    _has_gpd = True
except:
    _has_gpd = False

try:
    from .exceptions import OPD_TooManyRequestsError, OPD_DataUnavailableError, OPD_arcgisAuthInfoError, OPD_SocrataHTTPError
except:
    from exceptions import OPD_TooManyRequestsError, OPD_DataUnavailableError, OPD_arcgisAuthInfoError, OPD_SocrataHTTPError

sleep_time = 0.1

# Global parameter for testing both with and without GeoPandas in testing
_use_gpd_force = None

# Default number of records to read per request
_default_limit = 100000
_url_error_msg = "There is likely an issue with the website. Open the URL {} with a web browser to confirm. " + \
                    "See a list of known site outages at https://github.com/openpolicedata/opd-data/blob/main/outages.csv"

# Flag to indicate if ArcGIS queries should be verified against the arcgis package. Used in testing
_verify_arcgis = False


# This is for use if import data sets using Socrata. It is not required.
# Requests made without an app_token will be subject to strict throttling limits
# Get a App Token here: http://dev.socrata.com/docs/app-tokens.html
# Copy the App Token
# Create an environment variable SODAPY_API_KEY and set it equal to the API key
# Setting environment variable in Linux: https://phoenixnap.com/kb/linux-set-environment-variable
# Windows: https://www.wikihow.com/Create-an-Environment-Variable-in-Windows-10
default_sodapy_key = os.environ.get("SODAPY_API_KEY")

class double_format(object):
    def __init__(self, string):
        self.string = string

    def __eq__(self, other) -> bool:
        return isinstance(other, double_format) and self.string == other.string
        # if isinstance(other, double_format):
        #     return other.string == self.string
        # else:
        #     return False

    def format(self, date_field, year):
        return self.string.format(date_field, year, date_field, year)
    
# Based on https://stackoverflow.com/a/73519818/9922439
class CustomHttpAdapter (requests.adapters.HTTPAdapter):
    # "Transport adapter" that allows us to use custom ssl_context.

    def __init__(self, ssl_context=None, **kwargs):
        self.ssl_context = ssl_context
        super().__init__(**kwargs)

    def init_poolmanager(self, connections, maxsize, block=False):
        self.poolmanager = urllib3.poolmanager.PoolManager(
            num_pools=connections, maxsize=maxsize,
            block=block, ssl_context=self.ssl_context)
        
def get_legacy_session():
    ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
    ctx.options |= 0x4  # OP_LEGACY_SERVER_CONNECT
    session = requests.session()
    session.mount('https://', CustomHttpAdapter(ctx))
    return session


class Data_Loader(ABC):
    """Base class for data loaders

    Methods
    -------
    load(year=None, nrows=None, pbar=True, agency=None, opt_filter=None, select=None, output_type=None)
        Load data for query
    get_count(year=None, agency=None, force=False, opt_filter=None, where=None)
        Get number of records/rows generated by query
    get_years(nrows=1)
        Get years contained in data set
    """

    _last_count = None

    @abstractmethod
    def get_count(self, year=None, *, agency=None, force=False, opt_filter=None, where=None):
        pass

    @abstractmethod
    def load(self, year=None, nrows=None, offset=0, *, pbar=True, agency=None, opt_filter=None, select=None, output_type=None):
        pass

    def get_years(self, *, nrows=1, check=None, **kwargs):
        '''Get years contained in data set
        
        Parameters
        ----------
        nrows : int
            (Optional) Number of records to load when checking each year
            
        Returns
        -------
        list
            list containing years in data set
        '''

        if self.date_field==None:
            raise ValueError("A date field is required to get years")

        check_input = check is not None

        if check_input and len(check)==0:
            return []

        if check_input:
            check = check.copy()
            check.sort(reverse=True)
            year = check.pop(0)
        else:
            year = date.today().year

        oldest_recent = 20
        max_misses_gap = 10
        max_misses = oldest_recent
        misses = 0
        years = []
        while misses < max_misses:
            count = self.get_count(year=year)

            if count==0:  # If doesn't have len attribute, it is None
                misses+=1
            else:
                misses = 0
                max_misses = max_misses_gap
                years.append(year)

            sleep(sleep_time)

            year-=1
            if check_input:
                if len(check)==0:
                    break
                while year not in check:
                    year-=1
                check.remove(year)

        return years


class Csv(Data_Loader):
    """
    A class for accessing data from CSV download URLs

    Attributes
    ----------
    url : str
        URL
    date_field : str
        Name of the column that contains the date
    agency_field : str
        Name of column that contains agency names

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True, agency=None)
        Load data for query
    get_count(year=None, agency=None, force=False)
        Get number of records/rows generated by query
    get_years(force=False)
        Get years contained in data set
    """

    def __init__(self, url, date_field=None, agency_field=None):
        '''Create Csv object

        Parameters
        ----------
        url : str
            URL for CSV data
        date_field : str
            (Optional) Name of the column that contains the date
        agency_field : str
                (Optional) Name of the column that contains the agency name (i.e. name of the police departments)
        '''
        
        self.url = url
        self.date_field = date_field
        self.agency_field = agency_field


    def get_count(self, year=None, *,  agency=None, force=False, **kwargs):
        '''Get number of records for a Csv data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        agency : str
            (Optional) Name of agency to filter for.
        force : bool
            (Optional) get_count for CSV file will only run if force=true. In many use cases, it will be more efficient to load the file and manually get the count.
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        if self._last_count is not None and self._last_count[0] == (self.url, year, agency):
            return self._last_count[1]
        if ".zip" not in self.url and year==None and agency==None:
            count = 0
            with requests.get(self.url, stream=True) as r:
                for chunk in r.iter_content(chunk_size=2**16):
                    count += chunk.count(b"\n")

            # Subtract off trailing newlines in last row
            newline = int.from_bytes(b"\n", "big")
            for c in reversed(chunk):
                if c==newline:
                    count-=1
                else:
                    break
        elif force:
            count = len(self.load(year=year, agency=agency))
        else:
            raise ValueError("Extracting the number of records for a single year of a CSV file requires reading the whole file in. In most cases, "+
                "running load() with a year argument to load in the data and manually finding the record count will be more "
                "efficient. If running get_count with a year argument is still desired, set force=True")
        
        self._last_count = ((self.url, year, agency), count)
        return count


    def load(self, year=None, nrows=None, offset=0, *, pbar=True, agency=None, **kwargs):
        '''Download CSV file to pandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested. None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the CSV
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
        agency : str
            (Optional) Name of the agency to filter for. None value returns data for all agencies.
            
        Returns
        -------
        pandas DataFrame
            DataFrame containing table imported from CSV
        '''
        
        if ".zip" in self.url:
            with warnings.catch_warnings():
                # Perhaps use requests iter_content/iter_lines as below to read large CSVs so progress can be shown
                warnings.simplefilter("ignore", category=pd.errors.DtypeWarning)
                try:
                    table = pd.read_csv(self.url, encoding_errors='surrogateescape')
                except urllib.error.HTTPError as e:
                    raise OPD_DataUnavailableError(*e.args, _url_error_msg.format(self.url))
                except Exception as e:
                    raise e
        else:
            use_legacy = False
            try:
                r = requests.head(self.url)
            except requests.exceptions.SSLError as e:
                if "[SSL: UNSAFE_LEGACY_RENEGOTIATION_DISABLED] unsafe legacy renegotiation disabled" in str(e.args[0]) or \
                    "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate" in str(e.args[0]):
                    use_legacy = True
                else:
                    raise e
            except Exception as e:
                raise
                
            if not use_legacy:
                if r.status_code==404:
                    # Try get instead
                    r = requests.get(self.url)
                try:
                    r.raise_for_status()
                    r.close()
                except requests.exceptions.HTTPError as e:
                    raise OPD_DataUnavailableError(*e.args, _url_error_msg.format(self.url))
                except Exception as e:
                    raise e
            
            def get(url, use_legacy):
                if use_legacy:
                    return get_legacy_session().get(url, params=None, stream=True)
                else:
                    return requests.get(url, params=None, stream=True)
                
            with get(self.url, use_legacy) as resp:
                try:
                    table = pd.read_csv(TqdmReader(resp, pbar=pbar), nrows=offset+nrows if nrows is not None else None, encoding_errors='surrogateescape')
                except (urllib.error.HTTPError, pd.errors.ParserError) as e:
                    raise OPD_DataUnavailableError(*e.args, _url_error_msg.format(self.url))
                except Exception as e:
                    raise e

        if offset>0:
            rows_limit = offset+nrows if nrows is not None and offset+nrows<len(table) else len(table)
            table = table.iloc[offset:rows_limit].reset_index(drop=True)
        if nrows is not None and len(table)>nrows:
            table = table.head(nrows)

        table = filter_dataframe(table, date_field=self.date_field, year_filter=year, 
            agency_field=self.agency_field, agency=agency)

        return table

    def get_years(self, *, force=False, **kwargs):
        '''Get years contained in data set
        
        Parameters
        ----------
        force : bool
            (Optional) If false, an exception will be thrown. It may be more efficient to load the table and extract years manually
            
        Returns
        -------
        list
            list containing years in data set
        '''

        if not force:
            raise ValueError("Extracting the years of a CSV file requires reading the whole file in. In most cases, "+
                "running load() with no arguments to load in the whole CSV file and manually finding the years will be more "
                "efficient. If running get_years is still desired, set force=True")
        else:
            if self.date_field==None:
                raise ValueError("No date field provided to access year information")
            df = self.load()
            date_col = pd.to_datetime(df[self.date_field])
            years = list(date_col.dt.year.dropna().unique())
            years.sort()
            return [int(x) for x in years]


class Excel(Data_Loader):
    """
    A class for accessing data from Excel download URLs

    Attributes
    ----------
    url : str
        URL
    date_field : str
        Name of the column that contains the date
    agency_field : str
        Name of column that contains agency names
    excel_file : Pandas ExcelFile
        Object for use in reading data

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True, agency=None)
        Load data for query
    get_count(year=None, agency=None, force=False)
        Get number of records/rows generated by query
    get_years(force=False)
        Get years contained in data set
    """

    def __init__(self, url, date_field=None, agency_field=None):
        '''Create Excel object

        Parameters
        ----------
        url : str
            URL for Excel data
        date_field : str
            (Optional) Name of the column that contains the date
        agency_field : str
                (Optional) Name of the column that contains the agency name (i.e. name of the police departments)
        '''
        
        self.url = url
        self.date_field = date_field
        self.agency_field = agency_field
        
        try:
            self.excel_file = pd.ExcelFile(url)
        except urllib.error.HTTPError as e:
            if str(e) in ["HTTP Error 406: Not Acceptable", 'HTTP Error 403: Forbidden']:
                # 406 error: https://stackoverflow.com/questions/34832970/http-error-406-not-acceptable-python-urllib2
                # File-like input for URL: https://stackoverflow.com/questions/57815780/how-can-i-directly-handle-excel-file-link-python/57815864#57815864
                headers = {'User-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A'}
                r = requests.get(url, stream=True, headers=headers)
                r.raise_for_status()
                file_like = BytesIO(r.content)
                self.excel_file = pd.ExcelFile(file_like)
            else:
                raise OPD_DataUnavailableError(*e.args, _url_error_msg.format(self.url))
        except XLRDError as e:
            if len(e.args)>0 and e.args[0] == "Workbook is encrypted" and \
                any([url.startswith(x) for x in ["http://www.rutlandcitypolice.com"]]):  # Only perform on known datasets to prevent security issues
                try:
                    import msoffcrypto
                except:
                    raise ImportError(f"{url} is encrypted. OpenPoliceData may be able to open it if msoffcrypto-tool " + 
                        "(https://pypi.org/project/msoffcrypto-tool/) is installed (pip install msoffcrypto-tool)")
                # Download file to temporary file
                r = requests.get(url)
                r.raise_for_status()
                # https://stackoverflow.com/questions/22789951/xlrd-error-workbook-is-encrypted-python-3-2-3
                fp_decrypt = tempfile.TemporaryFile(suffix=".xls")
                with tempfile.TemporaryFile() as fp:
                    fp.write(r.content)
                    fp.seek(0)

                    # Try and unencrypt workbook with magic password
                    wb_msoffcrypto_file = msoffcrypto.OfficeFile(fp)

                    # https://nakedsecurity.sophos.com/2013/04/11/password-excel-velvet-sweatshop/
                    wb_msoffcrypto_file.load_key(password='VelvetSweatshop')
                    wb_msoffcrypto_file.decrypt(fp_decrypt)

                fp_decrypt.seek(0)
                self.excel_file = pd.ExcelFile(fp_decrypt)
            else:
                raise
        except:
            raise


    def get_count(self, year=None, *,  agency=None, force=False, **kwargs):
        '''Get number of records for a Excel data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        agency : str
            (Optional) Name of agency to filter for.
        force : bool
            (Optional) get_count for Excel file will only run if force=true. In many use cases, it will be more efficient to load the file and manually get the count.
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        if self._last_count is not None and self._last_count[0]==(self.url, year, agency):
            return self._last_count[1]
        elif force:
            count = len(self.load(year=year, agency=agency))
            self._last_count = ((self.url, year, agency), count)
            return count
        else:
            raise ValueError("Extracting the number of records for an Excel file requires reading the whole file in. In most cases, "+
                "running load() to load in the data and manually finding the record count will be more "
                "efficient. If running get_count with a year argument is still desired, set force=True")


    def __get_sheets(self):
        names = self.excel_file.sheet_names
        if all([x.isdigit() for x in names]):
            possible_years = [int(x) for x in names]
            # possible_years.sort()

            cur_year = date.today().year
            def is_year(x):
                return x > 2000 and x<=cur_year

            last_year = None
            year_dict = {}
            years_found = True
            for k in range(len(possible_years)):
                year = possible_years[k]
                if is_year(year):
                    if last_year!=None and year!=last_year+1:
                        raise ValueError("Unable to parse sheet names")
                        years_found = False
                        break
                    year_dict[year] = names[k]
                elif len(names[k])==4:
                    # Check for typo
                    success = False
                    for j in range(len(names[k])-1):
                        new_name = list(names[k])
                        tmp = new_name[j]
                        new_name[j] = new_name[j+1]
                        new_name[j+1] = tmp
                        new_name = str("".join(new_name))
                        new_year = int(new_name)
                        if is_year(new_year) and new_year==last_year+1:
                            year = new_year
                            year_dict[new_year] = names[k]
                            success = True
                            break

                    if not success:
                        raise ValueError("Unable to parse sheet names")
                        years_found = False
                        break
                else:
                    raise ValueError("Unable to parse sheet names")
                    years_found = False
                    break

                last_year = year

            if years_found:
                return year_dict, True

        return names, False


    def load(self, year=None, nrows=None, offset=0, *, agency=None, **kwargs):
        '''Download Excel file to pandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested. None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the Excel
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        agency : str
            (Optional) Name of the agency to filter for. None value returns data for all agencies.
            
        Returns
        -------
        pandas DataFrame
            DataFrame containing table imported from Excel file

        Note: Older Excel files (.xls) and OpenDocument file formats (.odf, .ods, .odt) are not supported. Please submit an issue if this is needed.
        '''

        nrows_read = offset+nrows if nrows is not None else None
        year_dict, has_year_sheets = self.__get_sheets()

        with warnings.catch_warnings():        
            # Progress bar is not used because TqdmReader object has no attribute 'seek' and would need to be modified to not have a newline operator
            warnings.simplefilter("ignore", category=pd.errors.DtypeWarning)
            if has_year_sheets:
                if year==None:
                    year = list(year_dict.keys())
                    year.sort()
                    year = [year[0], year[-1]]
                if not isinstance(year, list):
                    year = [year, year]

                table = None
                cols_added = 0
                for y in range(year[0], year[1]+1):
                    if y in year_dict:
                        df = pd.read_excel(self.excel_file, nrows=nrows_read, sheet_name=year_dict[y])

                        df = self.__clean(df)

                        if isinstance(table, type(None)):
                            table = df
                            col_matches = [[k] for k in range(len(df.columns))]
                        else:
                            if not df.columns.equals(table.columns):
                                # Conditional for preventing column names from being too different
                                if len(df.columns)+cols_added == len(table.columns) and \
                                    (df.columns == table.columns[:len(df.columns)]).sum()>=len(df.columns)-3-cols_added:
                                    # Try to find a typo
                                    for m in [j for j in range(len(df.columns)) if table.columns[j]!=df.columns[j]]:
                                        for k in col_matches[m]:
                                            if table.columns[k]==df.columns[m]:
                                                break
                                            try:
                                                from rapidfuzz import fuzz
                                            except:
                                                raise ImportError(f"{self.url} requires installation of rapidfuzz " + 
                                                    "(https://pypi.org/project/rapidfuzz/) to load data from multiple years (pip install rapidfuzz)")

                                            if fuzz.ratio(table.columns[k], df.columns[m]) > 80:
                                                print(f"Identified difference in column names when combining sheets {year_dict[y-1]} and {year_dict[y]}. " + 
                                                    f"Column names are '{table.columns[k]}' and '{df.columns[m]}'. This appears to be a typo. " + 
                                                    f"These columns are assumed to be the same and will be combined as column '{table.columns[k]}'")
                                                df.columns = [table.columns[k] if j==m else df.columns[j] for j in range(len(df.columns))]
                                                break
                                        else:
                                            print(f"Column '{table.columns[m]}' in current DataFrame does not match '{df.columns[m]}' in new DataFrame. "+ 
                                                "When they are concatenated, both columns will be included.")
                                            col_matches[m].append(len(table.columns))
                                            cols_added+=1
                                            # raise ValueError(f"Column {table.columns[k]} in table does not match {df.columns[k]} in df")
                                else:
                                    raise ValueError("Columns don't match")
                            table = pd.concat([table, df], ignore_index=True)

                        if nrows_read!=None and len(table)>=nrows_read:
                            break

                if isinstance(table, type(None)):
                    return table
            else:
                table = pd.read_excel(self.excel_file, nrows=nrows_read)
                table = self.__clean(table)               

        if offset>0:
            rows_limit = nrows_read if nrows_read is not None and nrows_read<len(table) else len(table)
            table = table.iloc[offset:rows_limit].reset_index(drop=True)
        if nrows is not None and len(table)>nrows:
            table = table.head(nrows)

        # Check for empty rows at the bottom
        num_empty = table.isnull().sum(axis=1)
        empty_rows = num_empty==len(table.columns)
        if empty_rows.any():
            # Check if all rows after first empty row are empty or almost empty
            empty_rows = empty_rows[empty_rows]
            num_empty = num_empty.loc[empty_rows.index[0]:]
            if ((num_empty / len(table.columns)) > 0.75).all():
                table = table.head(empty_rows.index[0])

        # Clean up column names
        table.columns = [x.strip() if isinstance(x, str) else x for x in table.columns]

        table = filter_dataframe(table, date_field=self.date_field, year_filter=year, 
            agency_field=self.agency_field, agency=agency)

        return table


    def __clean(self, df):
        # Row names may not be the 1st row in which case columns need to be fixed
        max_drops = 5
        num_drops = 0
        while sum([(pd.isnull(x) or "Unnamed" in x) for x in df.columns]) / len(df.columns) > 0.5:
            new_cols = [x for x in df.iloc[0]]
            if all([isinstance(x, str) or pd.isnull(x) for x in new_cols]):
                df.columns = new_cols
                df.drop(index=df.index[0], inplace=True)
                df.reset_index(drop=True, inplace=True)
                num_drops+=1

                if len(df)==0 or num_drops>=max_drops:
                    raise ValueError("Unable to find column names")

        df = df.convert_dtypes()

        # Check for 1st row being row numbers
        if pd.isnull(df.columns[0]) and list(df.iloc[:,0]) in [[k+1 for k in range(0, len(df))], [k for k in range(0, len(df))]]:
            df = df.iloc[:, 1:]

        return df
        

    def get_years(self, *, force=False, **kwargs):
        '''Get years contained in data set
        
        Parameters
        ----------
        force : bool
            (Optional) If false, an exception will be thrown. It may be more efficient to load the table and extract years manually
            
        Returns
        -------
        list
            list containing years in data set
        '''

        year_dict, has_year_sheets = self.__get_sheets()

        if has_year_sheets:
            years = list(year_dict.keys())
            years.sort()
            return list(year_dict.keys())
        if not force:
            raise ValueError("Extracting the years of a Excel file requires reading the whole file in. In most cases, "+
                "running load() with no arguments to load in the whole CSV file and manually finding the years will be more "
                "efficient. If running get_years is still desired, set force=True")
        else:
            if self.date_field==None:
                raise ValueError("No date field provided to access year information")
            df = self.load()
            if is_datetime(df[self.date_field]):
                years = list(df[self.date_field].dt.year.dropna().unique())
            elif is_numeric_dtype(df[self.date_field]):
                years = list(df[self.date_field].dropna().unique())
            else:
                raise TypeError("Unknown date column format")
            return [int(x) for x in years]


class Arcgis(Data_Loader):
    """
    A class for accessing data from ArcGIS clients

    Attributes
    ----------
    url : str
        URL
    date_field : str
        Name of the column that contains the date
    max_record_count : int
        Maximum number of records that can be returned per request
    is_table : bool
        Indicates if dataset is a table. Tables do not have GPS data

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True)
        Load data for query
    get_count(year=None, where=None)
        Get number of records/rows generated by query
    get_years()
        Get years contained in data set
    """
    
    def __init__(self, url, date_field=None):
        '''Create Arcgis object

        Parameters
        ----------
        url : str
            URL for ArcGIS data
        date_field : str
            (Optional) Name of the column that contains the date
        '''

        self._date_format = None
        self.date_field = date_field
        self.verify = False

        # Table vs. Layer: https://developers.arcgis.com/rest/services-reference/enterprise/layer-feature-service-.htm
        # The layer resource represents a single feature layer or a nonspatial table in a feature service. 
        # A feature layer is a table or view with at least one spatial column.
        # For tables, it provides basic information about the table such as its ID, name, fields, types, and templates. 
        # For feature layers, in addition to the table information, it provides information such as its geometry type, min and max scales, and spatial reference.

        p = re.search(r"(MapServer|FeatureServer)/\d+", url)
        self.url = url[:p.span()[1]]

        # Get metadata
        meta = self.__request()

        if "maxRecordCount" in meta:
            self.max_record_count = meta["maxRecordCount"]
        else:
            self.max_record_count = None

        if meta["type"]=="Feature Layer":
            self.is_table = False
        elif meta["type"]=="Table":
            self.is_table = True
        else:
            raise ValueError("Unexpected ArcGIS layer type: {}".format(meta["type"]))

        self.__set_verify(_verify_arcgis)


    def __set_verify(self, verify):
        # Sets whether to validate OPD queries against ones using arcgis package
        if not verify:
            self.verify = verify
        else:
            # https://developers.arcgis.com/python/
            try:
                from arcgis.features import FeatureLayerCollection
                self.verify =verify
            except:
                self.verify = False
                return

            last_slash =self.url.rindex("/")
            layer_num = self.url[last_slash+1:]
            base_url = self.url[:last_slash]
            try:
                layer_collection = FeatureLayerCollection(base_url)
            except Exception as e:
                if len(e.args)>0:
                    if "Error Code: 500" in e.args[0]:
                        raise OPD_DataUnavailableError(self.url, e.args)
                    elif "A general error occurred: 'authInfo'" in e.args[0]:
                        raise OPD_arcgisAuthInfoError(self.url, e.args)
                raise e
            except: raise

            is_table = True
            self.__active_layer = None
            for layer in layer_collection.layers:
                layer_url = layer.url
                if layer_url[-1] == "/":
                    layer_url = layer_url[:-1]
                if layer_num == layer_url[last_slash+1:]:
                    self.__active_layer = layer
                    is_table = False
                    break

            if is_table != self.is_table:
                raise ValueError("is_table is not read in properly")

            if self.is_table:
                for layer in layer_collection.tables:
                    layer_url = layer.url
                    if layer_url[-1] == "/":
                        layer_url = layer_url[:-1]
                    if layer_num == layer_url[last_slash+1:]:
                        self.__active_layer = layer
                        break

            if self.__active_layer == None:
                raise ValueError("Unable to find layer")


    def get_count(self, year=None, *,  where=None, **kwargs):
        '''Get number of records for a Arcgis data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        where : str
            (Optional) SQL where query
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''
        
        if self._last_count is not None and self._last_count[0]==(year,where):
            record_count = self._last_count[1]
            where_query = self._last_count[2]
        elif where==None:
            where_query, record_count = self.__construct_where(year)
        else:
            where_query = where
            try:
                record_count = self.__request(where=where, return_count=True)["count"]
                if self.verify:
                    record_count_orig = self.__active_layer.query(where=where, return_count_only=True)
                    if record_count_orig!=record_count:
                        raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                else:
                    raise
            except:
                raise

        self._last_count = ((year,where), record_count, where_query)

        return record_count


    def __request(self, where=None, return_count=False, out_fields="*", out_type="json", offset=0, count=None, sp_ref=None):
        
        # Running with no inputs or just an out_type will return metadata only
        url = self.url + "/"
        params = {}
        if where != None:
            url+="query"
            params["where"] = where
            params["outFields"] = out_fields
            if return_count:
                params["returnCountOnly"] = True
            else:
                # Don't add offset for returning record count. The maximum value returned appears to be the maxRecordCount not the total count of records.
                # If it's ever desired to get the record with an offset, recommend getting the record count without the offset and then subtracting the offset.
                params["resultOffset"] = offset
                if sp_ref!=None:
                    params["outSR"] = sp_ref
                if self.date_field!=None:
                    params["orderByFields"] = self.date_field
                
            if count!=None:
                params["resultRecordCount"] = count

        params["f"] = out_type
        params["cacheHint"] = False

        try:
            r = requests.get(url, params=params)
            r.raise_for_status()
        except requests.exceptions.SSLError as e:
            if "[SSL: UNSAFE_LEGACY_RENEGOTIATION_DISABLED] unsafe legacy renegotiation disabled" in str(e.args[0]):
                with get_legacy_session() as session:
                    r = session.get(url, params=params)
                    
                r.raise_for_status()
            else:
                raise e
        except requests.HTTPError as e:
            if len(e.args)>0:
                if "503 Server Error" in e.args[0]:
                    raise OPD_DataUnavailableError(self.url, e.args)

            else: raise e
        except: raise
        
        return r.json()


    def __construct_where(self, year=None):
        where_query = ""
        if self._last_count is not None and self._last_count[0]==(year,None):
            record_count = self._last_count[1]
            where_query = self._last_count[2]
        elif self.date_field!=None and year!=None:
            where_query, record_count = self._build_date_query(year)
        else:
            where_query = '1=1'
            try:
                record_count = self.__request(where=where_query, return_count=True)["count"]
                if self.verify:
                    record_count_orig = self.__active_layer.query(where=where_query, return_count_only=True)
                    if record_count_orig!=record_count:
                        raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                else:
                    raise
            except:
                raise

        self._last_count = ((year,None), record_count, where_query)

        return where_query, record_count

    
    def _build_date_query(self, year):

        # List of error messages that can occur for bad queries as we search for the right query format
        query_err_msg = ["Unable to complete operation", "Failed to execute query", "Unable to perform query", "Database error has occurred", 
                         "'where' parameter is invalid", "Parsing error"]
        
        where_query = ""
        zero_found = False
        if self._date_format in [0,1] or self._date_format==None:
            start_date, stop_date = _process_date(year)
            
            for k in range(0,2):
                if k==0:
                    where_query = f"{self.date_field} >= '{start_date}' AND  {self.date_field} < '{stop_date}'"
                else:
                    # break
                    # Dataset (San Jose crash data) that required this does not function well so removing its functionality for now to speed up this function.
                    # This is the recommended way but it has been found to not work sometimes. One dataset was found that requires this.
                    # https://gis.stackexchange.com/questions/451107/arcgis-rest-api-unable-to-complete-operation-on-esrifieldtypedate-in-query
                    stop_date_tmp = stop_date.replace("T"," ")
                    where_query = f"{self.date_field} >= TIMESTAMP '{start_date}' AND  {self.date_field} < TIMESTAMP '{stop_date_tmp}'"
            
                try:
                    record_count = self.__request(where=where_query, return_count=True)
                    if "count" not in record_count:
                        err = "Error Code {}: ".format(record_count["error"]["code"])
                        if len(record_count["error"]["message"])!=0:
                            err+=record_count["error"]["message"]
                            err+=" "
                        if len(record_count["error"]["details"])>0 and len(record_count["error"]["details"][0])>0:
                            err+=record_count["error"]["details"][0]
                        raise KeyError(err)
                    record_count = record_count["count"]
                    if self.verify:
                        record_count_orig = self.__active_layer.query(where=where_query, return_count_only=True)
                        if record_count_orig!=record_count:
                            raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
                    if self._date_format!=None or record_count>0:
                        self._date_format = k
                        return where_query, record_count
                    else:
                        zero_found = True
                except Exception as e:
                    if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                        raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                    elif len(e.args)>0 and any([x in e.args[0] for x in query_err_msg]):
                        # This query throws an error for this dataset. Try another one below
                        pass
                    else:
                        raise
                except:
                    raise


        where_formats = [
            "{} LIKE '%[0-9][0-9]/[0-9][0-9]/{}%'",   # mm/dd/yyyy
            double_format("{} LIKE '{}/[0-9][0-9]' OR {} LIKE '{}/[0-9]'"),                # yyyy/mm
            "{} = {}",                # yyyy
            double_format("{} LIKE '[0-9][0-9]-{}' OR {} LIKE '[0-9]-{}'")   # mm-yyyy or m-yyyy
        ]
        # Make year iterable
        year = [year] if isinstance(year, numbers.Number) else year

        if self._date_format not in [None, 0, 1] and any([isinstance(x,str) and len(x)!=4 for x in year]):
            # Currently can only handle years
            raise ValueError("Currently unable to handle non-year inputs")

        for format in where_formats:
            if self._date_format==format or self._date_format==None:
                where_query = format.format(self.date_field, year[0])
                for x in year[1:]:
                    where_query = f"{where_query} or " + format.format(self.date_field, x)

                try:
                    record_count = self.__request(where=where_query, return_count=True)
                    if "count" not in record_count:
                        err = "Error Code {}: ".format(record_count["error"]["code"])
                        if len(record_count["error"]["message"])!=0:
                            err+=record_count["error"]["message"]
                            err+=" "
                        if len(record_count["error"]["details"])>0 and len(record_count["error"]["details"][0])>0:
                            err+=record_count["error"]["details"][0]
                        raise KeyError(err)
                    record_count = record_count["count"]
                    if self.verify:
                        record_count_orig = self.__active_layer.query(where=where_query, return_count_only=True)
                        if record_count_orig!=record_count:
                            raise ValueError(f"Record count of {record_count} does not equal count from arcgis package of {record_count_orig}")
                    if self._date_format!=None or record_count>0:
                        self._date_format = format
                        return where_query, record_count
                    else:
                        zero_found = True
                except Exception as e:
                    if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                        raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                    elif len(e.args)>0 and any([x in e.args[0] for x in query_err_msg]):
                        # This query throws an error for this dataset. Try another one below
                        pass
                    else:
                        raise
                except:
                    raise

        if not zero_found:
            raise AttributeError(f"Unable to find date format for {self.url}")

        return "", 0

    
    def load(self, year=None, nrows=None, offset=0, *, pbar=True, **kwargs):
        '''Download table from ArcGIS to pandas or geopandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the table
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
            
        Returns
        -------
        pandas or geopandas DataFrame
            DataFrame containing table imported from ArcGIS
        '''
        
        where_query, record_count = self.__construct_where(year)
        
        # Update record count for request record offset
        record_count-=offset
        if record_count<=0:
            return None

        batch_size = self.max_record_count or _default_limit
        nrows = nrows if nrows!=None and record_count>=nrows else record_count
        batch_size = nrows if nrows < batch_size else batch_size
        num_batches = ceil(nrows / batch_size)
            
        pbar = pbar and num_batches>1
        if pbar:
            bar = tqdm(desc=self.url, total=nrows, leave=False) 
            
        features = []
        for batch in range(num_batches):
            bs = batch_size if batch<num_batches-1 else nrows-batch*batch_size
            try:
                data = self.__request(where=where_query, offset=offset+batch*batch_size, count=bs)
                features.extend(data["features"])
                if self.verify:
                    layer_query_result_old = self.__active_layer.query(where=where_query, result_offset=batch*batch_size, 
                        result_record_count=batch_size, return_all_records=False)
                    sdf = layer_query_result_old.sdf

                    attributes = pd.DataFrame.from_records([x["attributes"] for x in data["features"]])
                    for col in [x["name"] for x in data["fields"] if x["type"]=='esriFieldTypeDate']:
                        attributes[col] = pd.to_datetime(attributes[col], unit="ms")
                    
                    if not self.is_table:
                        geom_old = sdf.pop("SHAPE")
                        has_point_geometry = any("geometry" in x and "x" in x["geometry"] for x in data["features"])
                        if not has_point_geometry and geom_old.apply(lambda x: x is not None).any():
                            raise KeyError("Geometry not found")
                        if _has_gpd and has_point_geometry:
                            x = [x["geometry"]["x"] if "geometry" in x else None for x in data["features"]]
                            y = [x["geometry"]["y"] if "geometry" in x else None for x in data["features"]]
                            x_old = [x["x"] if x!=None else None for x in geom_old]
                            y_old = [x["y"] if x!=None else None for x in geom_old]
                            if x!=x_old and any([x!=None for x in x_old]):
                                raise ValueError(f"X coordinates do not match for {self.url}")
                            if y!=y_old and any([x!=None for x in y_old]):
                                raise ValueError(f"Y coordinates do not match for {self.url}")

                    if not sdf.columns.equals(attributes.columns):
                        # A case was found where data from arcgis package had extra OBJECT_ID column (OBJECT_ID1 and OBJECT_ID)
                        # These columns are not used anyway so just remove them
                        missing_cols = [x for x in sdf.columns if x not in attributes.columns]
                        for col in missing_cols:
                            if col in ["OBJECTID"]:
                                sdf.pop(col)
                            else:
                                raise ValueError(f"Column '{col}' exists in arcgis query but not opd query")

                    if not sdf.equals(attributes):
                        raise ValueError(f"DataFrames do not match for {self.url}")

                if batch==0:
                    date_cols = [x["name"] for x in data["fields"] if x["type"]=='esriFieldTypeDate']
                    if not self.is_table:
                        wkid = data["spatialReference"]["wkid"]
                    if len(data["features"]) not in [batch_size, nrows]:
                        num_rows = len(data["features"])
                        raise ValueError(f"Number of rows is {num_rows} but is expected to be max rows to read {batch_size} or total number of rows {nrows}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                else:
                    raise
            except:
                raise

            if pbar:
                bar.update(len(data["features"]))

        if pbar:
            bar.close()

        df = pd.DataFrame.from_records([x["attributes"] for x in features])
        for col in date_cols:
            df[col] = pd.to_datetime(df[col], unit="ms", errors='coerce')

        if len(df) > 0:
            has_point_geometry = any("geometry" in x and "x" in x["geometry"] for x in features)
            if not self.is_table and has_point_geometry:
                if _use_gpd_force is not None:
                    if not _has_gpd and _use_gpd_force:
                        raise ValueError("User cannot force GeoPandas usage when it is not installed")
                    use_gpd = _use_gpd_force
                else:
                    use_gpd = _has_gpd

                if use_gpd:
                    geometry = []
                    for feat in features:
                        if "geometry" not in feat:
                            geometry.append(None)
                        elif feat["geometry"]["x"]=="NaN":
                            geometry.append(Point(nan, nan))
                        else:
                            geometry.append(Point(feat["geometry"]["x"], feat["geometry"]["y"]))

                    try:
                        df = gpd.GeoDataFrame(df, crs=wkid, geometry=geometry)
                    except CRSError:
                        # CRS input method recommended by pyproj team to deal with CRSError for wkid = 102685
                        crs = CRS.from_authority("ESRI", wkid)
                        df = gpd.GeoDataFrame(df, crs=crs, geometry=geometry)
                    except Exception as e:
                        raise e
                else:
                    geometry = [feat["geometry"] if "geometry" in feat else None for feat in features]

                    if "geolocation" not in df:
                        df["geolocation"] = geometry

            return df
        else:
            return None


class Carto(Data_Loader):
    """
    A class for accessing data from Carto clients

    Attributes
    ----------
    url : str
        URL
    data_set : str
        Dataset ID
    date_field : str
        Name of the column that contains the date

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True)
        Load data for query
    get_count(year=None, where=None)
        Get number of records/rows generated by query
    get_years()
        Get years contained in data set
    """
    
    def __init__(self, url, data_set, date_field=None):
        '''Create Carto object

        Parameters
        ----------
        url : str
            URL or username for Carto data
        data_set : str
            Dataset ID
        date_field : str
            (Optional) Name of the column that contains the date
        '''

        # https://carto.com/developers/sql-api/guides/making-calls/
        # Format of URL is https://{username}.carto.com/api/v2/sql

        # Strip out username to ensure that URL is properly formatted
        username = url
        if username.startswith("https://"):
            username = username.replace("https://", "")

        if ".carto" in username:
            username = username[:username.find(".carto")]

        url_clean = "https://" + username + ".carto.com/api/v2/sql"

        self.url = url_clean
        self.data_set = data_set
        self.date_field = date_field


    def get_count(self, year=None, **kwargs):
        '''Get number of records for a data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        if self._last_count is not None and self._last_count[0]==year:
            return self._last_count[1]
        else:
            where = self.__construct_where(year)
            json = self.__request(where=where, return_count=True)
            count = json["rows"][0]["count"]

        self._last_count = (year, count, where)

        return count


    def __request(self, where=None, return_count=False, out_fields="*", out_type="GeoJSON", offset=0, count=None):

        query = "SELECT "
        params = {}
        if return_count:
            query+="count(*)"
        else:
            query+=out_fields
            params["format"] = out_type

        query+=f" FROM {self.data_set}"

        if where != None:
            query+=" WHERE "+ where

        query+=f" OFFSET {offset}"

        if count!=None:
            query+=f" LIMIT {count}"

        params["q"] = query

        r = requests.get(self.url, params=params)

        try:
            r.raise_for_status()
        except requests.HTTPError as e:
            if len(e.args)>0:
                if "503 Server Error" in e.args[0]:
                    raise OPD_DataUnavailableError(self.url, e.args)
                else:
                    raise

            else: raise e
        except: raise
        
        return r.json()


    def __construct_where(self, year=None):
        if self.date_field!=None and year!=None:
            start_date, stop_date = _process_date(year, date_field=self.date_field)
            where_query = f"{self.date_field} >= '{start_date}' AND {self.date_field} <= '{stop_date}'"
        else:
            where_query = None

        return where_query

    
    def load(self, year=None, nrows=None, offset=0, *, pbar=True, **kwargs):
        '''Download table to pandas or geopandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the table
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
            
        Returns
        -------
        pandas or geopandas DataFrame
            DataFrame containing downloaded
        '''
        
        if self._last_count is not None and self._last_count[0]==year:
            record_count = self._last_count[1]
            where_query = self._last_count[2]
        else:
            where_query = self.__construct_where(year)
            json = self.__request(where=where_query, return_count=True)
            record_count = json["rows"][0]["count"]
            self._last_count = (year, record_count, where_query)

        record_count-=offset
        if record_count<=0:
            return None

        batch_size = _default_limit
        nrows = nrows if nrows!=None and record_count>=nrows else record_count
        batch_size = nrows if nrows < batch_size else batch_size
        num_batches = ceil(nrows / batch_size)
            
        pbar = pbar and num_batches>1
        if pbar:
            bar = tqdm(desc=self.url, total=nrows, leave=False)

        # When requesting data as GeoJSON, no type information is return so request it now
        type_info = self.__request(count=0, out_type="JSON")
            
        features = []
        for batch in range(num_batches):
            bs = batch_size if batch<num_batches-1 else nrows-batch*batch_size

            try:
                data = self.__request(where=where_query, offset=offset+batch*batch_size, count=bs)
                features.extend(data["features"])

                if batch==0 and len(features)>0:
                    date_cols = [key for key, x in type_info["fields"].items() if x["type"]=='date']
                    if len(data["features"]) not in [batch_size, nrows]:
                        num_rows = len(data["features"])
                        raise ValueError(f"Number of rows is {num_rows} but is expected to be max rows to read {batch_size} or total number of rows {nrows}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.url))
                else:
                    raise
            except:
                raise

            if pbar:
                bar.update(len(data["features"]))

        if pbar:
            bar.close()

        df = pd.DataFrame.from_records([x["properties"] for x in features])
        for col in date_cols:
            df[col] = pd.to_datetime(df[col])

        if len(df) > 0:
            has_point_geometry = any("geometry" in x and x["geometry"]!=None for x in features)
            if has_point_geometry:
                if _use_gpd_force is not None:
                    if not _has_gpd and _use_gpd_force:
                        raise ValueError("User cannot force GeoPandas usage when it is not installed")
                    use_gpd = _use_gpd_force
                else:
                    use_gpd = _has_gpd

                if use_gpd:
                    geometry = []
                    for feat in features:
                        if "geometry" not in feat or feat["geometry"]==None or len(feat["geometry"]["coordinates"])<2:
                            geometry.append(None)
                        else:
                            geometry.append(Point(feat["geometry"]["coordinates"][0], feat["geometry"]["coordinates"][1]))

                    df = gpd.GeoDataFrame(df, crs=4326, geometry=geometry)
                else:
                    geometry = [feat["geometry"] if "geometry" in feat else None for feat in features]

                    if "geolocation" not in df:
                        df["geolocation"] = geometry
                    else:
                        raise KeyError("geolocation already exists in DataFrame")

            return df
        else:
            return None


class Socrata(Data_Loader):
    """
    A class for accessing data from Socrata clients

    Attributes
    ----------
    url : str
        URL of data homepage
    data_set : str
        Dataset ID
    date_field : str
        Name of the column that contains the date
    client : sodapy.Socrata
        Socrata client

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True, opt_filter=None, select=None, output_type=None)
        Load data for query
    get_count(year=None, opt_filter=None, where=None)
        Get number of records/rows generated by query
    get_years()
        Get years contained in data set
    """

    def __init__(self, url, data_set, date_field=None, key=default_sodapy_key):
        '''Create Socrata object

        Parameters
        ----------
        url : str
            URL for Socrata data
        data_set : str
            Dataset ID for Socrata data
        date_field : str
            (Optional) Name of the column that contains the date
        key : str
            (Optional) Socrata app token to prevent throttling of the data request
        '''
        self.url = url
        self.data_set = data_set
        self.date_field = date_field
        # Unauthenticated client only works with public data sets. Note 'None'
        # in place of application token, and no username or password:
        self.client = SocrataClient(self.url, key, timeout=60)


    def __construct_where(self, year, opt_filter):
        where = ""
        if self.date_field!=None and year!=None:
            start_date, stop_date = _process_date(year, date_field=self.date_field)
            where = self.date_field + " between '" + start_date + "' and '" + stop_date +"'"

        if opt_filter is not None:
            if not isinstance(opt_filter, list):
                opt_filter = [opt_filter]

            andStr = " AND "
            for filt in opt_filter:
                where += andStr + filt

            if where[0:len(andStr)] == andStr:
                where = where[len(andStr):]

        return where


    def get_count(self, year=None, *,  opt_filter=None, where=None, **kwargs):
        '''Get number of records for a Socrata data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        opt_filter : str
            (Optional) Additional filter to apply to data (beyond any date filter specified by self.date_field and year)
        where: str
            (Optional) where statement for Socrata query. If None, where statement will be constructed from self.date_field, year, and opt_filter
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        if where==None:
            where = self.__construct_where(year, opt_filter)

        if self._last_count is not None and self._last_count[0]==(year, opt_filter, where):
            return self._last_count[1]

        try:
            results = self.client.get(self.data_set, where=where, select="count(*)")
        except (requests.HTTPError, requests.exceptions.ReadTimeout) as e:
            raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.url))
        except Exception as e: 
            if len(e.args)>0 and (e.args[0]=='Unknown response format: text/html' or \
                "Read timed out" in e.args[0]):
                raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.url))
            else:
                raise e  
            
        try:
            num_rows = float(results[0]["count"])
        except:
            num_rows = float(results[0]["count_1"]) # Value used in VT Shootings data

        count = int(num_rows)
        self._last_count = ((year, opt_filter, where),count)

        return count


    def load(self, year=None, nrows=None, offset=0, *, pbar=True, opt_filter=None, select=None, output_type=None, **kwargs):
        '''Download table from Socrata to pandas or geopandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the table
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
        opt_filter : str
            (Optional) Additional filter to apply to data (beyond any date filter specified by self.date_field and year)
        select : str
            (Optional) select statement to REST API
        output_type : str
            (Optional) Data type for the output. Allowable values: GeoDataFrame, DataFrame, set, list. Default: GeoDataFrame or DataFrame
            
        Returns
        -------
        pandas or geopandas DataFrame
            DataFrame containing table
        '''

        N = 1  # Initialize to value > 0 so while loop runs
        start_offset = offset

        where = self.__construct_where(year, opt_filter)
        
        if _use_gpd_force is not None:
            if not _has_gpd and _use_gpd_force:
                raise ValueError("User cannot force GeoPandas usage when it is not installed")
            use_gpd = _use_gpd_force
        else:
            use_gpd = _has_gpd

        record_count = int(self.get_count(where=where))
        record_count-=offset
        if record_count<=0:
            return None
        batch_size =  _default_limit
        nrows = nrows if nrows!=None and record_count>=nrows else record_count
        batch_size = nrows if nrows < batch_size else batch_size
        num_batches = ceil(nrows / batch_size)
            
        show_pbar = pbar and num_batches>1 and select==None
        if show_pbar:
            bar = tqdm(desc=f"URL: {self.url}, Dataset: {self.data_set}", total=num_batches, leave=False)

        order = None
        if select == None:
            # order guarantees data order remains the same when paging
            # Order by date if available otherwise the data ID. 
            # https://dev.socrata.com/docs/paging.html#2.1
            order = ":id" if self.date_field==None else self.date_field

        while N > 0:
            try:
                results = self.client.get(self.data_set, where=where,
                    limit=batch_size,offset=offset, select=select, order=order)
            except requests.HTTPError as e:
                raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.url))
            except Exception as e: 
                if len(e.args)>0 and (e.args[0]=='Unknown response format: text/html' or \
                    "Read timed out" in e.args[0]):
                    raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.url))
                else:
                    raise e

            if use_gpd and output_type==None:
                # Check for geo info
                for r in results:
                    if "geolocation" in r or "geocoded_column" in r:
                        output_type = "GeoDataFrame"
                        break

            if output_type == "set":
                if offset==start_offset:
                    df = set()

                if len(results)>0:
                    filt_key = select.replace("DISTINCT ", "")
                    results = [row[filt_key] for row in results if len(row)>0]
                    results = set(results)
                    df.update(results)

            elif output_type == "list":
                if offset==start_offset:
                    df = list()

                if len(results)>0:
                    [df.append(row[select]) for row in results]

            elif use_gpd and output_type=="GeoDataFrame":
                output_type = "GeoDataFrame"
                # Presumed to be a list of properties that possibly include coordinates
                geojson = {"type" : "FeatureCollection", "features" : []}
                for p in results:
                    feature = {"type" : "Feature", "properties" : p}
                    if "geolocation" in feature["properties"]:
                        geo = feature["properties"].pop("geolocation")
                        if list(geo.keys()) == ["human_address"]:
                            feature["geometry"] = {"type" : "Point", "coordinates" : (nan, nan)}  
                        elif "coordinates" in geo:
                            feature["geometry"] = geo
                        else:
                            feature["geometry"] = {"type" : "Point", "coordinates" : (float(geo["longitude"]), float(geo["latitude"]))}
                    elif "geocoded_column" in feature["properties"]:
                        feature["geometry"] = feature["properties"].pop("geocoded_column")
                    else:
                        feature["geometry"] = {"type" : "Point", "coordinates" : (nan, nan)} 
                    
                    geojson["features"].append(feature)

                if len(results)>0:
                    new_gdf = gpd.GeoDataFrame.from_features(geojson, crs=4326)
                        
                    if offset==start_offset:
                        df = new_gdf
                    else:
                        df = pd.concat([df, new_gdf], ignore_index=True)
            else:
                output_type = "DataFrame"
                rows = pd.DataFrame.from_records(results)
                if offset==start_offset:
                    df = pd.DataFrame(rows)
                else:
                    df = pd.concat([df, rows], ignore_index=True)

            N = len(results)
            offset += N

            if show_pbar:
                bar.update()

            if N>=nrows:
                break

        if show_pbar:
            bar.close()

        if isinstance(df, pd.DataFrame) and nrows is not None and len(df)>nrows:
            df = df.head(nrows)
        return df


def _process_date(date, date_field=None):
    if not isinstance(date, list):
        date = [date, date]

    if len(date) !=2:
        raise ValueError("date should be a list of length 2: [startYear, stopYear]")

    if date[0] > date[1]:
        raise ValueError('date[0] needs to be smaller than or equal to date[1]')

    if type(date[0]) == str:
        # This should already be in date format
        start_date = date[0]
    elif date_field != None and date_field.lower() == "year":
        # Assuming this as actually a string or numeric field for the year rather than a datestamp
        start_date = str(date[0])
    else:
        start_date = str(date[0]) + "-01-01"

    if type(date[1]) == str:
        # This should already be in date format
        stop_date = date[1]
    elif date_field != None and date_field.lower() == "year":
        # Assuming this as actually a string or numeric field for the year rather than a datestamp
        stop_date = str(date[1])
    else:
        stop_date  = str(date[1]) + "-12-31T23:59:59.999"

    return start_date, stop_date


def filter_dataframe(df, date_field=None, year_filter=None, agency_field=None, agency=None):
    '''Filter dataframe by agency and/or year (range)
    
    Parameters
    ----------
    df : pandas or geopandas dataframe
        Dataframe containing the data
    date_field : str
        (Optional) Name of the column that contains the date
    year_filter : int, list
        (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
    agency_field : str
        (Optional) Name of the column that contains the agency name (i.e. name of the police departments)
    agency : str
        (Optional) Name of the agency to filter for. None value returns data for all agencies.
    '''

    if date_field != None and not hasattr(df[date_field], "dt"):
        with warnings.catch_warnings():
            # Ignore future warning about how this operation will be attempted to be done inplace:
            # In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. 
            # To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)`
            warnings.simplefilter("ignore", category=FutureWarning)
            if date_field.lower()!="year":
                df.loc[:, date_field] = pd.to_datetime(df[date_field])
    
    if year_filter != None and date_field != None:
        if isinstance(year_filter, list):
            df = df[(df[date_field].dt.year >= year_filter[0]) & (df[date_field].dt.year <= year_filter[1])]
        else:
            date_col = pd.to_datetime(df[date_field])
            df = df[date_col.dt.year == year_filter]

    if agency != None and agency_field != None:
        df = df.query(agency_field + " = '" + agency + "'")

    return df


# https://stackoverflow.com/questions/73093656/progress-in-bytes-when-reading-csv-from-url-with-pandas
class TqdmReader:
    # Older versions of pandas check if reader has these properties even though they are not used
    write = []
    __iter__ = []
    def __init__(self, resp, pbar=True, nrows=None):
        total_size = int(resp.headers.get("Content-Length", 0))

        self.rows_read = 0
        if nrows != None:
            self.nrows = nrows
        else:
            self.nrows = float("inf")
        self.resp = resp
        self.pbar = pbar
        if self.pbar:
            self.bar = tqdm(
                desc=resp.url,
                total=total_size,
                unit="iB",
                unit_scale=True,
                unit_divisor=1024,
                leave=False
            )

        self.reader = self.read_from_stream()

    def read_from_stream(self):
        for line in self.resp.iter_lines():
            line += b"\n"
            if self.pbar:
                self.bar.update(len(line))
            yield line

    def read(self, n=0):
        try:
            if self.rows_read >= self.nrows:
                # Number of rows read is greater than user-requested limit
                return ""
            self.rows_read += 1
            return next(self.reader)
        except StopIteration:
            if self.pbar:
                self.bar.update(self.bar.total - self.bar.n)
            return ""