import os
import warnings
import pandas as pd
from numpy import nan
import requests
from sodapy import Socrata as SocrataClient
from tqdm import tqdm
from math import ceil
import re

from .data_loader import Data_Loader, _process_date, _url_error_msg, _use_gpd_force, _default_limit, _has_gpd
from ..exceptions import OPD_SocrataHTTPError
from .. import log

if _has_gpd:
    import geopandas as gpd

logger = log.get_logger()

# This is for use if import data sets using Socrata. It is not required.
# Requests made without an app_token will be subject to strict throttling limits
# Get a App Token here: http://dev.socrata.com/docs/app-tokens.html
# Copy the App Token
# Create an environment variable SODAPY_API_KEY and set it equal to the API key
# Setting environment variable in Linux: https://phoenixnap.com/kb/linux-set-environment-variable
# Windows: https://www.wikihow.com/Create-an-Environment-Variable-in-Windows-10
default_sodapy_key = os.environ.get("SODAPY_API_KEY")

class Socrata(Data_Loader):
    """
    A class for accessing data from Socrata clients

    Parameters
    ----------
    url : str
        URL of data homepage
    data_set : str
        Dataset ID
    date_field : str
        Name of the column that contains the date
    client : sodapy.Socrata
        Socrata client

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True, opt_filter=None, select=None, output_type=None)
        Load data for query
    get_count(year=None, opt_filter=None, where=None)
        Get number of records/rows generated by query
    get_years()
        Get years contained in data set
    """

    def __init__(self, url, data_set, date_field=None, key=default_sodapy_key):
        '''Create Socrata object

        Parameters
        ----------
        url : str
            URL for Socrata data
        data_set : str
            Dataset ID for Socrata data
        date_field : str
            (Optional) Name of the column that contains the date
        key : str
            (Optional) Socrata app token to prevent throttling of the data request
        '''
        self.url = url
        self.data_set = data_set
        self.date_field = date_field
        # Unauthenticated client only works with public data sets. Note 'None'
        # in place of application token, and no username or password:
        self.client = SocrataClient(self.url, key, timeout=90)


    def __construct_where(self, year, opt_filter):
        where = ""
        if self.date_field!=None and year!=None:
            filter_year = False
            assume_date = False
            try:
                # Get metadata to ensure that date is not formatted as text
                meta = self.client.get_metadata(self.data_set)
                column = [x for x in meta['columns'] if x['fieldName']==self.date_field]
                if len(column)>0 and 'dataTypeName' in column[0] and column[0]['dataTypeName']=='text':
                    # The date column is text. It may have some metadata about it's largest value which 
                    # will tell us if it's in YYYY-MM-DD format in which case our filtering will still work.
                    # If not, we can only filter by year with a text search.
                    if not ('cachedContents' in column[0] and 'largest' in column[0]['cachedContents'] and \
                        isinstance(column[0]['cachedContents']['largest'], str) and \
                            re.search(r'^\d{4}\-\d{2}\-\d{2}', column[0]['cachedContents']['largest'])):
                        filter_year = True
            except:
                assume_date = True

            if not assume_date and len(column)==0:
                raise ValueError(f"Date field {self.date_field} not found in dataset")
            if filter_year:
                start_date, stop_date = _process_date(year, date_field=self.date_field, force_year=True)
                where = ''
                for y in range(int(start_date),int(stop_date)+1):
                    # %25 is % wildcard symbol
                    if self.url=='data.bloomington.in.gov' and self.data_set=='gpr2-wqbb':
                        # This dataset has a text date field and contains YYYY/MM/DD and MM/DD/YY formats
                        yy = str(y)[2:]
                        where+=self.date_field + f" LIKE '_/_/{yy}' OR " + \
                               self.date_field + f" LIKE '_/__/{yy}' OR " + \
                               self.date_field + f" LIKE '__/_/{yy}' OR " + \
                               self.date_field + f" LIKE '__/__/{yy}' OR " + \
                               self.date_field + rf" LIKE '{y}%' OR "
                    else:
                        where+=self.date_field + rf" LIKE '%{y}%' OR "
                where = where[:-4]
            else:
                start_date, stop_date = _process_date(year, date_field=self.date_field)
                where = self.date_field + " between '" + start_date + "' and '" + stop_date +"'"

        if opt_filter is not None:
            if not isinstance(opt_filter, list):
                opt_filter = [opt_filter]

            andStr = " AND "
            for filt in opt_filter:
                where += andStr + filt

            if where[0:len(andStr)] == andStr:
                where = where[len(andStr):]

        return where
    

    def isfile(self):
        '''Returns False to indicate that Socrata data is not file-based

        Returns
        -------
        False
        '''
        return False
    
    def get_api_url(self):
        url = self.url[:-1] if self.url.endswith('/') else self.url
        url = url if url.startswith('http') else 'https://'+url
        return f"{url}/resource/{self.data_set}.json"


    def get_count(self, year=None, *,  opt_filter=None, where=None, **kwargs):
        '''Get number of records for a Socrata data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        opt_filter : str
            (Optional) Additional filter to apply to data (beyond any date filter specified by self.date_field and year)
        where: str
            (Optional) where statement for Socrata query. If None, where statement will be constructed from self.date_field, year, and opt_filter
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        if where==None:
            where = self.__construct_where(year, opt_filter)

        if self._last_count is not None and self._last_count[0]==(year, opt_filter, where):
            logger.debug("Request matches previous count request. Returning saved count.")
            return self._last_count[1]
        
        logger.debug(f"Request dataset {self.data_set} from {self.url}")
        logger.debug(f"\twhere={where}")
        logger.debug(f"\tselect=count(*)")

        try:
            results = self.client.get(self.data_set, where=where, select="count(*)")
        except (requests.HTTPError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e:
            raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.get_api_url()))
        except Exception as e: 
            if len(e.args)>0 and isinstance(e.args[0],str) and (e.args[0].startswith('Unknown response format: text/html') or \
                "Read timed out" in e.args[0]):
                raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.get_api_url()))
            else:
                raise e  
            
        try:
            num_rows = float(results[0]["count"])
        except:
            num_rows = float(results[0]["count_1"]) # Value used in VT Shootings data

        count = int(num_rows)
        self._last_count = ((year, opt_filter, where),count)

        return count


    def load(self, year=None, nrows=None, offset=0, *, pbar=True, opt_filter=None, select=None, output_type=None, sortby=None, **kwargs):
        '''Download table from Socrata to pandas or geopandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the table
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
        opt_filter : str
            (Optional) Additional filter to apply to data (beyond any date filter specified by self.date_field and year)
        select : str
            (Optional) select statement to REST API
        output_type : str
            (Optional) Data type for the output. Allowable values: GeoDataFrame, DataFrame, set, list. Default: GeoDataFrame or DataFrame
        sortby : str
            (Optional) Columns to sort by. Allowable values: None (defaults to id) or "date"
            
        Returns
        -------
        pandas or geopandas DataFrame
            DataFrame containing table
        '''

        N = 1  # Initialize to value > 0 so while loop runs
        start_offset = offset

        where = self.__construct_where(year, opt_filter)
        
        if _use_gpd_force is not None:
            if not _has_gpd and _use_gpd_force:
                raise ValueError("User cannot force GeoPandas usage when it is not installed")
            use_gpd = _use_gpd_force
        else:
            use_gpd = _has_gpd

        record_count = int(self.get_count(where=where))
        record_count-=offset
        if record_count<=0:
            return pd.DataFrame()
        batch_size =  _default_limit
        nrows = nrows if nrows!=None and record_count>=nrows else record_count
        batch_size = nrows if nrows < batch_size else batch_size
        num_batches = ceil(nrows / batch_size)
            
        show_pbar = pbar and num_batches>1 and select==None
        if show_pbar:
            bar = tqdm(desc=f"URL: {self.url}, Dataset: {self.data_set}", total=num_batches, leave=False)

        order = None
        if select == None:
            if sortby=="date":
                if self.date_field:
                    order = self.date_field
                else:
                    warnings.warn(DeprecationWarning('Date sorting was requested but no date field found. Results will not be sorted. '+
                                                     'This will result in an error in the next release (V1.0)'))
                    order = ":id"
            elif sortby:
                order = sortby
            else:
                # order guarantees data order remains the same when paging
                # Order by date if available otherwise the data ID. 
                # https://dev.socrata.com/docs/paging.html#2.1
                order = ":id"

        while N > 0:
            logger.debug(f"Request dataset {self.data_set} from {self.url}")
            logger.debug(f"\twhere={where}")
            logger.debug(f"\tselect={select}")
            logger.debug(f"\tlimit={batch_size}")
            logger.debug(f"\toffset={offset}")
            logger.debug(f"\torder={order}")
            try:
                results = self.client.get(self.data_set, where=where,
                    limit=batch_size,offset=offset, select=select, order=order)
            except requests.HTTPError as e:
                raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.get_api_url()))
            except Exception as e: 
                arg_str = None
                err = e
                while True:
                    if len(err.args):
                        if isinstance(err.args[0],str):
                            arg_str = err.args[0]
                            break
                        elif isinstance(err.args[0],Exception):
                            err = err.args[0]
                        else:
                            break
                    else:
                        break
                if arg_str and (arg_str=='Unknown response format: text/html' or \
                    "Read timed out" in arg_str):
                    raise OPD_SocrataHTTPError(self.url, self.data_set, *e.args, _url_error_msg.format(self.get_api_url()))
                else:
                    raise e

            if use_gpd and output_type==None:
                # Check for geo info
                for r in results:
                    if "geolocation" in r or "geocoded_column" in r:
                        output_type = "GeoDataFrame"
                        break

            if output_type == "set":
                if offset==start_offset:
                    df = set()

                if len(results)>0:
                    filt_key = select.replace("DISTINCT ", "")
                    results = [row[filt_key] for row in results if len(row)>0]
                    results = set(results)
                    df.update(results)

            elif output_type == "list":
                if offset==start_offset:
                    df = list()

                if len(results)>0:
                    [df.append(row[select]) for row in results]

            elif use_gpd and output_type=="GeoDataFrame":
                output_type = "GeoDataFrame"
                # Presumed to be a list of properties that possibly include coordinates
                geojson = {"type" : "FeatureCollection", "features" : []}
                for p in results:
                    feature = {"type" : "Feature", "properties" : p}
                    if "geolocation" in feature["properties"]:
                        geo = feature["properties"].pop("geolocation")
                        if list(geo.keys()) == ["human_address"]:
                            feature["geometry"] = {"type" : "Point", "coordinates" : (nan, nan)}  
                        elif "coordinates" in geo:
                            feature["geometry"] = geo
                        else:
                            feature["geometry"] = {"type" : "Point", "coordinates" : (float(geo["longitude"]), float(geo["latitude"]))}
                    elif "geocoded_column" in feature["properties"]:
                        feature["geometry"] = feature["properties"].pop("geocoded_column")
                    else:
                        feature["geometry"] = {"type" : "Point", "coordinates" : (nan, nan)} 
                    
                    geojson["features"].append(feature)

                if len(results)>0:
                    logger.debug("Geometry found. Contructing geopandas GeoDataFrame")
                    new_gdf = gpd.GeoDataFrame.from_features(geojson, crs=4326)
                        
                    if offset==start_offset:
                        df = new_gdf
                    else:
                        df = pd.concat([df, new_gdf], ignore_index=True)
            else:
                output_type = "DataFrame"
                rows = pd.DataFrame.from_records(results)
                if offset==start_offset:
                    df = pd.DataFrame(rows)
                else:
                    df = pd.concat([df, rows], ignore_index=True)

            N = len(results)
            offset += N

            if show_pbar:
                bar.update()

            if N>=nrows:
                break

        if show_pbar:
            bar.close()

        if isinstance(df, pd.DataFrame) and nrows is not None and len(df)>nrows:
            df = df.head(nrows)
        return df

