from math import ceil
import warnings
import pandas as pd
import re
import requests
from tqdm import tqdm

from .data_loader import Data_Loader, _url_error_msg, str2json, _process_date
from ..datetime_parser import to_datetime
from ..exceptions import OPD_DataUnavailableError, OPD_TooManyRequestsError
from .. import log

logger = log.get_logger()

class Ckan(Data_Loader):
    """
    A class for accessing data from CKAN clients

    Parameters
    ----------
    url : str
        URL
    data_set : str
        Dataset ID
    date_field : str
        Name of the column that contains the date
    query : str
        Query that will be perfored for each request

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True)
        Load data for query
    get_count(year=None, where=None)
        Get number of records/rows generated by query
    get_years()
        Get years contained in data set
    """
    
    def __init__(self, url, data_set, date_field=None, query=None):
        '''Create Ckan object

        Parameters
        ----------
        url : str
            URL or username for Carto data
        data_set : str
            Dataset ID
        date_field : str
            (Optional) Name of the column that contains the date
        query : str, dict
            (Optional) Additional query that will be added to each request
        '''

        # https://docs.ckan.org/en/2.9/maintaining/datastore.html

        if url.startswith("https://"):
            url = url.replace("https://", "")
        if url.endswith('/'):
            url = url[:-1]

        url_clean = "https://" + url + "/api/3/action/datastore_search_sql"

        self.url = url_clean
        self.data_set = data_set
        self.date_field = date_field
        self.query = str2json(query)
        self._sort_by_date = False

    
    def isfile(self):
        '''Returns False to indicate that CKAN data is not file-based

        Returns
        -------
        False
        '''
        return False
    
    def get_api_url(self):
        return f'{self.url}?sql=SELECT * FROM "{self.data_set}"'


    def get_count(self, year=None, opt_filter=None, **kwargs):
        '''Get number of records for a data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        if self._last_count is not None and self._last_count[0]==year and self._last_count[1]==opt_filter:
            logger.debug("Request matches previous count request. Returning saved count.")
            return self._last_count[2]
        else:
            where = self.__construct_where(year, opt_filter)
            json = self.__request(where=where, return_count=True)
            count = json['result']['records'][0]['count']

        self._last_count = (year, opt_filter, count, where)

        return count


    def __request(self, where=None, return_count=False, out_fields="*", out_type="json", offset=0, count=None, orderby="_id"):

        if isinstance(out_fields, list):
            out_fields = '"' + '", "'.join(out_fields) + '"'
        elif not out_fields:
            out_fields = '*'

        orderby = self.date_field if orderby=='date' else orderby
        orderby = orderby if orderby else "_id"

        query = "SELECT "
        params = {}
        if return_count:
            query+="COUNT("+ out_fields + ")"
        else:
            query+=out_fields
            params["format"] = out_type

        query+=f' FROM "{self.data_set}"'

        if where != None:
            query+=" WHERE "+ where

        default_where = ''
        for k,v in self.query.items():
            if isinstance(v,str):
                v = f"'{v}'"
            default_where += f" AND {k}={v}"

        if len(default_where):
            if where != None:
                query+=default_where
            else:
                query+=" WHERE"+ default_where[4:]

        if not return_count and count!=0 and not out_fields.startswith("DISTINCT"):
            # Order results to ensure data order remains constant if paging
            query+=' ORDER BY "'+ orderby + '"'

        query+=f" OFFSET {offset}"

        if count!=None:
            query+=f" LIMIT {count}"

        params["sql"] = query

        logger.debug(f"Request data from {self.url}")
        for k,v in params.items():
            logger.debug(f"\t{k} = {v}")

        try:
            r = requests.get(self.url, params=params)
        except requests.exceptions.SSLError as e:
            raise OPD_DataUnavailableError(self.url, e.args, _url_error_msg.format(self.get_api_url()))

        try:
            r.raise_for_status()
        except requests.HTTPError as e:
            if len(e.args)>0:
                if any(x in e.args[0] for x in ["503 Server Error", '409 Client Error']):
                    raise OPD_DataUnavailableError(self.url, e.args, _url_error_msg.format(self.get_api_url()))
                else:
                    raise

            else: raise e
        except: raise
        
        return r.json()


    def __construct_where(self, year=None, opt_filter=None, filter_year=False, sample_data=None):
        if self.date_field!=None and year!=None:
            datetime_format = None
            if not sample_data:
                sample_data = self.__request(count=100)
            
            date_col_info = [x for x in sample_data['result']["fields"] if x["id"]==self.date_field]
            if len(date_col_info)==0:
                raise ValueError(f"Date column {self.date_field} not found")
            filter_year = date_col_info[0]["type"] not in ['timestamp','date']
            if filter_year and date_col_info[0]["type"] == 'text':
                # See if year can be filtered by YYYY-MM-DD 
                dates = [x[self.date_field] for x in sample_data['result']['records']]
                p = re.compile(r'^20\d{2}\-\d{2}\-\d{2}')
                if all([p.search(x) for x in dates]):
                    filter_year = False
                    # Identify time format
                    times = [p.sub('', x) for x in dates]
                    if len(times[0])>0:
                        if times[0][0]==' ':
                            times = [x[1:] for x in times]
                        else:
                            raise ValueError(f"Dates in {self.date_field} are text (not date) values and have unknown format (i.e. {dates[0]})")
                        
                        if all([re.search(r'^\d{2}:\d{2}:\d{2}$',x) for x in times]):
                            datetime_format = r'%Y-%m-%d %H:%M:%S'
                        elif all(m:=[re.search(r'^\d{2}:\d{2}:\d{2}\+(\d{2})$',x) for x in times]):
                            utc_offsets = [x.groups(1)[0] for x in m]
                            if all([x==utc_offsets[0] for x in utc_offsets]):
                                datetime_format = r'%Y-%m-%d %H:%M:%S+' + utc_offsets[0]
                            else:
                                raise ValueError(f"Dates in {self.date_field} are text (not date) values and have varying UTC offset")
                        else:
                            raise ValueError(f"Dates in {self.date_field} are text (not date) values and have unknown format (i.e. {dates[0]})")

            if filter_year:
                start_date, stop_date = _process_date(year, date_field=self.date_field, force_year=True)
                where = '('
                for y in range(int(start_date),int(stop_date)+1):
                    # %25 is % wildcard symbol
                    where+='"' + self.date_field + '"' + rf" LIKE '%{y}%' OR "
                where = where[:-4] + ')'
            else:
                self._sort_by_date = True
                start_date, stop_date = _process_date(year, date_field=self.date_field, datetime_format=datetime_format)
                where = f"""("{self.date_field}" >= '{start_date}' AND "{self.date_field}" <= '{stop_date}')"""
        else:
            where = None

        if opt_filter:
            where = where if where else ""
            if not isinstance(opt_filter, list):
                opt_filter = [opt_filter]

            andStr = " AND "
            for filt in opt_filter:
                where += andStr + filt

            if where[0:len(andStr)] == andStr:
                where = where[len(andStr):]

        return where

    
    def load(self, year=None, nrows=None, offset=0, *, pbar=True, opt_filter=None, select=None, output_type=None, sortby='_id', 
             format_date=True, **kwargs):
        '''Download table to pandas or geopandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the table
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
        opt_filter : str
            (Optional) Additional filter to apply to data (beyond any date filter specified by self.date_field and year)
        select : str
            (Optional) select statement to REST API. Default '*' (all fields)
        output_type : str
            (Optional) Data type for the output. Allowable values: 'DataFrame' and 'set'. Default: DataFrame
        sortby : str
            (Optional) Columns to sort by. Default: '_id'
        format_date : bool, optional
            If True, known date columns (based on presence of date_field in datasets table or data type information provided by dataset owner) will be automatically formatted
            to be pandas datetimes (or pandas Period in rare cases), by default True
            
        Returns
        -------
        pandas or geopandas DataFrame
            DataFrame containing downloaded
        '''

        data = self.__request(count=100)
        date_cols = [x['id'] for x in data['result']["fields"] if x["type"] in ['timestamp','date']]
        
        if self._last_count is not None and self._last_count[0]==year and self._last_count[1]==opt_filter:
            record_count = self._last_count[2]
            where_query = self._last_count[3]
        else:
            where_query = self.__construct_where(year, opt_filter, sample_data=data)
            json = self.__request(where=where_query, return_count=True, out_fields=select)
            record_count = json['result']['records'][0]['count']
            self._last_count = (year, opt_filter, record_count, where_query)

        record_count-=offset
        if record_count<=0:
            return pd.DataFrame()

        # Default fetch limit per https://docs.ckan.org/en/2.9/maintaining/datastore.html#ckanext.datastore.logic.action.datastore_search_sql
        batch_size = 32000
        nrows = nrows if nrows!=None and record_count>=nrows else record_count
        batch_size = nrows if nrows < batch_size else batch_size
        num_batches = ceil(nrows / batch_size)
            
        pbar = pbar and num_batches>1
        if pbar:
            bar = tqdm(desc=self.url, total=nrows, leave=False)

        if select:
            fields = select
        else:
            # CKAN includes a large _full_text and _id columns that are not useful
            # Get info on columns in order to exclude these columns from the returned data
            
            fields = [x['id'] for x in data['result']['fields'] if x['id'] not in ['_id','_full_text']]

        if sortby=="date":
            if self.date_field:
                sortby = self.date_field
            else:
                warnings.warn(DeprecationWarning('Date sorting was requested but no date field found. Results will not be sorted. '+
                                                     'This will result in an error in the next release (V1.0)'))
                sortby = "_id"
        elif not sortby:
            # order by_id guarantees data order remains the same when paging
            sortby = "_id"
            
        features = []
        for batch in range(num_batches):
            bs = batch_size if batch<num_batches-1 else nrows-batch*batch_size

            try:
                data = self.__request(where=where_query, offset=offset+batch*batch_size, count=bs, out_fields=fields, orderby=sortby)
                features.extend(data['result']['records'])

                if batch==0 and len(features)>0:
                    if len(features) not in [batch_size, nrows]:
                        raise ValueError(f"Number of rows is {len(features)} but is expected to be max rows to read {batch_size} or total number of rows {nrows}")
            except Exception as e:
                if len(e.args)>0 and "Error Code: 429" in e.args[0]:
                    raise OPD_TooManyRequestsError(self.url, *e.args, _url_error_msg.format(self.get_api_url()))
                else:
                    raise
            except:
                raise

            if pbar:
                bar.update(len(data['result']['records']))

        if pbar:
            bar.close()

        df = pd.DataFrame(features)
        if format_date:
            for col in date_cols:
                if col in df:
                    logger.debug(f"Column {col} had a data type of date. Converting values to datetime objects.")
                    df[col] = to_datetime(df[col])

        if len(df) > 0:
            if output_type=='set':
                return df.iloc[:,0].unique()
            else:
                return df
        else:
            return pd.DataFrame()

