import pandas as pd

from .data_loader import Data_Loader, filter_dataframe
from ..datetime_parser import to_datetime
from .. import log


logger = log.get_logger()

class Html(Data_Loader):
    """
    A class for accessing data from HTML download URLs

    Parameters
    ----------
    url : str
        URL
    date_field : str
        Name of the column that contains the date
    agency_field : str
        Name of column that contains agency names

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True, agency=None)
        Load data for query
    get_count(year=None, agency=None, force=False)
        Get number of records/rows generated by query
    get_years(force=False)
        Get years contained in data set
    """

    def __init__(self, url, date_field=None, agency_field=None):
        '''Create Html object

        Parameters
        ----------
        url : str
            URL for HTML data
        date_field : str
            (Optional) Name of the column that contains the date
        agency_field : str
                (Optional) Name of the column that contains the agency name (i.e. name of the police departments)
        '''
        
        self.url = url
        self.date_field = date_field
        self.agency_field = agency_field


    def isfile(self):
        '''Returns True to indicate that Html data is file-based

        Returns
        -------
        True
        '''
        return True


    def get_count(self, year=None, *,  agency=None, force=False, **kwargs):
        '''Get number of records for a Html data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        agency : str
            (Optional) Name of agency to filter for.
        force : bool
            (Optional) get_count for HLT file will only run if force=true. In many use cases, it will be more efficient to load the file and manually get the count.
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        logger.debug(f"Calculating row count for {self.url}")
        if self._last_count is not None and self._last_count[0] == (self.url, year, agency):
            logger.debug("Request matches previous count request. Returning saved count.")
            return self._last_count[1]
        
        if force:
            count = len(self.load(year=year, agency=agency))
        else:
            raise ValueError("Extracting the number of records for a single year of a HTML file requires reading the whole file in. In most cases, "+
                "running load() with a year argument to load in the data and manually finding the record count will be more "
                "efficient. If running get_count with a year argument is still desired, set force=True")
        
        self._last_count = ((self.url, year, agency), count)
        return count


    def load(self, year=None, nrows=None, offset=0, *, pbar=True, agency=None, format_date=True, **kwargs):
        '''Download HTML file to pandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested. None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the HTML table
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        pbar : bool
            (Optional) If true (default), a progress bar will be displayed
        agency : str
            (Optional) Name of the agency to filter for. None value returns data for all agencies.
        format_date : bool, optional
            If True, known date columns (based on presence of date_field in datasets table or data type information provided by dataset owner) will be automatically formatted
            to be pandas datetimes (or pandas Period in rare cases), by default True
            
        Returns
        -------
        pandas DataFrame
            DataFrame containing table imported from the HTML table
        '''

        if isinstance(nrows, float):
            nrows = int(nrows)
        
        logger.debug(f"Loading file from {self.url}")

        header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'}
        table = pd.read_html(self.url, storage_options=header)
                
        if len(table)>1:
            raise ValueError(f"More than 1 table found at {self.url}")

        
        table = table[0]

        if self.url=='https://www.openpolicedata.com/CedarLakeIN/Citations/2018Citations.php' and len(table)>994:
            # There is a known issue with 1 of the rows
            vals = table.loc[994,table.loc[994].notnull()].tolist()
            if len(vals)<len(table.columns):
                newvals = []
                for v in vals:
                    v = v.split(',')
                    v = [x.strip().strip('"').strip('\\') for x in v]
                    newvals.extend(v)

                if len(newvals)==len(table.columns):
                    table.loc[994,:] = newvals

        
        table = filter_dataframe(table, date_field=self.date_field, year_filter=year, 
            agency_field=self.agency_field, agency=agency, format_date=format_date)

        if offset>0:
            rows_limit = offset+nrows if nrows is not None and offset+nrows<len(table) else len(table)
            logger.debug(f"Extracting {rows_limit} rows starting at {offset}")
            table = table.iloc[offset:rows_limit].reset_index(drop=True)
        if nrows is not None and len(table)>nrows:
            logger.debug(f"Extracting the first {nrows} rows")
            table = table.head(nrows)

        return table

    def get_years(self, *, force=False, **kwargs):
        '''Get years contained in data set
        
        Parameters
        ----------
        force : bool
            (Optional) If false, an exception will be thrown. It may be more efficient to load the table and extract years manually
            
        Returns
        -------
        list
            list containing years in data set
        '''

        if not force:
            raise ValueError("Extracting the years of an HTML table requires reading the whole file in. In most cases, "+
                "running load() with no arguments to load in the whole HTML table and manually finding the years will be more "
                "efficient. If running get_years is still desired, set force=True")
        else:
            if self.date_field==None:
                raise ValueError("No date field provided to access year information")
            df = self.load()
            if self.date_field.lower()=="year":
                years = df[self.date_field].unique()
            else:
                date_col = to_datetime(df[self.date_field])
                years = list(date_col.dt.year.dropna().unique())
            years.sort()
            return [int(x) for x in years]
