import calendar
from datetime import date
from io import BytesIO
import openpyxl
import pandas as pd
from pandas.api.types import is_numeric_dtype
from pandas.api.types import is_datetime64_any_dtype as is_datetime
from rapidfuzz import fuzz
import re
import requests
import tempfile
import urllib
import warnings
from xlrd.biffh import XLRDError
from zipfile import ZipFile

from .data_loader import Data_Loader, UrlIoContextManager, _url_error_msg, get_legacy_session, filter_dataframe
from .. import dataset_id, log, httpio
from ..exceptions import OPD_DataUnavailableError

logger = log.get_logger()

class Excel(Data_Loader):
    """
    A class for accessing data from Excel download URLs

    Parameters
    ----------
    url : str
        URL
    date_field : str
        Name of the column that contains the date
    agency_field : str
        Name of column that contains agency names
    excel_file : Pandas ExcelFile
        Object for use in reading data

    Methods
    -------
    load(year=None, nrows=None, offset=0, pbar=True, agency=None)
        Load data for query
    get_count(year=None, agency=None, force=False)
        Get number of records/rows generated by query
    get_years(force=False)
        Get years contained in data set
    """

    def __init__(self, url, data_set=None, date_field=None, agency_field=None):
        '''Create Excel object

        Parameters
        ----------
        url : str
            URL for Excel data
        date_field : str
            (Optional) Name of the column that contains the date
        agency_field : str
            (Optional) Name of the column that contains the agency name (i.e. name of the police departments)
        data_set : str
            (Optional) Excel sheet to use or name of Excel file in zip file. If not provided, an error will be thrown when loading data if there is more than 1 sheet
        '''
        
        self.url = url.replace(' ','%20')
        self.date_field = date_field
        self.agency_field = agency_field

        is_zip = ".zip" in self.url
        self.sheet, file_in_zip = dataset_id.parse_excel_dataset(is_zip, data_set)
        
        try:
            if is_zip:
                with UrlIoContextManager(self.url) as fp:
                    with ZipFile(fp, 'r') as z:
                        if not file_in_zip:
                            if len(z.namelist())>1:
                                raise ValueError(f"More than one file found in zip file at {self.url}. One file must be specified if there is more than one file.")
                            else:
                                file_in_zip = z.namelist()[0]
                        elif file_in_zip not in z.namelist():
                            raise ValueError(f'Unable to find file {file_in_zip} in {self.url}')

                        self.excel_file = pd.ExcelFile(BytesIO(z.read(file_in_zip)))
            else:
                self.excel_file = pd.ExcelFile(self.url)
        except urllib.error.HTTPError as e:
            if str(e) in ["HTTP Error 406: Not Acceptable", 'HTTP Error 403: Forbidden']:
                # 406 error: https://stackoverflow.com/questions/34832970/http-error-406-not-acceptable-python-urllib2
                # File-like input for URL: https://stackoverflow.com/questions/57815780/how-can-i-directly-handle-excel-file-link-python/57815864#57815864
                headers = {'User-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A'}
                headers2 = {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0',
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
                    'Accept-Language': 'en-US,en;q=0.5',
                    # 'Accept-Encoding': 'gzip, deflate, br',
                    'DNT': '1',
                    'Connection': 'keep-alive',
                    'Upgrade-Insecure-Requests': '1',
                    'Sec-Fetch-Dest': 'document',
                    'Sec-Fetch-Mode': 'navigate',
                    'Sec-Fetch-Site': 'none',
                    'Sec-Fetch-User': '?1',
                }
                for k, h in enumerate([headers, headers2]):
                    r = requests.get(self.url, stream=True, headers=h)
                    try:
                        r.raise_for_status()
                        break
                    except:
                        if k==1:
                            raise
                file_like = BytesIO(r.content)
                self.excel_file = pd.ExcelFile(file_like)
            else:
                raise OPD_DataUnavailableError(*e.args, _url_error_msg.format(self.url))
        except urllib.error.URLError as e:
            if "[SSL: UNSAFE_LEGACY_RENEGOTIATION_DISABLED] unsafe legacy renegotiation disabled" in str(e.args[0]):
                with get_legacy_session() as session:
                    r = session.get(self.url)
                    
                r.raise_for_status()
                file_like = BytesIO(r.content)
                self.excel_file = pd.ExcelFile(file_like)
            elif isinstance(e.args[0],TimeoutError):
                raise OPD_DataUnavailableError(*e.args, _url_error_msg.format(self.url))
            else:
                raise e
        except XLRDError as e:
            if len(e.args)>0 and e.args[0] == "Workbook is encrypted" and \
                any([self.url.startswith(x) for x in ["http://www.rutlandcitypolice.com"]]):  # Only perform on known datasets to prevent security issues
                try:
                    import msoffcrypto
                except:
                    raise ImportError(f"{self.url} is encrypted. OpenPoliceData may be able to open it if msoffcrypto-tool " + 
                        "(https://pypi.org/project/msoffcrypto-tool/) is installed (pip install msoffcrypto-tool)")
                # Download file to temporary file
                r = requests.get(self.url)
                r.raise_for_status()
                # https://stackoverflow.com/questions/22789951/xlrd-error-workbook-is-encrypted-python-3-2-3
                fp_decrypt = tempfile.TemporaryFile(suffix=".xls")
                with tempfile.TemporaryFile() as fp:
                    fp.write(r.content)
                    fp.seek(0)

                    # Try and unencrypt workbook with magic password
                    wb_msoffcrypto_file = msoffcrypto.OfficeFile(fp)

                    # https://nakedsecurity.sophos.com/2013/04/11/password-excel-velvet-sweatshop/
                    wb_msoffcrypto_file.load_key(password='VelvetSweatshop')
                    wb_msoffcrypto_file.decrypt(fp_decrypt)

                fp_decrypt.seek(0)
                self.excel_file = pd.ExcelFile(fp_decrypt)
            else:
                raise
        except Exception as e:
            raise e


    def isfile(self):
        '''Returns True to indicate that Excel data is file-based

        Returns
        -------
        True
        '''
        return True


    def get_count(self, year=None, *,  agency=None, force=False, _first_time=True, **kwargs):
        '''Get number of records for a Excel data request
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested.  None value returns data for all years.
        agency : str
            (Optional) Name of agency to filter for.
        force : bool
            (Optional) get_count for Excel file will only run if force=true. In many use cases, it will be more efficient to load the file and manually get the count.
            
        Returns
        -------
        int
            Record count or number of rows in data request
        '''

        logger.debug(f"Calculating row count for {self.url}")
        if self._last_count is not None and self._last_count[0]==(self.url, year, agency):
            logger.debug("Request matches previous count request. Returning saved count.")
            return self._last_count[1]
        elif force:
            count = len(self.load(year=year, agency=agency, _first_time=_first_time))
            self._last_count = ((self.url, year, agency), count)
            return count
        else:
            raise ValueError("Extracting the number of records for an Excel file requires reading the whole file in. In most cases, "+
                "running load() to load in the data and manually finding the record count will be more "
                "efficient. If running get_count with a year argument is still desired, set force=True")


    def __get_sheets(self):
        names = self.excel_file.sheet_names
        if sum([x.isdigit() for x in names]) / len(names) > 0.75 and len(names)>1:
            logger.debug("Different years of data may be stored in separate Excel sheets. Evaluating...")
            possible_years = [int(x) for x in names if x.isdigit()]
            year_names = [x for x in names if x.isdigit()]

            if self.url=='https://www.arcgis.com/sharing/rest/content/items/73672aa470da4095a88fcac074ee00e6/data':
                # This is the Louisville OIS dataset. The sheets for years before 2011 are empty
                year_names = [x for x,y in zip(year_names, possible_years) if y>2010]
                possible_years = [x for x in possible_years if x>2010]
            
            min_year = min(possible_years)

            if min_year < 2000 or min_year > date.today().year:
                raise ValueError(f"Sheet name {min_year} is not recognized as a year in {self.url}")

            year_dict = {}
            years_found = True
            for i in range(len(possible_years)):
                year = min_year + i
                k = [k for k,x in enumerate(possible_years) if x==year]
                if len(k)==1:
                    logger.debug(f"Identified likely year sheet: {year_names[k[0]]}")
                    year_dict[year] = year_names[k[0]]
                elif len(k)==0:
                    # Check for typo
                    m = [y for y in year_names if {x for x in str(year)}=={x for x in y} and y not in year_dict.values()]
                    if len(m)!=1:
                        raise ValueError("Unable to parse sheet names")
                    year_dict[year] = m[0]
                else:
                    raise ValueError("Unable to parse sheet names")

            if years_found:
                logger.debug("Treating Excel file as different years stored in separate Excel sheets.")
                return year_dict, True

        return names, False


    def load(self, year=None, nrows=None, offset=0, *, agency=None, format_date=True, _first_time=True, **kwargs):
        '''Download Excel file to pandas DataFrame
        
        Parameters
        ----------
        year : int, list
            (Optional) Either the year or the year range [first_year, last_year] for the data that is being requested. None value returns data for all years.
        nrows : int
            (Optional) Only returns the first nrows rows of the Excel
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 to return records starting from the first.
        agency : str
            (Optional) Name of the agency to filter for. None value returns data for all agencies.
        format_date : bool, optional
            If True, known date columns (based on presence of date_field in datasets table or data type information provided by dataset owner) will be automatically formatted
            to be pandas datetimes (or pandas Period in rare cases), by default True
            
        Returns
        -------
        pandas DataFrame
            DataFrame containing table imported from Excel file

        Note: Older Excel files (.xls) and OpenDocument file formats (.odf, .ods, .odt) are not supported. Please submit an issue if this is needed.
        '''

        logger.debug(f"Loading file from {self.url}")
        nrows_read = offset+nrows if nrows is not None else None
        sheets, has_year_sheets = self.__get_sheets()

        if has_year_sheets:
            if year==None:
                year = list(sheets.keys())
                year.sort()
                year = [year[0], year[-1]]
            if not isinstance(year, list):
                year = [year, year]

            table = pd.DataFrame()
            cols_added = 0
            for y in range(year[0], year[1]+1):
                if y in sheets:
                    logger.debug(f"Loading data from sheet {sheets[y]}")
                    with warnings.catch_warnings():
                        warnings.filterwarnings("ignore", category=UserWarning, message='Data validation extension is not supported')
                        df = pd.read_excel(self.excel_file, nrows=nrows_read, sheet_name=sheets[y])

                    df = self.__clean(df, sheets[y], has_year_sheets)

                    if len(table)==0:
                        table = df
                        col_matches = [[k] for k in range(len(df.columns))]
                    else:
                        if not df.columns.equals(table.columns):
                            # Conditional for preventing column names from being too different
                            if len(df.columns)+cols_added == len(table.columns) and \
                                (df.columns == table.columns[:len(df.columns)]).sum()>=len(df.columns)-3-cols_added:
                                # Try to find a typo
                                for m in [j for j in range(len(df.columns)) if table.columns[j]!=df.columns[j]]:
                                    for k in col_matches[m]:
                                        if table.columns[k]==df.columns[m]:
                                            break

                                        if fuzz.ratio(table.columns[k], df.columns[m]) > 80 or \
                                            fuzz.token_sort_ratio(table.columns[k], df.columns[m])>90:
                                            warnings.warn(f"Identified difference in column names when combining sheets {sheets[y-1]} and {sheets[y]}. " + 
                                                f"Column names are '{table.columns[k]}' and '{df.columns[m]}'. This appears to be a typo. " + 
                                                f"These columns are assumed to be the same and will be combined as column '{table.columns[k]}'")
                                            df.columns = [table.columns[k] if j==m else df.columns[j] for j in range(len(df.columns))]
                                            break
                                    else:
                                        warnings.warn(f"Column '{table.columns[m]}' in current DataFrame does not match '{df.columns[m]}' in new DataFrame. "+ 
                                            "When they are concatenated, both columns will be included.")
                                        col_matches[m].append(len(table.columns))
                                        cols_added+=1
                                        # raise ValueError(f"Column {table.columns[k]} in table does not match {df.columns[k]} in df")
                            else:
                                raise ValueError("Columns don't match")
                        logger.debug("Concatenating data from multiple year sheets")
                        table = pd.concat([table, df], ignore_index=True)

                    if nrows_read!=None and len(table)>=nrows_read:
                        break
        else:
            dfs = []
            for s in (self.sheet if self.sheet else [None]):
                if isinstance(s,str):
                    s = s.strip()
                    if '*' in s:
                        p = s.replace('*','.*')
                        s = [x for x in sheets if re.search(p,x)]
                        assert len(s)==1
                        s = s[0]
                
                self.__check_sheet(s, sheets)
                sheet_name = 0 if s is None else s
                logger.debug(f"Loading sheet: {sheet_name}")
                table = pd.read_excel(self.excel_file, nrows=nrows_read, sheet_name=sheet_name)
                dfs.append(table)
            table = pd.concat(dfs, ignore_index=True)

            if _first_time and \
                self.url=='https://data-openjustice.doj.ca.gov/sites/default/files/dataset/2023-12/RIPA-Stop-Data-2022.zip':
                # According to https://data-openjustice.doj.ca.gov/sites/default/files/dataset/2024-01/RIPA Dataset Read Me 2022.pdf,
                # cases need to be added in that did not originally upload
                with httpio.open(self.url) as fp:
                    with ZipFile(fp) as z:
                        df = pd.read_excel(BytesIO(z.read('12312022 Supplement RIPA SD.xlsx')))

                df = df[df['AGENCY_NAME'].isin(table['AGENCY_NAME'].unique())]
                if len(df)>0:
                    table = pd.concat([table,df], ignore_index=True)

            table = self.__clean(table)

        # Check for empty rows at the bottom
        num_empty = table.isnull().sum(axis=1)
        empty_rows = num_empty==len(table.columns)
        if empty_rows.any():
            # Check if all rows after first empty row are empty or almost empty
            empty_rows = empty_rows[empty_rows]
            num_empty = num_empty.loc[empty_rows.index[0]:]
            if ((num_empty / len(table.columns)) > 0.75).all():
                logger.debug(f"Detected empty rows at the bottom of the table. Keeping the first {empty_rows.index[0]} rows")
                table = table.head(empty_rows.index[0])

        # Clean up column names
        table.columns = [x.strip() if isinstance(x, str) else x for x in table.columns]

        table = filter_dataframe(table, date_field=self.date_field, year_filter=year, 
            agency_field=self.agency_field, agency=agency, format_date=format_date)
        
        if offset>0:
            rows_limit = nrows_read if nrows_read is not None and nrows_read<len(table) else len(table)
            logger.debug(f"Extracting {rows_limit} rows starting at {offset}")
            table = table.iloc[offset:rows_limit].reset_index(drop=True)
        if nrows is not None and len(table)>nrows:
            logger.debug(f"Extracting the first {nrows} rows")
            table = table.head(nrows)

        return table


    def __check_sheet(self, cur_sheet, sheets):
        if cur_sheet is not None and cur_sheet not in sheets:
            raise ValueError(f"Sheet {cur_sheet} not found in Excel file at {self.url}")


    def __find_repeated_columns_names(self, df, first_col_row, last_col_row, sheet):
        # Look for rows that are just the column names to find if there are multiple tables in the sheet
        not_col_names = df.apply(lambda x: not all([y==first_col_row[k] or (pd.isnull(y) and pd.isnull(first_col_row[k])) for k,y in enumerate(x)]), axis=1)

        if not_col_names.all():
            return df
            
        if last_col_row:
            if self.url != 'https://cdn.muckrock.com/outbound_request_attachments/OmahaPoliceDepartment/87672/OIS202010-2019202.xlsx':
                raise ValueError(f"Currently, this technique is tailored to the Omaha dataset not {self.url}. Please report this error.")
            nextrows = [x+1 for x in not_col_names.index[~not_col_names]]
            nextrow_check = df.loc[nextrows].apply(lambda x: all([y.lower()==last_col_row[k].lower() or (pd.isnull(y) and pd.isnull(last_col_row[k])) for k,y in enumerate(x)]), axis=1)
            if not nextrow_check.all():
                raise ValueError("Unable to match all rows of multi-row header")
            not_col_names.loc[nextrows] = False

            df = df[not_col_names]

            # Check columns that are above column headers
            header_rows = [k for k,x in not_col_names.items() if not x]
            rem_rows = []
            rem_patterns = {}
            for x in header_rows:
                for idx in range(x-1, -1, -1):
                    if idx in rem_rows:
                        break
                    if idx not in df.index:
                        continue

                    # Look for cases where most columns are NaN and at least the first N columns are NaN
                    notnulls = df.loc[idx].notnull()
                    if notnulls.sum()==0:
                        rem_rows.append(idx)
                    elif notnulls.sum()==1:
                        if notnulls.iloc[:2].any():
                            rem_rows.append(idx)
                        else:
                            # This is a column that likely should have been merged with an above column
                            break
                    elif notnulls.sum()==2 or (notnulls.sum()==3 and notnulls.iloc[0] and isinstance(df.loc[idx].iloc[0], str) and df.loc[idx].iloc[0][0]=='*'):
                        # This is expected to be a total for some category so there should be a category name followed by a number
                        # 2 cells should be adjacent when accounting for cell merging
                        all_col_idx = [k for k,x in enumerate(notnulls) if x]
                        # Ignore not indicated by asterisk
                        col_idx = all_col_idx[1:] if len(all_col_idx)==3 else all_col_idx

                        if not isinstance(df.loc[idx].iloc[col_idx[0]],str):
                            raise ValueError(f"{df.loc[idx].iloc[col_idx[0]]} was expected to be a label")
                        
                        if not isinstance(df.loc[idx].iloc[col_idx[1]],int):
                            raise ValueError(f"{df.loc[idx].iloc[col_idx[1]]} was expected to be a count")

                        if col_idx[1]-col_idx[0] >1:
                            for x in sheet.merged_cells.ranges:
                                # Loop over merged cells to find the ones corresponding to non-null values in new_cols
                                cells = [y for y in x.cells]
                                if cells[0][0] == idx+2 and cells[0][1]==col_idx[0]+1: # +2 accounts for 1st row being the column names and zero indexing
                                    if cells[-1][1]==col_idx[1]:
                                        # Merged cell is adjacent to number
                                        break    
                            else:
                                raise ValueError("Unable to find expected merged cell")
                            
                        for k in all_col_idx:
                            if isinstance(df.loc[idx].iloc[k],str):
                                pattern = "".join([r'\d' if x.isdigit() else x for x in df.loc[idx].iloc[k]])
                                pattern = pattern.replace('*',r'\*')
                            elif isinstance(df.loc[idx].iloc[k],int):
                                pattern = r'\d+'
                            else:
                                raise ValueError(f"Unknown value {df.loc[idx].iloc[k]}")
                            if k not in rem_patterns:
                                rem_patterns[k] = set()
                            rem_patterns[k].add(pattern)
                        rem_rows.append(idx)
                    else:
                        break
            df = df.drop(index=rem_rows)
            rem_rows = []
            for k in df.index:
                idx = [j for j,x in enumerate(df.loc[k].notnull()) if x]
                if len(idx)>0 and all(x in rem_patterns for x in idx):
                    for i in idx:
                        if not any([re.search(x, str(df.loc[k].iloc[i])) for x in rem_patterns[i]]):
                            break
                    else:  # All rows match a pattern
                        rem_rows.append(k)

            df = df.drop(index=rem_rows)
        else:
            df = df[not_col_names]

        # Look for gaps between tables and/or tables that don't contain any data (including ones with a row that just says there is no data)
        df = df[df.iloc[:,2:].notnull().any(axis=1)]

        return df


    def __find_column_names(self, df, sheet_name):
        first_col_row = None
        sheet = None

        # Check if the entire column is null for any unnamed columns
        unnamed_cols = [x for x in df.columns if pd.isnull(x) or 'Unnamed' in x]
        delete_cols_tf = df[unnamed_cols].apply(lambda x: [pd.isnull(y) or (isinstance(y,str) and len(y.strip())==0) for y in x]).mean()==1  # Null or empty string
        delete_cols = delete_cols_tf.index[delete_cols_tf]
        if len(delete_cols)>0 and len(delete_cols)!=len(delete_cols_tf):
            m = (~delete_cols_tf).sum()
            if len(delete_cols_tf)<15 or (~delete_cols_tf[-m:]).sum()!=m>2:
                raise NotImplementedError(f'Unable to parse columns in {self.url}')
            # Only non-null unnamed columns are far to the right of the last named column
            delete_cols_tf[-m:] = True
            delete_cols = delete_cols_tf.index[delete_cols_tf]
        elif len(unnamed_cols)>0 and len(delete_cols)==0 and all(['Unnamed' in x for x in df.columns[-len(unnamed_cols):]]) and \
            (df[df.columns[-len(unnamed_cols):]].notnull().sum()<=1).all():
            # All unnamed columns have 0 or 1 non-null value
            delete_cols = unnamed_cols

        df = df.drop(columns=delete_cols).reset_index(drop=True)

        # Remove columns that will be deleted from unnamed_cols
        unnamed_cols = [x for x in unnamed_cols if x not in delete_cols]
        if len(unnamed_cols)==0:
            return df, df.columns, None, sheet  # All unnamed columns have been dealt with
        
        if len(delete_cols)>0 and len(unnamed_cols)>0:
            raise NotImplementedError(f"Unable to parse columns in {self.url}")
        
        # Find 1st unnamed column
        for idx_unnamed, x in enumerate(df.columns):
            if 'Unnamed' in x:
                break
        
        if idx_unnamed==0 and len(unnamed_cols)==1 and all(x==k+1 for k,x in enumerate(df[unnamed_cols[0]])):
            # First column is just row numbers
            df = df.iloc[:,1:]
        elif idx_unnamed < 2 and all(['Unnamed' in x for x in df.columns[idx_unnamed:]]):
            # First row is likely just some information about the table
            # First columns are likely in first row of data
            col_row = 0
            new_cols = [x for x in df.iloc[0]]
            while not all([isinstance(x, str) or pd.isnull(x) for x in new_cols]) or \
                all([pd.isnull(x) for x in new_cols[1:]]) or \
                ((ct:=sum(pd.notnull(x) for x in new_cols))<=2 and ct/len(new_cols)<0.4):  # Most column names are null
                col_row+=1
                new_cols = [x for x in df.iloc[col_row]]
            
            new_cols = [x.strip() if isinstance(x,str) else x for x in new_cols]
            first_col_row = new_cols.copy()
            last_col_row = None
            if sum([pd.notnull(x) for x in new_cols]) / len(new_cols) < 0.2 and \
                df.iloc[col_row+1].apply(lambda x: isinstance(x,str)).all():  # Most columns are null. Check if the next rows is all strings
                # There are likely multiple rows of columns
                r = requests.get(self.url)
                r.raise_for_status()
                wb = openpyxl.load_workbook(BytesIO(r.content))
                if sheet_name:
                    sheet = wb[sheet_name]
                else:
                    sheet = wb.active

                merge_found = False
                for x in sheet.merged_cells.ranges:
                    # Loop over merged cells to find the ones corresponding to non-null values in new_cols
                    cells = [y for y in x.cells]
                    if cells[0][0] == col_row+2: # Accounts for 1st row being the column names and zero indexing
                        if pd.notnull(new_cols[cells[0][1]-1]):
                            # Expect rest of column names in range to be null
                            if not all([pd.isnull(x) for x in new_cols[cells[0][1]:cells[-1][1]]]):
                                raise ValueError("Merged cell has multiple non-empty columns")
                            
                            merge_found = True
                            for k in range(cells[0][1], cells[-1][1]):
                                new_cols[k] = new_cols[cells[0][1]-1]
                        else:
                            raise ValueError("Merged cell does not contain string")

                if not merge_found:
                    raise ValueError("Unable to find expected merged cell")
                
                last_col_row = list(df.iloc[col_row+1])
                new_cols = [x+" "+y if pd.notnull(x) else y for x,y in zip(new_cols, df.iloc[col_row+1])]
                col_row+=1
            
            logger.debug(f"Detected that first row does not contain column headers: {df.columns}")
            logger.debug(f"Making the  second row the column headers: {new_cols}")
            df.columns = new_cols
            df = df.iloc[col_row+1:]
        else:
            # This is likely the result of column headers that span multiple rows
            # with some headers in the first row in merged Excel columns that will not 
            # be merged in the pandas columns

            # Validate that columns match expected pattern
            for k, c in enumerate(df.columns):
                if 'Unnamed' in c:
                    if pd.isnull(df.loc[0,c]): # If the column name is empty, the first row should be part of the column name
                        raise ValueError(f"Unexpected condition in column {c} where first row is null for url {self.url}")
                elif pd.notnull(df.loc[0,c]):  # Both column name and 1st row have value. Column name is expected to be a merged column
                    if k == len(df.columns) or 'Unnamed' not in df.columns[k+1]:
                        raise ValueError(f"Unexpected column pattern with columns {df.columns} and first row {df.iloc[0]} for url {self.url}")
                    
            # Merge 1st row with columns
            new_cols = []
            addon = ''
            for c in df.columns:
                if pd.isnull(df.loc[0,c]):
                    addon = ''
                    new_cols.append(c)
                elif c.lower().endswith('info'):
                    addon = re.sub(r'[Ii]nfo', '', c).strip() + ' '
                    new_cols.append(addon + df.loc[0,c])
                else:
                    new_cols.append(addon + df.loc[0,c])

            df = df.copy() # Avoids any warnings from pandas
            df.columns = new_cols
            df = df.iloc[1:]

        if first_col_row is None:
            first_col_row = df.columns
            last_col_row = None

        return df, first_col_row, last_col_row, sheet
    
    def __clean_blank_space(self, df):
        all_months = list(calendar.month_name)[1:]
        all_months.extend(list(calendar.month_abbr)[1:])
        all_months = [x.lower() for x in all_months]
        # Remove titles for multiple tables on 1 sheet. They are often a year + a description in the 1st column.
        possible_title_rows = (df.notnull().sum(axis=1)==1) & df.iloc[:,0].notnull()
        for idx, val in df[possible_title_rows].iloc[:,0].items():
            if not isinstance(val, str) or not (m:=re.search(r'^20\d\d\s([a-z\s]+$)', val, re.IGNORECASE)) or \
                m.group(1).lower() in all_months:  # Ensure that string matches pattern and string after year is not a month
                possible_title_rows.loc[idx] = False
        df = df[~possible_title_rows]

        # Look for null rows followed by sparsely populated rows
        nulls = df.isnull()
        nullrows = nulls.all(axis=1)
        df = df[~nullrows]
        if nullrows.any() and nullrows[nullrows].index[-1]:
            last_null = nullrows[nullrows].index[-1]
            # Look for mostly null rows
            if last_null<df.index[-1] and all(df.loc[last_null+1:].notnull().sum(axis=1)<=2):
                # Check if they all start with an asterisk indicating some sort of note
                if df.loc[last_null+1:].apply(lambda x: x.apply(lambda y: pd.isnull(y) or (isinstance(y,str) and y.startswith('*')))).all().all():
                    df = df.drop(index=range(last_null+1,df.index[-1]+1), errors='ignore')
        elif (all_null_except1:=nulls.iloc[:,1:].all(axis=1)).any():
            # Find cases where the only non-null in the last N rows is the 1st column and the first column appears to just be some iterating number
            last_false = all_null_except1[~all_null_except1].index[-1]
            if last_false!=df.index[-1]:
                # The end is all_null_except1
                testvals = df.loc[last_false:, df.columns[0]]
                if testvals.apply(lambda x: isinstance(x,int)).all():
                    diff = testvals.diff().iloc[1:] # Remove first one which is the last good value
                    if (diff<=1).all() and (diff!=1).sum()<=1:  # Number appears to generally be iterating
                        df = df.drop(index=[x for x in df.index if x>last_false])

        return df


    def __clean(self, df, sheet_name=None, has_year_sheets=False):
        if any([(pd.isnull(x) or "Unnamed" in x) for x in df.columns]):
            # At least 1 column name was empty.
            # This can occur when there are empty columns or when there is 
            # background information or blank rows above the column rows
            df, first_col_row, last_col_row, sheet = self.__find_column_names(df, sheet_name)
            df = self.__find_repeated_columns_names(df, first_col_row, last_col_row, sheet)
            df = self.__clean_blank_space(df)
            df = df.reset_index(drop=True)

        if has_year_sheets:
            if sheet_name and 'Year' not in df:
                df['Year'] = int(sheet_name)
            if sheet_name and sheet_name.isdigit() and 1990 < int(sheet_name) < 2100 and \
                'Month' in df and 'Day' in df:
                # Rearrange columns
                col_order = []
                year_added = False
                day_found = False
                for c in df.columns:
                    if not year_added and c=='Month':
                        if day_found:
                            col_order.append(c)
                            col_order.append('Year')
                        else:
                            col_order.append('Year')
                            col_order.append(c)
                        year_added = True
                    elif c=='Day':
                        day_found = True
                        col_order.append(c)
                    elif c!='Year':
                        col_order.append(c)
                df = df[col_order]

        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=RuntimeWarning)
            df = df.convert_dtypes()

        # Check for 1st column being row numbers
        if pd.isnull(df.columns[0]) and list(df.iloc[:,0]) in [[k+1 for k in range(0, len(df))], [k for k in range(0, len(df))]]:
            logger.debug("Removing the 1st column which is just the row number")
            df = df.iloc[:, 1:]

        return df
        

    def get_years(self, *, force=False, **kwargs):
        '''Get years contained in data set
        
        Parameters
        ----------
        force : bool
            (Optional) If false, an exception will be thrown. It may be more efficient to load the table and extract years manually
            
        Returns
        -------
        list
            list containing years in data set
        '''

        sheets, has_year_sheets = self.__get_sheets()

        if has_year_sheets:
            years = list(sheets.keys())
            years.sort()
            return list(sheets.keys())
        if not force:
            raise ValueError("Extracting the years of a Excel file requires reading the whole file in. In most cases, "+
                "running load() with no arguments to load in the whole CSV file and manually finding the years will be more "
                "efficient. If running get_years is still desired, set force=True")
        else:
            if pd.isnull(self.date_field):
                raise ValueError("No date field provided to access year information")
            df = self.load()
            if is_datetime(df[self.date_field]):
                years = list(df[self.date_field].dt.year.dropna().unique())
            elif is_numeric_dtype(df[self.date_field]):
                years = list(df[self.date_field].dropna().unique())
            else:
                raise TypeError("Unknown date column format")
            return [int(x) for x in years]
