from __future__ import annotations
import copy
import numbers
import os.path as path
import pandas as pd
from datetime import datetime
from dateutil.parser._parser import ParserError
import logging
from packaging import version
import re
from collections.abc import Iterator
import sys
try:
    from typing import Literal
except:
    from typing_extensions import Literal
import warnings

from . import data_loaders
from . import datasets
from . import __version__
from . import preproc
from . import defs
from . import exceptions
from .deprecated._decorators import deprecated, input_swap
from .datetime_parser import to_datetime

class Table:
    """
    A class that contains a DataFrame for a dataset along with meta information

    Parameters
    ----------
    details : pandas Series
        Series containing information about the dataset
    state : str
        Name of state where agencies in table are
    source_name : str
        Name of source
    agency : str
        Name of agency
    table_type : TableType enum
        Type of data contained in table
    year : int, list, MULTI
        Indicates years contained in table
    description : str
        Description of data source
    url : str
        Direct URL for accessing the table data
    source_url : str
        Main URL to view information about the data and sometimes view the data. This site contains a link to download the data.
    readme : str
        URL for data dictionary if it exists
    urls : dict
        Dictionary of url, source_url, and readme
    table : pandas of geopandasDataFrame
        Data accessed from source

    Methods
    -------
    expand(person_type="subject",
           mismatch='error')
        Expand rows that contain information on multiple subjects or officers into multiple rows
    get_csv_filename()
        Get default name of CSV file
    get_age_col(role='SUBJECT')
        Get age column (if found) after running standardize
    get_gender_col(role='SUBJECT')
        Get gender column (if found) after running standardize
    get_race_col(role='SUBJECT')
        Get race column (if found) after running standardize
    get_transform_map(orig=None, 
                      new=None, 
                      minimize=False)
        Get documentation of standardizations performed after running standardize
    merge(right, 
        std_id=False,
        keep_raw=True,
        on=None, 
        how='inner', 
        left_on=None, 
        right_on=None, 
        **kwargs)
        Merge 2 related Table objects together
    standardize(race_cats = None,
        agg_race_cat = False,
        eth_cats = None,
        gender_cats = None,
        keep_raw=True,
        known_cols = None,
        verbose = False,
        no_id = "keep",
        race_eth_combo = "merge",
        merge_date_time=True,
        empty_time = "NaT"
        Standardize column names and data values for loaded data in self.table
    to_csv(output_dir=None, filename=None)
        Convert table to CSV file
    """

    details: str= None
    state: str = None
    source_name: str = None
    agency: str = None
    table_type: defs.TableType = None
    year: int | str | list[int] = None
    description: str = None
    url: str = None
    source_url: str = None
    readme: str | None = None
    urls: dict = None

    # Data
    table: pd.DataFrame | None = None

    # Whether data is standardized
    is_std = False

    # From source
    _data_type: str = None
    _dataset_id: str | None = None
    date_field: str | None = None
    agency_field: str | None = None
    __transforms = None

    def __init__(self, 
        source: pd.DataFrame | pd.Series | dict, 
        table: pd.DataFrame | None = None, 
        year_filter: str | int | list[int] = None, 
        agency: str | None = None
        ) -> None:
        '''Construct Table object
        This is intended to be generated by the Source.load and Source.load_from_csv classes

        Parameters
        ----------
        source : pandas or geopandas Series
            Series containing information on the source
        table : pandas or geopandas 
            Name of state where agencies in table are
        '''
        if isinstance(source, pd.DataFrame):
            if len(source) == 0:
                raise LookupError("DataFrame is empty")
            elif len(source) > 1:
                raise LookupError("DataFrame has more than 1 row")

            source = source.iloc[0]

        self.details = source
        self.table = table

        self.state = source["State"]
        self.source_name = source["SourceName"]

        if agency != None:
            self.agency = agency
        else:
            self.agency = source["Agency"] if "Agency" in source else None

        try:
            self.table_type = defs.TableType(source["TableType"])  # Convert to Enum
        except:
            warnings.warn("{} is not a known table type in defs.TableType".format(source["TableType"]))
            self.table_type = source["TableType"]

        if year_filter != None:
            self.year = year_filter
        else:
            self.year = source["Year"]

        self.description = source["Description"] if "Description" in source else None
        self.url = source["URL"] if "URL" in source else None
        self._data_type = defs.DataType(source["DataType"]) if "DataType" in source else None  # Convert to Enum

        if "dataset_id" in source and not pd.isnull(source["dataset_id"]):
            self._dataset_id = source["dataset_id"]

        if "date_field" in source and not pd.isnull(source["date_field"]):
            self.date_field = source["date_field"]
        
        if "agency_field" in source and not pd.isnull(source["agency_field"]):
            self.agency_field = source["agency_field"]

        if "source_url" in source and not pd.isnull(source["source_url"]):
            self.source_url = source["source_url"]

        if "readme" in source and not pd.isnull(source["readme"]):
            self.readme = source["readme"]

        self.urls = {'source_url':self.source_url, 'readme':self.readme, 'data':self.url}


    def get_race_col(self, role:Literal['SUBJECT','OFFICER']='SUBJECT'):
        """Get name of race column

        Parameters
        ----------
        role : Literal[&#39;SUBJECT&#39;,&#39;OFFICER&#39;], optional
            Whether to return officer or subject column name, by default 'SUBJECT'

        Returns
        -------
        defs.columns
            Name of column
        """
        assert(role.upper() in ['OFFICER','SUBJECT'])
        if role.upper()=='OFFICER':
            if defs.columns.RE_GROUP_OFFICER in self.table:
                return defs.columns.RE_GROUP_OFFICER
            elif defs.columns.RE_GROUP_OFFICER_SUBJECT in self.table:
                return defs.columns.RE_GROUP_OFFICER_SUBJECT 
            else:
                return None
        else:
            if defs.columns.RE_GROUP_SUBJECT in self.table:
                return defs.columns.RE_GROUP_SUBJECT
            elif defs.columns.RE_GROUP_OFFICER_SUBJECT in self.table:
                return defs.columns.RE_GROUP_OFFICER_SUBJECT 
            else:
                return None
            

    def get_gender_col(self, role:Literal['SUBJECT','OFFICER']='SUBJECT'):
        """Get name of gender column

        Parameters
        ----------
        role : Literal[&#39;SUBJECT&#39;,&#39;OFFICER&#39;], optional
            Whether to return officer or subject column name, by default 'SUBJECT'

        Returns
        -------
        defs.columns
            Name of column
        """
        assert(role.upper() in ['OFFICER','SUBJECT'])
        if role.upper()=='OFFICER':
            if defs.columns.GENDER_OFFICER in self.table:
                return defs.columns.GENDER_OFFICER
            elif defs.columns.GENDER_OFFICER_SUBJECT in self.table:
                return defs.columns.GENDER_OFFICER_SUBJECT 
            else:
                return None
        else:
            if defs.columns.GENDER_SUBJECT in self.table:
                return defs.columns.GENDER_SUBJECT
            elif defs.columns.GENDER_OFFICER_SUBJECT in self.table:
                return defs.columns.GENDER_OFFICER_SUBJECT 
            else:
                return None

    def get_age_col(self, role:Literal['SUBJECT','OFFICER']='SUBJECT'):
        """Get name of age column

        Parameters
        ----------
        role : Literal[&#39;SUBJECT&#39;,&#39;OFFICER&#39;], optional
            Whether to return officer or subject column name, by default 'SUBJECT'

        Returns
        -------
        defs.columns
            Name of column
        """
        assert(role.upper() in ['OFFICER','SUBJECT'])
        if role.upper()=='OFFICER':
            if defs.columns.AGE_OFFICER in self.table:
                return defs.columns.AGE_OFFICER
            elif defs.columns.AGE_OFFICER_SUBJECT in self.table:
                return defs.columns.AGE_OFFICER_SUBJECT
            elif defs.columns.AGE_RANGE_OFFICER in self.table:
                return defs.columns.AGE_RANGE_OFFICER
            elif defs.columns.AGE_RANGE_OFFICER_SUBJECT in self.table:
                return defs.columns.AGE_RANGE_OFFICER_SUBJECT
            else:
                return None
        else:
            if defs.columns.AGE_SUBJECT in self.table:
                return defs.columns.AGE_SUBJECT
            elif defs.columns.AGE_OFFICER_SUBJECT in self.table:
                return defs.columns.AGE_OFFICER_SUBJECT
            elif defs.columns.AGE_RANGE_SUBJECT in self.table:
                return defs.columns.AGE_RANGE_SUBJECT
            elif defs.columns.AGE_RANGE_OFFICER_SUBJECT in self.table:
                return defs.columns.AGE_RANGE_OFFICER_SUBJECT
            else:
                return None


    def merge(self, 
              right: pd.DataFrame | pd.Series | Table, 
              std_id: bool = False,
              keep_raw: bool =True,
              on: str | pd.Index | None = None, 
              how: Literal['inner', 'outer', 'left', 'right', 'cross'] = 'inner', 
              left_on: str | pd.Index | None = None, 
              right_on: str | pd.Index | None = None, 
              **kwargs) -> Table:
        """Merge 2 Tables together or a Table with a pandas DataFrame or Series

        Parameters
        ----------
        right : pd.DataFrame | pd.Series | Table
            Table, DataFrame, or Series to merge
        std_id : bool, optional
            If True, name of incident ID column will be standardized in the merged table. by default False    
        keep_raw: bool, optional
            If std_id is True, determines whether to keep original column if incident ID column found
        on : str | pd.Index | None, optional
            Column(s) to join on. They must be found in both left and right.
            If None and left_on and right_on are None, merge will attempt to find an incident ID column to use. by default None
        how : Literal[&#39;inner&#39;, &#39;outer&#39;, &#39;left&#39;, &#39;right&#39;, &#39;cross&#39;], optional
            Type of merge to be performed. See pandas DataFrame.merge for details 
            (https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.merge.html), by default 'inner'
        left_on : str | pd.Index | None, optional
            Column(s) to join on in the left DataFrame. , by default None
        right_on : str | pd.Index | None, optional
            Column(s) to join on in the right DataFrame., by default None

        Returns
        -------
        Table
            Merged table
        """
        # Returns a new Table that is the merger of 2 tables
        if right_is_table:=isinstance(right, Table):
            df2 = right.table
        elif isinstance(right, pd.Series):
            df2 = right.to_frame()
        else:
            df2 = right

        if not self.is_std and std_id:
            raise ValueError("ID standardization is currently only possible for left tables where Table.standardize has been applied")

        use_id = not on and not left_on and not right_on
        df1 = self.table
        if use_id or std_id:
            id_col1, id_col2, df1_new, df2_new, mapping = preproc.find_id_column(self.table, df2, std_id, keep_raw)
            if use_id:
                if id_col1:
                    left_on = defs.columns.INCIDENT_ID if std_id else id_col1
                    right_on = defs.columns.INCIDENT_ID if std_id else id_col2
                else:
                    raise exceptions.AutoMergeError("Unable to automatically find ID that relates tables")
            if std_id and id_col1:
                df1 = df1_new
                df2 = df2_new
                self.__transforms.append(mapping)

        df = pd.merge(df1, df2, on=on, how=how, left_on=left_on, right_on=right_on, suffixes=(None, '_FromMerged'), **kwargs)
        new_table = copy.deepcopy(self)
        new_table.table = df
        if right_is_table and right.is_std:
            new_table._Table__transforms.extend(right.get_transform_map())
        for k,v in new_table.__dict__.items():
            if k not in ['table', '_Table__transforms']:
                setattr(new_table, k, [v, getattr(right, k) if right_is_table else None])

        return new_table



    def __repr__(self) -> str:
        skip = ["details", "table"]
        return ',\n'.join("%s: %s" % item for item in vars(self).items() if (item[0] not in skip and item[0][0] != "_"))

    def to_csv(self, 
               output_dir: str | None = None, 
               filename: str | None = None
               ) -> str:
        '''Export table to CSV file. Use default filename for data that will
        be reloaded as an openpolicedata Table object

        Parameters
        ----------
        output_dir - str
            (Optional) Output directory. Default: current directory
        filename - str
            (Optional) Filename. Default: Result of get_csv_filename()
        '''
        if filename == None:
            filename = self.get_csv_filename()
        if output_dir != None:
            filename = path.join(output_dir, filename)
        if not isinstance(self.table, pd.core.frame.DataFrame):
            raise ValueError("There is no table to save to CSV")

        self.table.to_csv(filename, index=False, errors="surrogateescape")

        return filename


    def get_csv_filename(self) -> str:
        '''Generate default filename based on table parameters

        Returns
        -------
        str
            Filename
        '''
        return get_csv_filename(self.state, self.source_name, self.agency, self.table_type, self.year)
    

    def get_transform_map(self, 
                          orig: str | None=None, 
                          new: str | None=None, 
                          minimize: bool=False):
        """Get details of standardization process if standardization has been run

        Parameters
        ----------
        minimize : bool, optional
            If True removes original value counts from output, by default False

        Returns
        -------
        Dictionary or list of dictionaries with column name and data value replacement information
        """
        
        result = copy.deepcopy(self.__transforms)
        if minimize:
            for r in result:
                del r.orig_value_counts
        
        if orig is not None or new is not None:
            for r in result:
                if (orig is None or r.orig_column_name==orig) and \
                    (new is None or r.new_column_name==new):
                    return r
            return None
        return result
    

    def expand(self, 
               person_type: Literal["subject", "officer"]="subject",
               mismatch: Literal['error','nan','splitsingle']='error'):
        """Expand demographics data into multiple row for datasets that have been standardized
         and that contain demographic and other data for more than one subject or officer per row.

        Parameters
        ----------
        person_type : Literal[&quot;subject&quot;, &quot;officer&quot;], optional
            Whether to expand subject or officer data, by default "subject"
        mismatch : Literal[&quot;error&quot;, &quot;nan&quot;, &quot;splitsingle&quot;], optional
            How to handle cases where the number of elements to be expanded is not equal across
            each column to expand. If 'error', an error will be thrown. If 'nan', columns that do not
            have enough values will be set to nan. If 'splitsingle', if the mismatched columns have a
            single value, that single value will be used for each resulting row. For example, if the
            the 2 columns of a row to be expanded are [ ['a',b'] , ['c'] ], there is a mismatch because
            the first column has 2 values and the 2nd has 1 when all columns to expand should have the
            same number. If mismatch='nan', the result will be [ 'a' , nan ] for the first column and 
            [ 'b', nan ] for the second column. If mismatch='splitsingle', the result will be 
            [ 'a' , 'c' ] for the first column and [ 'b', 'c' ] for the second column., by default "error"
        """
        
        if not self.is_std:
            raise ValueError("standardize must be run on a table to apply expand")
        
        is_subject = person_type.lower()=='subject'
        expand_cols = []
        for t in self.__transforms:
            expand = (is_subject and ("SUBJECT" in t.new_column_name or t.new_column_name==defs.columns.FATAL_SUBJECT)) or \
                        (not is_subject and "OFFICER" in t.new_column_name)
            if expand:
                if self.table[t.new_column_name].apply(lambda x: isinstance(x,dict)).any():
                    expand_cols.append(t.new_column_name)

        if len(expand_cols)>0:
            new_df = self.table.copy()
            for c in expand_cols:
                new_df[c] = new_df[c].apply(lambda x: x.values() if isinstance(x,dict) else x)

            try:
                self.table = new_df.explode(expand_cols, ignore_index=True)
            except AttributeError as e:
                # This is an issue with geopandas when exploding lists of columns
                if len(expand_cols)==1:
                    self.table = new_df.explode(expand_cols[0], ignore_index=True)
                else:
                    warnings.warn("Original table is a geopandas DataFrame, which has a known bug when expanding. "+
                                    "Converting to pandas DataFrame.")
                    self.table = pd.DataFrame(new_df).explode(expand_cols, ignore_index=True)
            except ValueError as e:
                mismatch = mismatch.lower()
                assert mismatch in ['error','nan','splitsingle']
                if mismatch!='error' and len(e.args)>0 and e.args[0]=='columns must have matching element counts':
                    dict_values = type({}.values())
                    num_vals = pd.DataFrame(0, index=new_df.index, columns=expand_cols)
                    # Try replacing length 1 arrays with their values
                    for c in expand_cols:
                        num_vals[c] = new_df[c].apply(lambda x: len(x) if isinstance(x, dict_values) else 1)
                    mismatches = num_vals.apply(lambda x: (x!=x.iloc[0]).any(), axis=1)
                    can_expand = True
                    for m in mismatches[mismatches].index:
                        max_count = num_vals.loc[m].max()
                        row = new_df.loc[m, expand_cols]
                        if mismatch=='nan':
                            fill = [pd.NA for _ in range(max_count)]
                            for k in row.index:
                                if num_vals.loc[m,k] < max_count:
                                    row[k] = fill
                        else:
                            for k in row.index:
                                if num_vals.loc[m,k] ==1:
                                    try:
                                        row[k] = [list(row[k])[0] for _ in range(max_count)]
                                    except:
                                        row[k] = [row[k] for _ in range(max_count)]
                                elif num_vals.loc[m,k]!=max_count:
                                    print("WARNING: Expansion is not possible with mismatch='splitsingle' due to 1 or more rows with "+
                                          "values of length greater than 1 and different lengths.")
                                    print(f"Example from index {m}:")
                                    print(new_df[expand_cols].loc[m])
                                    print("WARNING: Aborting expansion")
                                    can_expand = False
                                    break
                        new_df.loc[m,expand_cols] = row

                    null_locs = new_df[expand_cols].isnull().any(axis=1)
                    for m in null_locs[null_locs].index:
                        max_count = num_vals.loc[m].max()
                        row = new_df.loc[m, expand_cols]
                        if row.apply(lambda x: isinstance(x, dict_values)).any():
                            new_df.loc[m,expand_cols] = row.apply(lambda x: [x for _ in range(0,max_count)] if pd.isnull(x) else x)

                    if can_expand:
                        self.table = pd.DataFrame(new_df).explode(expand_cols, ignore_index=True)
                else:
                    raise
    

    def standardize(self, 
        race_cats: dict | str | None = None,
        agg_race_cat: bool = False,
        eth_cats: dict | None = None,
        gender_cats: dict | None = None,
        keep_raw: bool =True,
        known_cols: dict | None = None,
        verbose: bool | str = False,
        no_id: Literal["keep", "null", "error", "test"] = "keep",
        race_eth_combo: Literal[False, "merge", "concat"] = "merge",
        merge_date_time: bool =True,
        empty_time: Literal["nat", "ignore"] = "NaT"
    ):
        """Standardize column names and data values for loaded table data

        Parameters
        ----------
        race_cats : dict | str | None, optional
            Indicates data values to use for race standardization. If None, the dictionary returned by 
            opd.defs.get_race_cats() is used. If a dictionary, the keys of the dictionary must be a subset
            of the values returned by opd.defs.get_race_keys(). The corresponding values indicate, which
            value to use for each category. If race_cats is 'expand', the dictionary returned by 
            opd.defs.get_race_cats(expand=True) will be used. If race_cats is 'compact', the dictionary returned by 
            opd.defs.get_race_cats(compact=True) will be used.
        agg_race_cat : bool, optional
            If True, standardization of race will be more aggressive in converting raw values to standardized
            ones. For example, if agg_race_cat is False, standardization will not convert 'East African' to 
            the category for Black while it will if True, by default False
        eth_cats : dict | None, optional
            Indicates data values to use for ethnicity standardization. If None, the dictionary returned by 
            opd.defs.get_eth_cats() is used. If a dictionary, the keys of the dictionary must be a subset
            of the values returned by opd.defs.get_eth_keys(). The corresponding values indicate, which
            value to use for each category. If eth_cats is 'compact', the dictionary returned by 
            opd.defs.get_eth_cats(compact=True) will be used.
        gender_cats : dict | None, optional
            Indicates data values to use for gender standardization. If None, the dictionary returned by 
            opd.defs.get_gender_cats() is used. If a dictionary, the keys of the dictionary must be a subset
            of the values returned by opd.defs.get_gender_keys(). The corresponding values indicate, which
            value to use for each category. If gender_cats is 'compact', the dictionary returned by 
            opd.defs.get_gender_cats(compact=True) will be used. 
        keep_raw : bool, optional
            If False, raw columns that are standardized will be removed. If True, they will be kept and 
            renamed to indcate that they are the original raw columns, by default True
        known_cols : dict | None, optional
            Dictionary of known column mappings. If None, the dictionary defaults to 
            any known columns for this dataset ({opd.defs.columns.DATE:self.date_field, opd.defs.columns.AGENCY:self.agency_field}).
            If a dictionary, the keys of the dictionary must be available columns for standardization (defs.columns)
            and the values must be columns in the table.
        verbose : bool | str, optional
            If True, details of the standardization will be printed. If a filename, details of the standardization will
            be logged to that file., by default False
        no_id : Literal[&quot;keep&quot;, &quot;null&quot;, &quot;error&quot;], optional
            Determines how values that are not able to be standardized are handled during standardization of demographics:
            - 'keep' (default): Keep the original value
            - 'null': Replace with a null value
            - 'error': Throw an error
            , by default "keep"
        race_eth_combo : Literal[False, &quot;merge&quot;, &quot;concat&quot;], optional
            Indicates whether and how combine race and ethnicity columns. If False, race and ethnicity columns
            are not combined. If 'merge', the combined race and ethnicity column will be for Latino of all races 
            and all race categories will be for non-Latino only. Functionally, the combined race and ethnicity 
            column will be the ethnicity value if the ethnicity is Latino or unnown and the race otherwise. 
            If 'concat', race and ethnicity values will be concatenated in the combined race and ethnicity column, 
            by default "merge"
        merge_date_time : bool, optional
            If True, if standardized date and standardize time columns are identified, they will be merged into a 
            combined datetime column, by default True
        empty_time : Literal[&quot;nat&quot;, &quot;ignore&quot;], optional
            Indicates how null times are treated in the standardized datetime column (if `merge_date_time` is True). If 'NaT', the resulting 
            datetime is a null value (pd.NaT). If 'ignore', the resulting datetime will be the date value without a time., by default "NaT"

        Returns
        -------
        None
        """
        if len(self.table)==0:
            self.is_std = True
            self.__transforms = []
            return
        
        if not self.is_std:
            if known_cols is None:
                known_cols = {defs.columns.DATE:self.date_field, defs.columns.AGENCY:self.agency_field}

            race_cats = defs.get_race_cats() if race_cats is None else race_cats
            race_cats = defs.get_race_cats(expand=True) if race_cats=="expand" else race_cats
            race_cats = defs.get_race_cats(compact=True) if race_cats=="compact" else race_cats
            eth_cats = eth_cats if eth_cats is not None else defs.get_eth_cats()
            eth_cats = defs.get_eth_cats(compact=True) if eth_cats=="compact" else eth_cats
            gender_cats = gender_cats if gender_cats is not None else defs.get_gender_cats()
            gender_cats = defs.get_gender_cats(compact=True) if gender_cats=="compact" else gender_cats
            if verbose:
                logger = logging.getLogger("opd-std")
                log_level = logger.level
            if isinstance(verbose,str):
                # verbose is a filename
                fh = logging.FileHandler(verbose)
                logger.addHandler(fh)
                for handler in logger.handlers:
                    if handler.name == "main":
                        # Temporarily up level of stream handler so that only print to file
                        handler.setLevel(logging.WARNING)
            if verbose:
                # Set logger to info so log messages in preproc.standardize will be displayed
                logger.setLevel(logging.INFO)
            try:                    
                self.table, self.__transforms = preproc.standardize(self.table, self.table_type, self.year,
                    known_cols=known_cols, 
                    source_name=self.source_name,
                    state=self.state,
                    keep_raw=keep_raw,
                    agg_race_cat=agg_race_cat,
                    race_cats=race_cats,
                    eth_cats=eth_cats,
                    gender_cats=gender_cats,
                    no_id=no_id,
                    race_eth_combo=race_eth_combo,
                    merge_date_time=merge_date_time,
                    empty_time=empty_time)
            except Exception as e:
                raise e
            finally:
                if verbose:
                    logger.setLevel(log_level)
                if isinstance(verbose,str):
                    logger.removeHandler(fh)
                    for handler in logger.handlers:
                        if handler.name == "main":
                            # Revert stream handler
                            handler.setLevel(logging.NOTSET)

            self.is_std = True
        else:
            raise ValueError("Dataset has already been cleaned. Aborting cleaning.")


class Source:
    """
    Class for exploring a data source and loading its data

    ...

    Parameters
    ----------
    datasets : pandas or geopandas DataFrame
        Contains information on datasets available from the source

    Methods
    -------
    get_tables_types()
        Get types of data availble from the source
    get_years(table_type, force)
        Get years available for 1 or more datasets
    get_agencies()
        Get agencies available for 1 or more datasets
    load()
        Load data from URL
    load_from_csv()
        Load data from a previously saved CSV file
    """

    datasets: pd.DataFrame = None
    __loader = None

    def __init__(self, 
                source_name: str, 
                state: str | None =None
                ) -> None:
        '''Constructor for Source class

        Parameters
        ----------
        source_name - str
            Source name from datasets table
        state - str
            (Optional) Name of state. Only necessary if source_name is not unique.

        Returns
        -------
        Source object
        '''
        self.datasets = datasets.query(source_name=source_name, state=state)

        # Ensure that all sources are from the same state
        if len(self.datasets) == 0:
            raise ValueError(f"No Sources Found for {source_name}")
        elif self.datasets["State"].nunique() > 1:
            raise ValueError(f"There are multiple sources matching the source name {source_name}. Please specify the state of the desired location in the 2nd argument.")


    def __repr__(self) -> str:
        return str(self.datasets)


    def get_tables_types(self) -> list[str]:
        '''Get types of data availble from the source

        Returns
        -------
        list
            List containing types of data available from source
        '''
        return list(self.datasets["TableType"].unique())


    def get_years(self, 
        table_type: str | defs.TableType, 
        force: bool = False, 
        manual: bool = False,
        datasets: pd.DataFrame = None
        ) -> list[int]:
        '''Get years available for 1 or more datasets

        Parameters
        ----------
        table_type - str or TableType enum
            Only returns years for requested table type
        force - bool
            (Optional) Some data types such as CSV files require reading the whole file to filter for years. By default, an error will be thrown that indicates running load may be more efficient. For these cases, set force=True to run get_years without error.
        manual - bool
            (Optional) If True, for datasets that contain multiple years, the years will be determined by making requests to the dataset rather than using the years stored in the dataset table. The default is False, which runs faster but may not be up-to-date.
        datasets - pd.DataFrame
            (Optional) Only select from datasets in this dataframe instead of self.datasets. datasets should be a subset of the rows in self.datasets.

        Returns
        -------
        list
            List of years available for 1 or more datasets
        '''
        dfs = self.__find_datasets(table_type, datasets)

        cur_year = datetime.now().year
        all_years = list(dfs["Year"])
        years = {x for x in all_years if isinstance(x,numbers.Number) or x==defs.NA}
        for k in [k for k,x in enumerate(all_years) if x==defs.MULTI]:
            df = dfs.iloc[k]
            try:
                _check_version(df)
            except (exceptions.OPD_FutureError,exceptions.OPD_MinVersionError) as e:
                if len(years)>0:
                    # Only throw error if this is on the only dataset so far
                    continue
                else:
                    raise e
            except:
                raise
            url = df["URL"]
            date_field = df["date_field"] if pd.notnull(df["date_field"]) else None
            
            loader = self.__get_loader(df["DataType"], url, df['query'], dataset_id=df["dataset_id"], date_field=date_field)

            if not manual and pd.notnull(df["coverage_start"]) and pd.notnull(df["coverage_end"]) and \
                hasattr(df["coverage_start"], 'year') and hasattr(df["coverage_end"], 'year'):
                years.update(range(df["coverage_start"].year, df["coverage_end"].year+1))
                years_to_check = _get_years_to_check(years, cur_year, force, loader.isfile())
                if len(years_to_check)>0:
                    # Check for updates
                    new_years = loader.get_years(force=force, check=years_to_check)
                    years.update(new_years)
            else:
                new_years = loader.get_years(force=force)
                years.update(new_years)
            
        years = list(years)
        years.sort()

        return years


    def get_agencies(self, 
                     table_type: str | defs.TableType | None = None, 
                     year: str | int | None = None, 
                     partial_name: str | None = None,
                     url_contains: str | None = None,
                     id_contains: str | None = None
                     ) -> list[str]:
        '''Get agencies available for 1 or more datasets

        Parameters
        ----------
        table_type - str or TableType enum
            (Optional) If set, only returns agencies for requested table type
        year - int or the strings opd.defs.MULTI or opd.defs.NONE
            (Optional)  If set, only returns agencies for requested year
        table_type - str or TableType enum
            (Optional) If set, only returns agencies for requested table type
        partial_name - str
            (Optional)  If set, only returns agencies containing the substring
            partial_name for datasets that contain multiple agencies
        url_contains - str | None
            (Optional) If set, URL must contain this string. Can be used in combination with id_contains when multiple datasets match a set of inputs.
        id_contains - str | None
            (Optional) If set, dataset ID must contain this string. Can be used in combination with url_contains when multiple datasets match a set of inputs.

        Returns
        -------
        list
            List of agencies available for 1 or more datasets
        '''

        src = self.__find_datasets(table_type)

        if url_contains:
            src = src[src['URL'].str.contains(url_contains, regex=False)]

        if pd.notnull(id_contains):
            src = src[src['dataset_id'].str.contains(id_contains, regex=False)]

        if year != None:
            src = src[src["Year"] == year]

        if len(src) == 1:
            src = src.iloc[0]
        else:
            print(src, file=sys.stderr)
            raise ValueError("Inputs must filter for a single source")            

        # If year is opd.defs.MULTI, need to use self._agencyField to query URL
        # Otherwise return self.agency
        if src["Agency"] == defs.MULTI:
            _check_version(src)
            loader = self.__get_loader(src["DataType"], src["URL"], src['query'], dataset_id=src["dataset_id"], 
                                       date_field=src["date_field"], agency_field=src["agency_field"])
            if src["DataType"] ==defs.DataType.CSV:
                raise NotImplementedError(f"Unable to get agencies for {src['DataType']}")
            elif src['DataType'] ==defs.DataType.ArcGIS:
                raise NotImplementedError(f"Unable to get agencies for {src['DataType']}")
            elif src["DataType"] ==defs.DataType.EXCEL:
                df = loader.load(year)
                return df[src["agency_field"]].unique().tolist()
            elif src['DataType'] ==defs.DataType.SOCRATA:
                opt_filter = 'LOWER('+ src["agency_field"] + ") LIKE '%" + partial_name.lower() + "%'" if partial_name else None

                select = "DISTINCT " + src["agency_field"]
                year = None if year == defs.MULTI else year

                agency_set = loader.load(year, opt_filter=opt_filter, select=select, output_type="set")
                return list(agency_set)
            elif src['DataType'] ==defs.DataType.CKAN:
                opt_filter = 'LOWER("'+ src["agency_field"] + '")' + " LIKE '%" + partial_name.lower() + "%'" if partial_name else None

                select = 'DISTINCT "' + src["agency_field"] + '"'
                year = None if year == defs.MULTI else year

                agency_set = loader.load(year, opt_filter=opt_filter, select=select, output_type="set")
                return list(agency_set)
            else:
                raise ValueError(f"Unknown data type: {src['DataType']}")
        else:
            return [src["Agency"]]


    @input_swap([0,1], ['table_type','year'], [defs.TableType, {'values':[defs.NA, defs.MULTI], 'types':[list, int]}], opt1=None)
    def get_count(self, 
                  table_type: str | defs.TableType | None = None,
                  year: str | int | list[int] | None = None, 
                  agency: str | None = None, 
                  force: bool = False,
                  verbose: bool | str = False,
                  url_contains: str | None = None,
                  id_contains: str | None = None
                  ) -> int:
        '''Get number of records for a data request

        Parameters
        ----------
        table_type - str or TableType enum
            (Optional) If set, requested dataset will be of this type
        year (Optional) - int or length 2 list or the string opd.defs.MULTI or opd.defs.NONE
            Used to identify the requested dataset if equal to its year value
            Otherwise, for datasets containing multiple years, this filters 
            the return data for a specific year (int input) or a range of years
            [X,Y] to return data for years X to Y
        agency - str
            (Optional) If set, for datasets containing multiple agencies, data will
            only be returned for this agency
        force - bool
            (Optional) For file-based data, an exception will be thrown unless force 
            is true. It may be more efficient to load the data and extract the years
            manually
        verbose - bool | str, optional
            If True, details of data loading will be logged. If a filename, details will
            be logged to that file., by default False
        url_contains - str | None
            (Optional) If set, URL must contain this string. Can be used in combination with id_contains when multiple datasets match a set of inputs.
        id_contains - str | None
            (Optional) If set, dataset ID must contain this string. Can be used in combination with url_contains when multiple datasets match a set of inputs.

        Returns
        -------
        Table
            Table object containing the requested data
        '''

        return self.__load(table_type, year, agency, True, pbar=False, return_count=True, force=force, verbose=verbose, 
                           url_contains=url_contains, id_contains=id_contains)
    
    @input_swap([0,1], ['table_type','year'], [defs.TableType, {'values':[defs.NA, defs.MULTI], 'types':[list, int]}], error=True, opt1=None)
    def load_iter(self,
                table_type: str | defs.TableType,
                year: str | int | list[int],  
                agency: str | None = None, 
                pbar: bool = False, 
                nbatch: int = 10000, 
                offset: int = 0, 
                force: bool =False,
                verbose: bool | str = False,
                url_contains: str | None = None,
                id_contains: str | None = None,
                format_date: bool = True
                ) -> Iterator[Table]:
        '''Get generator to load data from URL in batches

        Parameters
        ----------
        table_type - str or TableType enum
            Table type to load
        year - int or length 2 list or the string opd.defs.MULTI or opd.defs.NONE
            Used to identify the requested dataset if equal to its year value
            Otherwise, for datasets containing multiple years, this filters 
            the return data for a specific year (int input) or a range of years
            [X,Y] to return data for years X to Y
        agency - str
            (Optional) If set, for datasets containing multiple agencies, data will
            only be returned for this agency
        pbar - bool
            (Optional) Whether to show progress bar when loading data. Default False
        nbatch - int
            (Optional) Number of records to load in each batch. Default is 10000.
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 
            to return records starting from the first.
        force - bool
            (Optional) For file-based data, an exception will be thrown unless force 
            is true. It will be more efficient to read the entire dataset all at once
        verbose : bool | str, optional
            If True, details of data loading will be logged. If a filename, details will
            be logged to that file., by default False
        url_contains - str | None
            (Optional) If set, URL must contain this string. Can be used in combination with id_contains when multiple datasets match a set of inputs.
        id_contains - str | None
            (Optional) If set, dataset ID must contain this string. Can be used in combination with url_contains when multiple datasets match a set of inputs.
        format_date : bool, optional
            If True, known date columns (based on presence of date_field in datasets table or data type information provided by dataset owner) will be automatically formatted
            to be pandas datetimes (or pandas Period in rare cases), by default True

        Returns
        -------
        Table generator
            generates Table objects containing the requested data
        '''

        count = self.get_count(table_type, year, agency, force, verbose=verbose, url_contains=url_contains, id_contains=id_contains)
        for k in range(offset, count, nbatch):
            yield self.__load(table_type, year, agency, True, pbar, nrows=min(nbatch, count-k), offset=k, 
                              verbose=verbose, url_contains=url_contains, id_contains=id_contains, format_date=format_date)
    
    @deprecated("load_from_url_gen is deprecated and will be removed in a future release. Please use load_iter instead. "+
                "load_iter uses the same inputs except table_type now comes before year.")
    def load_from_url_gen(self, 
                          year: str | int | list[int], 
                          table_type: str | defs.TableType | None = None, 
                          agency: str | None = None, 
                          pbar: bool = False, 
                          nbatch: int = 10000, 
                          offset: int = 0, 
                          force: bool =False,
                          verbose: bool | str = False
                          ) -> Iterator[Table]:
        '''load_from_url_gen is deprecated. Please use load_iter instead.
        '''

        count = self.get_count(table_type, year, agency, force, verbose=verbose)
        for k in range(offset, count, nbatch):
            yield self.__load(table_type, year, agency, True, pbar, nrows=min(nbatch, count-k), offset=k, verbose=verbose)

    
    @input_swap([0,1], ['table_type','year'], [defs.TableType, {'values':[defs.NA, defs.MULTI], 'types':[list, int]}], error=True, opt1=None)
    def load(self, 
            table_type: str | defs.TableType, 
            year: str | int | list[int], 
            agency: str | None = None,
            pbar: bool = True,
            nrows: int | None = None, 
            offset: int = 0,
            sortby=None,
            verbose: bool | str = False,
            url_contains: str | None = None,
            id_contains: str | None = None,
            format_date: bool = True
            ) -> Table:
        '''Load data from URL

        Parameters
        ----------
        table_type - str or TableType enum
            Table type to load
        year - int or length 2 list or the string opd.defs.MULTI or opd.defs.NONE
            Used to identify the requested dataset if equal to its year value
            Otherwise, for datasets containing multiple years, this filters 
            the return data for a specific year (int input) or a range of years
            [X,Y] to return data for years X to Y
        agency - str
            (Optional) If set, for datasets containing multiple agencies, data will
            only be returned for this agency
        pbar - bool
            (Optional) Whether to show progress bar when loading data. Default True
        nrows - int or None
            (Optional) Number of records to read. Default is None for all records.
        offset - int
            (Optional) Number of records to offset from first record. Default is 0 
            to return records starting from the first.
        verbose : bool | str, optional
            If True, details of data loading will be logged. If a filename, details will
            be logged to that file., by default False
        url_contains - str | None
            (Optional) If set, URL must contain this string. Can be used in combination with id_contains when multiple datasets match a set of inputs.
        id_contains - str | None
            (Optional) If set, dataset ID must contain this string. Can be used in combination with url_contains when multiple datasets match a set of inputs.
        format_date : bool, optional
            If True, known date columns (based on presence of date_field in datasets table or data type information provided by dataset owner) will be automatically formatted
            to be pandas datetimes (or pandas Period in rare cases), by default True

        Returns
        -------
        Table
            Table object containing the requested data
        '''

        return self.__load(table_type, year, agency, True, pbar, nrows=nrows, offset=offset, sortby=sortby, 
                           verbose=verbose, url_contains=url_contains, id_contains=id_contains, format_date=format_date)

    
    @deprecated("load_from_url is deprecated and will be removed in a future release. Please use load instead. "+
                "load uses the same inputs except table_type now comes before year.")
    def load_from_url(self, 
                      year: str | int | list[int], 
                      table_type: str | defs.TableType | None = None, 
                      agency: str | None = None,
                      pbar: bool = True,
                      nrows: int | None = None, 
                      offset: int = 0,
                      sortby=None,
                      verbose: bool | str = False
                      ) -> Table:
        '''load_from_url is deprecated and will be removed in a future release. Please use load instead.
        '''

        return self.__load(table_type, year, agency, True, pbar, nrows=nrows, offset=offset, sortby=sortby, verbose=verbose)

    def __find_datasets(self, table_type, src=None):
        if src is None:
            src = self.datasets.copy()
        elif isinstance(src, pd.Series):
            src = src.to_frame().T
        if table_type != None:
            src = src[src["TableType"].str.upper() == str(table_type).upper()]

        return src
    
    def __filter_for_source(self, table_type, year, url_contains, id_contains, errors=True):
        orig_src = self.__find_datasets(table_type)
        src = orig_src.copy()

        if isinstance(year, list) and len(year)>2:
            raise ValueError("year input must either be a single year or a list containing a start and stop year")
        
        if url_contains:
            src = src[src['URL'].str.contains(url_contains, regex=False)]

        if pd.notnull(id_contains):
            src = src[src['dataset_id'].str.contains(id_contains, regex=False)]

        matchingYears = src["Year"]==year if not isinstance(year, list) else pd.Series(False, src.index)

        if (filter_by_year:=not matchingYears.any()):
            src = src[src["Year"]==defs.MULTI]
        else:
            src = src[matchingYears]

        if isinstance(year,list):
            year_filter = []
            for y in year:
                if isinstance(y, str) and re.search(r'\d{4}-\d{2}-\d{2}', y):
                    year_filter.append(int(y[:4]))
                else:
                    year_filter.append(y)
        else:
            year_filter = year

        if len(src)>1 and year!=defs.MULTI:
            # Try to find a single multi-year dataset containing the year
            contains = pd.Series(False, index=src.index)
            for k in src.index:
                ds_years = self.get_years(table_type, datasets=src.loc[[k]])
                if isinstance(year,list):
                    if any([x>=year_filter[0] and x<=year_filter[1] for x in ds_years]):
                        contains[k] = True
                elif year in ds_years:
                    contains[k] = True
            src = src[contains]

        if len(src)>0 and isinstance(year,list) and not url_contains and pd.isnull(id_contains):
            # Ensure that year range does not also match a single year dataset
            if (orig_src['Year'].apply(lambda x: x!=defs.MULTI and x>=year_filter[0] and x<=year_filter[1])).any():
                raise ValueError(f"Year range cannot contain the year corresponding to a single year dataset.\n "
                                 f"A dataset exists for the year {year}\n "+
                                 "If the requested year range was correct, the url_contains or id_contains input can be used to specify a dataset in ambiguous cases "+
                                 "by setting url_contains to a unique substring of the desired dataset's URL or id_contains to the dataset ID. The URL(s) for the datasets matching "+
                                 f"the current inputs are {list(src['URL'])}")
            
        if isinstance(src, pd.core.frame.DataFrame):
            if len(src) == 0:
                err_msg = f"There are no sources matching {table_type=} and {year=}"
                if url_contains:
                    err_msg+=f" and {url_contains=}"
                if pd.notnull(id_contains):
                    err_msg+=f" and {id_contains=}"
                raise ValueError(err_msg)
            elif len(src) > 1:
                if errors:
                    err_msg = f"There is more than one source matching {table_type=} and {year=}"
                    if url_contains:
                        err_msg+=f" and {url_contains=}"
                    if pd.notnull(id_contains):
                        err_msg+=f" and {id_contains=}"
                    if isinstance(year, list):
                        raise ValueError(err_msg+" It is possible that the year range covers more the one dataset." +
                                        " Set the year input to not contain years for multiple datasets and/or use the url_contains or id_contains "+
                                        "input to specify a single dataset "+
                                        "by setting url_contains to a unique substring of the desired dataset's URL or id_contains to the dataset ID. "
                                        f"The URL(s) for the datasets matching the current inputs are {list(src['URL'])}")
                    elif year==defs.MULTI:
                        raise ValueError(err_msg+f" Therea are multiple multi-year datasets with year={defs.MULTI}." +
                                        " Use the url_contains or id_contains "+
                                        "input to specify a single dataset "+
                                        "by setting url_contains to a unique substring of the desired dataset's URL or id_contains to the dataset ID. "
                                        f"The URL(s) for the datasets matching the current inputs are {list(src['URL'])}")
                    else:
                        raise ValueError(err_msg+
                                        " Set the year input to a single year or a year range and/or "+
                                        "use the url_contains or id_contains input to specify a single dataset "+
                                        "by setting url_contains to a unique substring of the desired dataset's URL or id_contains to the dataset ID. "
                                        f"The URL(s) for the datasets matching the current inputs are {list(src['URL'])}")
                else:
                    # This is only for testing
                    pass
            else:
                src = src.iloc[0]

        return src, filter_by_year


    def __load(self, table_type, year, agency, load_table, pbar=True, return_count=False, force=False, 
               nrows=None, offset=0, sortby=None, verbose=False, url_contains=None, id_contains=None, format_date=True):
        # Make copy so original isn't changed
        year = year.copy() if isinstance(year, list) else year

        src, filter_by_year = self.__filter_for_source(table_type, year, url_contains, id_contains)

        # Load data from URL. For year or agency equal to opd.defs.MULTI, filtering can be done
        url = src["URL"]

        if filter_by_year:
            year_filter = year
        else:
            year_filter = None

        if not pd.isnull(src["dataset_id"]):
            dataset_id = src["dataset_id"]
        else:
            dataset_id = None

        table_year = None
        if not pd.isnull(src["date_field"]):
            date_field = src["date_field"]
            if year_filter != None:
                table_year = year_filter
        else:
            date_field = None
        
        table_agency = None
        if not pd.isnull(src["agency_field"]):
            agency_field = src["agency_field"]
            if agency != None and src['DataType'] !=defs.DataType.ArcGIS:
                table_agency = agency
        else:
            agency_field = None
        
        #It is assumed that each data loader method will return data with the proper data type so date type etc...
        if load_table:
            _check_version(src)
            loader = self.__get_loader(src['DataType'], url, src['query'], dataset_id=dataset_id, 
                                       date_field=date_field, agency_field=agency_field)

            opt_filter = None
            if agency != None and agency_field != None:
                # Double up any apostrophes for SQL query
                agency = agency.replace("'","''")
                if src['DataType']==defs.DataType.CKAN:
                    opt_filter = 'LOWER("' + agency_field + '"' + ") = '" + agency.lower() + "'"
                else:
                    opt_filter = 'LOWER(' + agency_field + ") = '" + agency.lower() + "'"

            logger = logging.getLogger("opd-load")
            if verbose:
                log_level = logger.level
            if isinstance(verbose,str):
                # verbose is a filename
                fh = logging.FileHandler(verbose)
                logger.addHandler(fh)
                for handler in logger.handlers:
                    if handler.name == "main":
                        # Temporarily up level of stream handler so that only print to file
                        handler.setLevel(logging.WARNING)
            if verbose:
                # Set logger to info so log messages in preproc.standardize will be displayed
                logger.setLevel(logging.INFO)

            logger.debug(f"Source URL: {src['source_url']}")
            logger.debug(f"Data URL: {url}")
            if dataset_id:
                logger.debug(f"]tDataset: {dataset_id}")
            logger.debug(f"Data URL: {src['DataType']}")
            if isinstance(src["readme"], str) and len(src["readme"].strip())>0:
                logger.debug(f"Data URL: {src['readme']}")
            
            try:
                if return_count:
                    return loader.get_count(year=year_filter, agency=agency, opt_filter=opt_filter, force=force)
                else:
                    table = loader.load(year=year_filter, agency=agency, opt_filter=opt_filter, nrows=nrows, pbar=pbar, offset=offset, sortby=sortby, 
                                        format_date=format_date)
                    if format_date:
                        date_field = self.__fix_date_field(table, date_field, src.name)
                        table = _check_date(table, date_field)
            except:
                raise
            finally:
                if verbose:
                    logger.setLevel(log_level)
                if isinstance(verbose,str):
                    logger.removeHandler(fh)
                    for handler in logger.handlers:
                        if handler.name == "main":
                            # Revert stream handler
                            handler.setLevel(logging.NOTSET)

        else:
            table = None

        return Table(src, table, year_filter=table_year, agency=table_agency)


    def load_from_csv(self, 
                      year: str | int | list[int],
                      output_dir: str | None = None, 
                      table_type: str | defs.TableType | None = None,
                      agency: str | None = None,
                      zip: bool =False,
                      url_contains: str | None = None,
                      id_contains: str | None = None,
                      format_date: bool = True,
                      filename: str | None = None
                      ) -> Table:
        '''Load data from previously saved CSV file
        
        Parameters
        ----------
        year - int or length 2 list or the string opd.defs.MULTI or opd.defs.NONE
            Used to identify the requested dataset if equal to its year value
            Otherwise, for datasets containing multiple years, this filters 
            the return data for a specific year (int input) or a range of years
            [X,Y] to return data for years X to Y
        output_dir - str
            (output_dirOptional) Directory where CSV file is stored
        table_type - str or TableType enum
            (Optional) If set, requested dataset will be of this type
        agency - str
            (Optional) If set, for datasets containing multiple agencies, data will
            only be returned for this agency
        zip - bool
            (Optional) Set to true if CSV is in a zip file with the same filename. Default: False
        url_contains - str | None
            (Optional) If set, URL must contain this string. Can be used in combination with id_contains when multiple datasets match a set of inputs.
        id_contains - str | None
            (Optional) If set, dataset ID must contain this string. Can be used in combination with url_contains when multiple datasets match a set of inputs.
        format_date : bool, optional
            If True, known date columns (based on presence of date_field in datasets table or data type information provided by dataset owner) will be automatically formatted
            to be pandas datetimes (or pandas Period in rare cases), by default True
        filename: str, optional
            If set, this will override the default filename based on the other inputs

        Returns
        -------
        Table
            Table object containing the requested data
        '''

        table = self.__load(table_type, year, agency, False, url_contains=url_contains, id_contains=id_contains, format_date=format_date)

        if not filename:
            filename = table.get_csv_filename()
            
        if output_dir != None:
            filename = path.join(output_dir, filename)   

        if zip:
            filename = filename.replace(".csv",'.zip')  
            
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", message=r"Columns \(.+\) have mixed types", category=pd.errors.DtypeWarning)
            table.table = pd.read_csv(filename, parse_dates=True, encoding_errors='surrogateescape')
        table.date_field = self.__fix_date_field(table.table, table.date_field, table.details.name)
        table.table = _check_date(table.table, table.date_field)  

        return table
    

    def find_related_tables(self,
                            table_type: str | defs.TableType | None,
                            year: str | int = None,
                            sub_type: 
                                Literal['INCIDENTS','SUBJECTS',"OFFICERS","SUBJECTS/OFFICERS",'PENALTIES','ALLEGATIONS',
                                        'NONMOTORIST','BACKGROUND','VEHICLES', None] = None,
                            exact_match: bool =False) -> tuple[str]:
        """For cases where information is split across tables (i.e. 'USE OF FORCE - INCIDENTS', 'USE OF FORCE - SUBJECTS', 'USE OF FORCE - OFFICERS'),
        find related tables for different subtypes. 'USE OF FORCE - INCIDENTS' will return 'USE OF FORCE - SUBJECTS' and 'USE OF FORCE - OFFICERS' 
        without sub_type specified and 'USE OF FORCE - SUBJECTS' if sub_type='SUBJECTS'

        Parameters
        ----------
        table_type : str | defs.TableType | None
            Type of table to find related tables for
        year - int or the strings opd.defs.MULTI or opd.defs.NONE. or None, optional
            Year of datasets to find related tables for. Optional if there is only one possible table for each table type.
        sub_type : Literal['INCIDENTS','SUBJECTS',"OFFICERS","SUBJECTS/OFFICERS",'PENALTIES','ALLEGATIONS',
                           'NONMOTORIST','BACKGROUND','VEHICLES', None], optional
            If specified, only return table containing data for the requested sub_type by default None
        exact_match : bool, optional
            If False, return tables that contain the specified sub_type (subtype="SUBJECTS" will return 'SUBJECTS' and SUBJECTS/OFFICERS tables).
            If True, return only return exact sub_type match  (subtype="SUBJECTS" will return 'SUBJECTS' table only). by default False

        Returns
        -------
        tuple[str]
            Tuple of related table types
        tuple[str]
            Tuple of year values corresponding to each table type
        """

        all_related_tables = [str(x) for x in defs.TableType if re.search(r'.+\-(?!INVOLVED)\s*', x)]
        types = set([re.sub(r'.+\-(?!INVOLVED)\s*','',x) for x in all_related_tables])
        if m:=re.search(r"^(?P<table_type>.+) - (?P<subtype>("+'|'.join(types)+r"))$", table_type, re.IGNORECASE):
            p = re.compile(rf"^{m.groupdict()['table_type']} - (?P<subtype>("+'|'.join(types)+r"))$", re.IGNORECASE)
            related = [(str(x),y) for x,y in zip(self.datasets['TableType'], self.datasets['Year']) if p.search(x) and x!=table_type]
            if sub_type:
                types = [p.search(x[0]).groupdict()['subtype'] for x in related]
                if exact_match:
                    related = [x for x,y in zip(related, types) if y==sub_type]
                else:
                    related = [x for x,y in zip(related, types) if sub_type in y]

            if year:
                related = [x for x in related if x[1]==year or x[1] in [defs.NA,defs.MULTI] or year in [defs.NA,defs.MULTI]]
            elif len(related)!=len(set([x[0] for x in related])):
                raise ValueError("Multiple possible years found for related tables. Please specify year.")
            return tuple(x for x,_ in related), tuple(x for _,x in related)
        else:
            # This table_type has no subtypes
            return (), ()


    def get_csv_filename(self, 
                         year: str | int | list[int],
                         output_dir: str | None = None, 
                         table_type: str | defs.TableType | None = None,
                         agency: str | None = None,
                         url_contains: str | None = None,
                         id_contains: str | None = None
                         ) -> str:
        '''Get auto-generated CSV filename
        
        Parameters
        ----------
        year - int or length 2 list or the string opd.defs.MULTI or opd.defs.NONE
            Used to identify the requested dataset if equal to its year value
            Otherwise, for datasets containing multiple years, this filters 
            the return data for a specific year (int input) or a range of years
            [X,Y] to return data for years X to Y
        output_dir - str
            (Optional) Directory where CSV file is stored
        table_type - str or TableType enum
            (Optional) If set, requested dataset will be of this type
        agency - str
            (Optional) If set, for datasets containing multiple agencies, data will
            only be returned for this agency
        url_contains - str | None
            (Optional) If set, URL must contain this string. Can be used in combination with id_contains when multiple datasets match a set of inputs.
        id_contains - str | None
            (Optional) If set, dataset ID must contain this string. Can be used in combination with url_contains when multiple datasets match a set of inputs.

        Returns
        -------
        str
            Auto-generated CSV filename
        '''

        table = self.__load(table_type, year, agency, False, url_contains=url_contains, id_contains=id_contains)

        filename = table.get_csv_filename()
        if output_dir != None:
            filename = path.join(output_dir, filename)             

        return filename

    def __get_loader(self, data_type, url, query, dataset_id=None, date_field=None, agency_field=None):
        if pd.isnull(dataset_id):
            dataset_id = None
        params = (data_type, url, dataset_id, date_field, agency_field)
        if self.__loader is not None and self.__loader[0]==params:
            return self.__loader[1]
        
        if dataset_id and ';' in dataset_id:
            # Multiple dataset IDs
            if data_type ==defs.DataType.CSV:
                loader = data_loaders.CombinedDataset(data_loaders.Csv, url, dataset_id, date_field=date_field, agency_field=agency_field)
            elif data_type ==defs.DataType.EXCEL:
                loader = data_loaders.CombinedDataset(data_loaders.Excel, url, dataset_id, date_field=date_field, agency_field=agency_field)
            else:
                raise ValueError(f"Not supported data type for CombinedDataset: {data_type}")
        else:
            if data_type ==defs.DataType.CSV:
                loader = data_loaders.Csv(url, date_field=date_field, agency_field=agency_field)
            elif data_type ==defs.DataType.EXCEL:
                loader = data_loaders.Excel(url, data_set=dataset_id, date_field=date_field, agency_field=agency_field) 
            elif data_type ==defs.DataType.ArcGIS:
                loader = data_loaders.Arcgis(url, date_field=date_field)
            elif data_type ==defs.DataType.SOCRATA:
                loader = data_loaders.Socrata(url, dataset_id, date_field=date_field)
            elif data_type ==defs.DataType.CARTO:
                loader = data_loaders.Carto(url, dataset_id, date_field=date_field, query=query)
            elif data_type ==defs.DataType.CKAN:
                loader = data_loaders.Ckan(url, dataset_id, date_field=date_field, query=query)
            else:
                raise ValueError(f"Unknown data type: {data_type}")

        self.__loader = (params, loader)

        return loader
    
    def __fix_date_field(self, table, date_field, loc):
        if date_field != None and table is not None and len(table)>0 and date_field not in table and \
            any([x.lower()==date_field.lower() for x in table.columns]):
            # Instances have been found where capitalization changes
            date_field = [x for x in table.columns if x.lower()==date_field.lower()][0]
            # Correct date field in tables
            self.datasets.loc[loc, "date_field"] = date_field
            datasets.datasets.loc[loc, "date_field"] = date_field

        return date_field


def _check_date(table, date_field):
    logger = logging.getLogger("opd-load")
    table = table.copy()
    if date_field != None and table is not None and len(table)>0 and date_field in table:
        dts = table[date_field]
        dts = dts[dts.notnull()]
        if len(dts) > 0:
            one_date = dts.iloc[0]  
            if isinstance(one_date, pd.Timestamp):
                if logger:
                    logger.debug(f"Converting values in column {date_field} to datetime objects")
                table[date_field] = to_datetime(table[date_field], ignore_errors=True)
            elif isinstance(one_date, str) and re.match(r'^20\d{2}(\-|/)\d{2}$',one_date):
                if logger:
                    logger.debug(f"Converting values in column {date_field} to monthly Period objects")
                table[date_field] = table[date_field].apply(pd.Period, args=('M'))
            elif (isinstance(one_date, numbers.Number) or isinstance(one_date,str)) and \
                ("year" in date_field.lower() or date_field.lower() == "yr" or \
                 (isinstance(one_date, numbers.Number) and ((dts>=1900) & (dts<2100)).all())):
                if logger:
                    logger.debug(f"Converting values in column {date_field} to annual Period objects")
                if isinstance(one_date, str) and re.search(r'\d{4}\sQ\d', one_date):
                    table[date_field] = table[date_field].str.replace(r'(\d{4}) (Q\d)', r'\1\2', regex=True)
                    table[date_field] = table[date_field].apply(pd.Period, args=('Q'))
                else:
                    table[date_field] = table[date_field].apply(pd.Period, args=('Y'))
            elif isinstance(one_date, str):
                p = re.compile(r'^Unknown string format: \d{4}-(\d{2}|__)-(\d{2}|__) present at position \d+$')
                def to_datetime_local(x):
                    try:
                        return to_datetime(x, ignore_errors=True)
                    except ParserError as e:
                        if len(e.args)>0 and p.match(e.args[0]) != None:
                            return pd.NaT
                        else:
                            raise
                    except:
                        raise

                if logger:
                    logger.debug(f"Converting values in column {date_field} to datetime objects")
                
                try:
                    # This way is much faster
                    table[date_field] = to_datetime(table[date_field], ignore_errors=True)
                except ValueError as e:
                    table[date_field] = table[date_field].apply(to_datetime_local)

                if pd.api.types.is_object_dtype(table[date_field]):
                    try:
                        # Attempt to convert
                        table = table.astype({date_field: 'datetime64[ns]'})
                    except (pd._libs.tslibs.parsing.DateParseError, ParserError) as e:
                        if 'out of range' in str(e) or \
                            re.search(r'Unknown(\sdatetime)? string format.+: \d+[-/]\d+[-/]\d+,?\s?\d+[-/]\d+[-/]\d+', str(e)):
                            pass
                        else:
                            raise
                    except UnicodeEncodeError:
                        pass
                    except ValueError as e:
                        if len(e.args)>0 and e.args[0].startswith('Cannot mix tz-aware with tz-naive values') and \
                            table[date_field].apply(lambda x: x.hour==0 and x.minute==0).all() and \
                            len(set([str(x.tzinfo) for x in table[date_field] if x.tzinfo is not None]))==1:
                            warnings.warn("Some date values have timezone and some do not. Setting timezone unware objects to have the same timezone as the rest.")
                            tz = [x.tzinfo for x in table[date_field] if x.tzinfo is not None][0]
                            is_not_aware = table[date_field].apply(lambda x: x.tzinfo is None or x.tzinfo.utcoffset(x) is None)
                            table.loc[is_not_aware, date_field] = table.loc[is_not_aware, date_field].apply(lambda x: x.replace(tzinfo=tz))
                            table[date_field] = table[date_field].convert_dtypes()
                        elif 'DateParseError' in str(type(e)) and '-__' in str(e):
                            # Ignore case where month and day are underscores (i.e. 2023-__-__)
                            pass
                        elif 'ParserError' in str(type(e)) and re.search(r'Unknown string format: \d+[-/]\d+[-/]\d+,?\s?\d+[-/]\d+[-/]\d+', str(e)):
                            # Comma separated list of dates cannot be parsed
                            pass
                        else:
                            raise
                    except TypeError as e:
                        if "Period'> is not convertible to datetime" in str(e):
                            # Mixture of datetimes and periods
                            pass
                        else:
                            raise
                    except Exception as e:
                        raise
                elif date_field.lower()=="year" and (table[date_field].dt.month==1).all() and \
                    (table[date_field].dt.day==1).all():
                    if logger:
                        logger.debug(f"Converting values in column {date_field} to annual Period objects")
                    table[date_field] = table[date_field].apply(pd.Period, args=('Y'))

                
            # Replace bad dates with NaT
            if logger:
                logger.debug(f"Replacing any values of 1900-01-01 00:00:00 in column {date_field} with null")
            table[date_field] = table[date_field].replace(datetime.strptime('1900-01-01 00:00:00', '%Y-%m-%d %H:%M:%S'), pd.NaT)

    return table


def get_csv_filename(
    state: str, 
    source_name: str, 
    agency: str, 
    table_type: str | defs.TableType, 
    year: str | int | list[int]
    ) -> str:
    '''Get default CSV filename for the given parameters. Enables reloading of data from CSV.
    
    Parameters
    ----------
    state - str
        Name of state
    source_name - str
        Name of source
    agency - str
        Name of agency
    table_type - str or TableType enum
        Type of data
    year = int or length 2 list or the string opd.defs.MULTI or opd.defs.NONE
        Year of data to load, range of years of data to load as a list [X,Y]
        to load years X to Y, or a string to indicate all of multiple year data
        (opd.defs.MULTI) or a dataset that has no year filtering ("N/A")

    Returns
    -------
    str
        Default CSV filename
    '''
    if isinstance(table_type, defs.TableType):
        table_type = table_type.value
        
    filename = f"{state}_{source_name}"
    if source_name != agency:
        filename += f"_{agency}"
    filename += f"_{table_type}"
    if isinstance(year, list):
        filename += f"_{year[0]}_{year[-1]}"
    else:
        filename += f"_{year}"

    # Clean up filename
    filename = filename.replace(",", "_").replace(" ", "_").replace("__", "_").replace("/", "_")

    filename += ".csv"

    return filename

def _check_version(df):
    min_version = df["min_version"] 
    if pd.notnull(min_version):
        src_name = df["SourceName"]
        state = df["State"]
        table_type = df["TableType"]
        year = df["Year"]
        if min_version.strip() == "-1":
            raise exceptions.OPD_FutureError(
                f"Year {year} {table_type} data for {src_name} in {state} cannot be loaded in this version. " + \
                    "It will be made available in a future release"
            )
        elif version.parse(__version__) < version.parse(min_version):
            raise exceptions.OPD_MinVersionError(
                f"Year {year} {table_type} data for {src_name} in {state} cannot be loaded in version {__version__} of openpolicedata. " + \
                    f"Update OpenPoliceData to at least version {min_version} to access this data."
            )

def _get_years_to_check(years, cur_year, force, isfile):
    max_year = max(years)
    years_to_check = []
    if  cur_year-2 <= max_year < cur_year and (force or not isfile):
        years_to_check = [x for x in range(max_year+1,cur_year+1)]

    return years_to_check
