import pandas as pd
import numpy as np
import logging
import re 
from typing import Optional
from .general import DataProcessorGeneral

logging.basicConfig(
    level = logging.INFO,                                 
    format = '%(asctime)s - %(levelname)s - %(message)s'  
)

class DataProcessorColorectal(DataProcessorGeneral):
    GROUP_STAGE_MAPPING = {
        # Stage 0/Occult
        '0': '0',
        
        # Stage I
        'I': 'I',
        
        # Stage II
        'II': 'II',
        'IIA': 'II',
        'IIB': 'II',
        'IIC': 'II',
        
        # Stage III
        'III': 'III',
        'IIIA': 'III',
        'IIIB': 'III',
        'IIIC': 'III',
        
        # Stage IV
        'IV': 'IV',
        'IVA': 'IV',
        'IVB': 'IV',
        'IVC': 'IV',
        
        # Unknown
        'Unknown': 'unknown'
    }
    
    ICD_9_EXLIXHAUSER_MAPPING = {
        # Congestive heart failure
        r'^39891|^40201|^40211|^40291|^40401|^40403|^40411|^40413|^40491|^40493|^4254|^4255|^4256|^4257|^4258|^4259|^428': 'chf',
        
        # Cardiac arrhythmias
        r'^4260|^42613|^4267|^4269|^42610|^42612|^4270|^4271|^4272|^4273|^4274|^4276|^4277|^4278|^4279|^7850|^99601|^99604|^V450|^V533': 'cardiac_arrhythmias',
        
        # Valvular disease
        r'^0932|^394|^395|^396|^397|^424|^7463|^7464|^7465|^7466|^V422|^V433': 'valvular_disease',
        
        # Pulmonary circulation disorders
        r'^4150|^4151|^416|^4170|^4178|^4179': 'pulm_circulation',
        
        # Peripheral vascular disorders
        r'^0930|^4373|^440|^441|^4431|^4432|^4433|^4434|^4435|^4436|^4437|^4438|^4439|^4471|^5571|^5579|^V434': 'pvd',
        
        # Hypertension, uncomplicated
        r'^401': 'htn_uncomplicated',
        
        # Hypertension, complicated
        r'^402|^403|^404|^405': 'htn_complicated',
        
        # Paralysis
        r'^3341|^342|^343|^3440|^3441|^3442|^3443|^3444|^3445|^3446|^3449': 'paralysis',
        
        # Other neurological disorders
        r'^3319|^3320|^3321|^3334|^3335|^33392|^334|^335|^3362|^340|^341|^345|^3481|^3483|^7803|^7843': 'other_neuro',
        
        # Chronic pulmonary disease
        r'^4168|^4169|^490|^491|^492|^493|^494|^495|^496|^497|^498|^499|^500|^501|^502|^503|^504|^505|^5064|^5081|^5088': 'chronic_pulm_disease',
        
        # Diabetes, uncomplicated
        r'^2500|^2501|^2502|^2503': 'diabetes_uncomplicated',
        
        # Diabetes, complicated
        r'^2504|^2505|^2506|^2507|^2508|^2509': 'diabetes_complicated',
        
        # Hypothyroidism
        r'^2409|^243|^244|^2461|^2468': 'hypothyroid',
        
        # Renal failure
        r'^40301|^40311|^40391|^40402|^40403|^40412|^40413|^40492|^40493|^585|^586|^5880|^V420|^V451|^V56': 'renal_failure',
        
        # Liver disease
        r'^07022|^07023|^07032|^07033|^07044|^07054|^0706|^0709|^4560|^4561|^4562|^570|^571|^5722|^5723|^5724|^5725|^5726|^5727|^5728|^5733|^5734|^5738|^5739|^V427': 'liver_disease',
        
        # Peptic ulcer disease excluding bleeding
        r'^5317|^5319|^5327|^5329|^5337|^5339|^5347|^5349': 'pud',
        
        # AIDS/HIV
        r'^042|^043|^044': 'aids_hiv',
        
        # Lymphoma
        r'^200|^201|^202|^2030|^2386': 'lymphoma',
        
        # Rheumatoid arthritis/collagen vascular diseases
        r'^446|^7010|^7100|^7101|^7102|^7103|^7104|^7108|^7109|^7112|^714|^7193|^720|^725|^7285|^72889|^72930': 'rheumatic',
        
        # Coagulopathy
        r'^286|^2871|^2873|^2874|^2875': 'coagulopathy',
        
        # Obesity
        r'^2780': 'obesity',
        
        # Weight loss
        r'^260|^261|^262|^263|^7832|^7994': 'weight_loss',
        
        # Fluid and electrolyte disorders
        r'^2536|^276': 'fluid',
        
        # Blood loss anemia
        r'^2800': 'blood_loss_anemia',
        
        # Deficiency anemia
        r'^2801|^2802|^2803|^2804|^2805|^2806|^2807|^2808|^2809|^281': 'deficiency_anemia',
        
        # Alcohol abuse
        r'^2652|^2911|^2912|^2913|^2915|^2916|^2917|^2918|^2919|^3030|^3039|^3050|^3575|^4255|^5353|^5710|^5711|^5712|^5713|^980|^V113': 'alcohol_abuse',
        
        # Drug abuse
        r'^292|^304|^3052|^3053|^3054|^3055|^3056|^3057|^3058|^3059|^V6542': 'drug_abuse',
        
        # Psychoses
        r'^2938|^295|^29604|^29614|^29644|^29654|^297|^298': 'psychoses',
        
        # Depression
        r'^2962|^2963|^2965|^3004|^309|^311': 'depression'
    }

    ICD_10_ELIXHAUSER_MAPPING = {
        # Congestive heart failure
        r'^I099|^I110|^I130|^I132|^I255|^I420|^I425|^I426|^I427|^I428|^I429|^I43|^I50|^P290': 'chf',
        
        # Cardiac arrhythmias
        r'^I441|^I442|^I443|^I456|^I459|^I47|^I48|^I49|^R000|^R001|^R008|^T821|^Z450|^Z950': 'cardiac_arrhythmias',
        
        # Valvular disease
        r'^A520|^I05|^I06|^I07|^I08|^I091|^I098|^I34|^I35|^I36|^I37|^I38|^I39|^Q230|^Q231|^Q232|^Q233|^Z952|^Z953|^Z954': 'valvular_disease',
        
        # Pulmonary circulation disorders
        r'^I26|^I27|^I280|^I288|^I289': 'pulm_circulation',
        
        # Peripheral vascular disorders
        r'^I70|^I71|^I731|^I738|^I739|^I771|^I790|^I792|^K551|^K558|^K559|^Z958|^Z959': 'pvd',
        
        # Hypertension, uncomplicated
        r'^I10': 'htn_uncomplicated',
        
        # Hypertension, complicated
        r'^I11|^I12|^I13|^I15': 'htn_complicated',
        
        # Paralysis
        r'^G041|^G114|^G801|^G802|^G81|^G82|^G830|^G831|^G832|^G833|^G834|^G839': 'paralysis',
        
        # Other neurological disorders
        r'^G10|^G11|^G12|^G13|^G20|^G21|^G22|^G254|^G255|^G312|^G318|^G319|^G32|^G35|^G36|^G37|^G40|^G41|^G931|^G934|^R470|^R56': 'other_neuro',
        
        # Chronic pulmonary disease
        r'^I278|^I279|^J40|^J41|^J42|^J43|^J44|^J45|^J46|^J47|^J60|^J61|^J62|^J63|^J64|^J65|^J66|^J67|^J684|^J701|^J703': 'chronic_pulm_disease',
        
        # Diabetes, uncomplicated
        r'^E100|^E101|^E109|^E110|^E111|^E119|^E120|^E121|^E129|^E130|^E131|^E139|^E140|^E141|^E149': 'diabetes_uncomplicated',
        
        # Diabetes, complicated
        r'^E102|^E103|^E104|^E105|^E106|^E107|^E108|^E112|^E113|^E114|^E115|^E116|^E117|^E118|^E122|^E123|^E124|^E125|^E126|^E127|^E128|^E132|^E133|^E134|^E135|^E136|^E137|^E138|^E142|^E143|^E144|^E145|^E146|^E147|^E148': 'diabetes_complicated',
        
        # Hypothyroidism
        r'^E00|^E01|^E02|^E03|^E890': 'hypothyroid',
        
        # Renal failure
        r'^I120|^I131|^N18|^N19|^N250|^Z490|^Z491|^Z492|^Z940|^Z992': 'renal_failure',
        
        # Liver disease
        r'^B18|^I85|^I864|^I982|^K70|^K711|^K713|^K714|^K715|^K717|^K72|^K73|^K74|^K760|^K762|^K763|^K764|^K765|^K766|^K767|^K768|^K769|^Z944': 'liver_disease',
        
        # Peptic ulcer disease excluding bleeding
        r'^K257|^K259|^K267|^K269|^K277|^K279|^K287|^K289': 'pud',
        
        # AIDS/HIV
        r'^B20|^B21|^B22|^B24': 'aids_hiv',
        
        # Lymphoma
        r'^C81|^C82|^C83|^C84|^C85|^C88|^C96|^C900|^C902': 'lymphoma',
        
        # Rheumatoid arthritis/collagen vascular diseases
        r'^L940|^L941|^L943|^M05|^M06|^M08|^M120|^M123|^M30|^M310|^M311|^M312|^M313|^M32|^M33|^M34|^M35|^M45|^M461|^M468|^M469': 'rheumatic',
        
        # Coagulopathy
        r'^D65|^D66|^D67|^D68|^D691|^D693|^D694|^D695|^D696': 'coagulopathy',
        
        # Obesity
        r'^E66': 'obesity',
        
        # Weight loss
        r'^E40|^E41|^E42|^E43|^E44|^E45|^E46|^R634|^R64': 'weight_loss',
        
        # Fluid and electrolyte disorders
        r'^E222|^E86|^E87': 'fluid',
        
        # Blood loss anemia
        r'^D500': 'blood_loss_anemia',
        
        # Deficiency anemia
        r'^D508|^D509|^D51|^D52|^D53': 'deficiency_anemia',
        
        # Alcohol abuse
        r'^F10|^E52|^G621|^I426|^K292|^K700|^K703|^K709|^T51|^Z502|^Z714|^Z721': 'alcohol_abuse',
        
        # Drug abuse
        r'^F11|^F12|^F13|^F14|^F15|^F16|^F18|^F19|^Z715|^Z722': 'drug_abuse',
        
        # Psychoses
        r'^F20|^F22|^F23|^F24|^F25|^F28|^F29|^F302|^F312|^F315': 'psychoses',
        
        # Depression
        r'^F204|^F313|^F314|^F315|^F32|^F33|^F341|^F412|^F432': 'depression'
    }
        
    VAN_WALRAVEN_WEIGHTS = {
        'chf': 7,
        'cardiac_arrhythmias': 5,
        'valvular_disease': -1,
        'pulm_circulation': 4,
        'pvd': 2,
        'htn_uncomplicated': 0,
        'htn_complicated': 0,
        'paralysis': 7,
        'other_neuro': 6,
        'chronic_pulm_disease': 3,
        'diabetes_uncomplicated': 0,
        'diabetes_complicated': 0,
        'hypothyroid': 0,
        'renal_failure': 5,
        'liver_disease': 11,
        'pud': 0,
        'aids_hiv': 0,
        'lymphoma': 9,
        'rheumatic': 0,
        'coagulopathy': 3,
        'obesity': -4,
        'weight_loss': 6,
        'fluid': 5,
        'blood_loss_anemia': -2,
        'deficiency_anemia': -2,
        'alcohol_abuse': 0,
        'drug_abuse': -7,
        'psychoses': 0,
        'depression': -3
    }

    ICD_9_METS_MAPPING = {
        # Lymph nodes
        r'^196': 'lymph_met',
        
        # Thoracic
        r'^1970|^1971|^1972|^1973': 'thoracic_met',

        # Liver
        r'^1977': 'liver_met',
        
        # Bone
        r'^1985': 'bone_met',
        
        # Brain/CNS
        r'^1983|^1984': 'brain_met',
        
        # Adrenal
        r'^1987': 'adrenal_met',
        
        # Peritoneum/retroperitoneum
        r'^1976': 'peritoneum_met',
        
        # Other GI tract
        r'^1974|^1975|^1978': 'gi_met',
        
        # Other sites
        r'^1980|^1981|^1982|^1986|^1988|^199': 'other_met'
    }

    ICD_10_METS_MAPPING = {
        # Lymph nodes
        r'^C77': 'lymph_met',

        # Thoracic
        r'^C780|^C781|^C782|^C783': 'thoracic_met',

        # Liver
        r'^C787': 'liver_met',

        # Bone
        r'^C795': 'bone_met',

        # Brain/CNS
        r'^C793|^C794': 'brain_met',

        # Adrenal
        r'^C797': 'adrenal_met',

        # Peritoneum/retroperitoneum
        r'^C786': 'peritoneum_met',

        # Other GI tract
        r'^C784|^C785|^C788': 'gi_met',

        # Other sites
        r'^C790|^C791|^C792|^C796|^C798|^C799|^C80': 'other_met'
    }

    def __init__(self):
        super().__init__() 

        # colorectal-specific attributes 
        self.enhanced_df = None
        self.biomarkers_df = None 
        self.diagnosis_df = None 
        self.mortality_df = None

    def process_enhanced(self,
                         file_path: str,
                         patient_ids: list = None,
                         drop_stage: bool = True, 
                         drop_dates: bool = True) -> Optional[pd.DataFrame]: 
        """
        Processes Enhanced_MetastaticCRC.csv to standardize categories, consolidate staging information, and calculate time-based metrics between key clinical events.

        Parameters
        ----------
        file_path : str
            Path to Enhanced_MetastaticCRC.csv file
        patient_ids : list, optional
            List of PatientIDs to process. If None, processes all patients
        drop_stage : bool, default=True
            If True, drops original GroupStage after consolidating into major groups
        drop_dates : bool, default=True
            If True, drops date columns (DiagnosisDate and MetDiagnosisDate) after calculating durations

        Returns
        -------
        pd.DataFrame or None
            - PatientID : object
                unique patient identifier
            - GroupStage_mod : category
                consolidated overall staging (0-IV, Unknown) at time of first diagnosis
            - days_diagnosis_to_met : float
                days from first diagnosis to metastatic disease 
            - met_diagnosis_year : category
                year of metastatic diagnosis 
            
            Original staging and date columns (DiagnosisDate and MetDiagnosisDate) retained if respective drop_* = False

        Notes
        -----
        Output handling: 
        - Duplicate PatientIDs are logged as warnings if found but retained in output
        - Processed DataFrame is stored in self.enhanced_df
        """
        # Input validation
        if patient_ids is not None:
            if not isinstance(patient_ids, list):
                raise TypeError("patient_ids must be a list or None")
        
        try:
            df = pd.read_csv(file_path)
            logging.info(f"Successfully read Enhanced_MetastaticCRC.csv file with shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")

            # Filter for specific PatientIDs if provided
            if patient_ids is not None:
                logging.info(f"Filtering for {len(patient_ids)} specific PatientIDs")
                df = df[df['PatientID'].isin(patient_ids)]
                logging.info(f"Successfully filtered Enhanced_MetastaticCRC.csv file with shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")
        
            # Convert categorical columns
            categorical_cols = ['GroupStage', 'CrcSite']
            df[categorical_cols] = df[categorical_cols].astype('category')

            # Recode stage variable using class-level mapping and create new column
            df['GroupStage_mod'] = df['GroupStage'].map(self.GROUP_STAGE_MAPPING).astype('category')

            # Drop original stage variable if specified
            if drop_stage:
                df = df.drop(columns=['GroupStage'])

            # Convert date columns
            date_cols = ['DiagnosisDate', 'MetDiagnosisDate']
            for col in date_cols:
                df[col] = pd.to_datetime(df[col])

            # Generate new variables 
            df['days_diagnosis_to_met'] = (df['MetDiagnosisDate'] - df['DiagnosisDate']).dt.days
            df['met_diagnosis_year'] = pd.Categorical(df['MetDiagnosisDate'].dt.year)

            if drop_dates:
                df = df.drop(columns = ['MetDiagnosisDate', 'DiagnosisDate'])

            # Check for duplicate PatientIDs
            if len(df) > df['PatientID'].nunique():
                duplicate_ids = df[df.duplicated(subset = ['PatientID'], keep = False)]['PatientID'].unique()
                logging.warning(f"Duplicate PatientIDs found: {duplicate_ids}")

            logging.info(f"Successfully processed Enhanced_MetastaticCRC.csv file with final shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")
            self.enhanced_df = df
            return df

        except Exception as e:
            logging.error(f"Error processing Enhanced_MetastaticCRC.csv file: {e}")
            return None
        
    def process_biomarkers(self,
                           file_path: str,
                           index_date_df: pd.DataFrame,
                           index_date_column: str, 
                           days_before: Optional[int] = None,
                           days_after: int = 0) -> Optional[pd.DataFrame]:
        """
        Processes Enhanced_MetCRCBiomarkers.csv by determining biomarker status for each patient within a specified time window relative to an index date. 

        Parameters
        ----------
        file_path : str
            Path to Enhanced_MetCRCBiomarkers.csv file
        index_date_df : pd.DataFrame
            DataFrame containing unique PatientIDs and their corresponding index dates. Only biomarker data for PatientIDs present in this DataFrame will be processed
        index_date_column : str
            Column name in index_date_df containing the index date
        days_before : int | None, optional
            Number of days before the index date to include. Must be >= 0 or None. If None, includes all prior results. Default: None
        days_after : int, optional
            Number of days after the index date to include. Must be >= 0. Default: 0
        
        Returns
        -------
        pd.DataFrame or None
            - PatientID : object
                unique patient identifier
            - BRAF_status : category
                positive if ever-positive, negative if only-negative, otherwise unknown
            - KRAS_status : category
                positive if ever-positive, negative if only-negative, otherwise unknown
            - NRAS_status : category
                positive if ever-positive, negative if only-negative, otherwise unknown
            - MMR/MSI_status : category
                positive if ever-positive, negative if only-negative, otherwise unknown

        Notes
        ------
        Biomarker cleaning and processing: 
        - For each biomarker, status is classified as:
            - 'positive' if any test result is positive (ever-positive)
            - 'negative' if any test is negative without positives (only-negative) 
            - 'unknown' if all results are indeterminate
        - MMR/MSI classification:
            - 'positive' includes: 'Loss of MMR protein expression (MMR protein deficiency found)' and 'MSI-H'
            - 'negative' includes: 'Normal MMR protein expression', 'MSS', and 'MSI-L'
            - MSI-L is classified as negative because these patients typically do not receive checkpoint 
              inhibitors like MSI-H patients and are clinically managed more similarly to MSS patients
        
        - Missing biomarker data handling:
            - All PatientIDs from index_date_df are included in the output
            - Patients without any biomarker tests will have NaN values for all biomarker columns
            - Missing ResultDate is imputed with SpecimenReceivedDate

        Output handling:     
        - Duplicate PatientIDs are logged as warnings if found but retained in output
        - Processed DataFrame is stored in self.biomarkers_df
        """
        # Input validation
        if not isinstance(index_date_df, pd.DataFrame):
            raise ValueError("index_date_df must be a pandas DataFrame")
        if 'PatientID' not in index_date_df.columns:
            raise ValueError("index_date_df must contain a 'PatientID' column")
        if not index_date_column or index_date_column not in index_date_df.columns:
            raise ValueError('index_date_column not found in index_date_df')
        if index_date_df['PatientID'].duplicated().any():
            raise ValueError("index_date_df contains duplicate PatientID values, which is not allowed")
        
        if days_before is not None:
            if not isinstance(days_before, int) or days_before < 0:
                raise ValueError("days_before must be a non-negative integer or None")
        if not isinstance(days_after, int) or days_after < 0:
            raise ValueError("days_after must be a non-negative integer")
        
        index_date_df = index_date_df.copy()
        # Rename all columns from index_date_df except PatientID to avoid conflicts with merging and processing 
        for col in index_date_df.columns:
            if col != 'PatientID':  # Keep PatientID unchanged for merging
                index_date_df.rename(columns={col: f'imported_{col}'}, inplace=True)

        # Update index_date_column name
        index_date_column = f'imported_{index_date_column}'

        try:
            df = pd.read_csv(file_path)
            logging.info(f"Successfully read Enhanced_MetCRCBiomarkers.csv file with shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")

            df['ResultDate'] = pd.to_datetime(df['ResultDate'])
            df['SpecimenReceivedDate'] = pd.to_datetime(df['SpecimenReceivedDate'])

            # Impute missing ResultDate with SpecimenReceivedDate
            df['ResultDate'] = np.where(df['ResultDate'].isna(), df['SpecimenReceivedDate'], df['ResultDate'])

            index_date_df[index_date_column] = pd.to_datetime(index_date_df[index_date_column])

            # Select PatientIDs that are included in the index_date_df the merge on 'left'
            df = df[df.PatientID.isin(index_date_df.PatientID)]
            df = pd.merge(
                    df,
                    index_date_df[['PatientID', index_date_column]],
                    on = 'PatientID',
                    how = 'left'
            )
            logging.info(f"Successfully merged Enhanced_MetCRCBiomarkers.csv df with index_date_df resulting in shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")
            
            # Create new variable 'index_to_result' that notes difference in days between resulted specimen and index date
            df['index_to_result'] = (df['ResultDate'] - df[index_date_column]).dt.days
            
            # Select biomarkers that fall within desired before and after index date
            if days_before is None:
                # Only filter for days after
                df_filtered = df[df['index_to_result'] <= days_after].copy()
            else:
                # Filter for both before and after
                df_filtered = df[
                    (df['index_to_result'] <= days_after) & 
                    (df['index_to_result'] >= -days_before)
                ].copy()

            # Create an empty dictionary to store the dataframes
            biomarker_dfs = {}

            # Process BRAF, KRAS, and NRAS
            for biomarker in ['BRAF', 'KRAS', 'NRAS']:
                biomarker_dfs[biomarker] = (
                    df_filtered
                    .query(f'BiomarkerName == "{biomarker}"')
                    .groupby('PatientID')['BiomarkerStatus']
                    .agg(lambda x: 'positive' if any('Mutation positive' in val for val in x)
                        else ('negative' if any('Mutation negative' in val for val in x)
                            else 'unknown'))
                    .reset_index()
                    .rename(columns={'BiomarkerStatus': f'{biomarker}_status'}) 
            )
                
            # Process MMR/MSI
            positive_values = {
                'Loss of MMR protein expression (MMR protein deficiency found)',
                'MSI-H' 
            }

            negative_values = {
                'Normal MMR protein expression (No loss of nuclear expression of MMR protein)',
                'MSS',
                'MSI-L'
            }

            biomarker_dfs['MMR/MSI'] = (
                df_filtered
                .query('BiomarkerName == "MMR/MSI"')
                .groupby('PatientID')['BiomarkerStatus']
                .agg(lambda x: 'positive' if any(val in positive_values for val in x)
                    else ('negative' if any(val in negative_values for val in x)
                        else 'unknown'))
                .reset_index()
                .rename(columns={'BiomarkerStatus': 'MMR/MSI_status'}) 
            )

            # Merge dataframes -- start with index_date_df to ensure all PatientIDs are included
            final_df = index_date_df[['PatientID']].copy()

            for biomarker in ['BRAF', 'KRAS', 'NRAS', 'MMR/MSI']:
                final_df = pd.merge(final_df, biomarker_dfs[biomarker], on = 'PatientID', how = 'left')

            # Convert to category type
            for biomarker_status in ['BRAF_status', 'KRAS_status', 'NRAS_status', 'MMR/MSI_status']:
                final_df[biomarker_status] = final_df[biomarker_status].astype('category')

            # Check for duplicate PatientIDs
            if len(final_df) > final_df['PatientID'].nunique():
                duplicate_ids = final_df[final_df.duplicated(subset = ['PatientID'], keep = False)]['PatientID'].unique()
                logging.warning(f"Duplicate PatientIDs found: {duplicate_ids}")

            logging.info(f"Successfully processed Enhanced_MetCRCBiomarkers.csv file with final shape: {final_df.shape} and unique PatientIDs: {(final_df['PatientID'].nunique())}")
            self.biomarkers_df = final_df
            return final_df

        except Exception as e:
            logging.error(f"Error processing Enhanced_MetCRCBiomarkers.csv file: {e}")
            return None
        
    def process_diagnosis(self, 
                          file_path: str,
                          index_date_df: pd.DataFrame,
                          index_date_column: str,
                          days_before: Optional[int] = None,
                          days_after: int = 0) -> Optional[pd.DataFrame]:
        """
        Processes Diagnosis.csv by mapping ICD 9 and 10 codes to Elixhauser comorbidity index and calculates a van Walraven score. 
        It also determines site of metastases based on ICD 9 and 10 codes. 
        
        Parameters
        ----------
        file_path : str
            Path to Diagnosis.csv file
        index_date_df : pd.DataFrame
            DataFrame containing unique PatientIDs and their corresponding index dates. Only diagnoses for PatientIDs present in this DataFrame will be processed
        index_date_column : str
            Column name in index_date_df containing the index date
        days_before : int | None, optional
            Number of days before the index date to include for window period. Must be >= 0 or None. If None, includes all prior results. Default: None
        days_after : int, optional
            Number of days after the index date to include for window period. Must be >= 0. Default: 0
        
        Returns
        -------
        pd.DataFrame or None
            - PatientID : object unique patient identifier
            - chf : binary indicator for congestive heart failure
            - cardiac_arrhythmia : binary indicator for cardiac arrhythmias
            - valvular_disease : binary indicator for valvular disease
            - pulm_circulation : binary indicator for pulmonary circulation disorders
            - pvd : binary indicator for peripheral vascular disease
            - htn_uncomplicated : binary indicator for hypertension, uncomplicated
            - htn_complicated : binary indicator for hypertension, complicated
            - paralysis : binary indicator for paralysis
            - other_neuro : binary indicator for other neurological disorders
            - chronic_pulm_disease : binary indicator for chronic pulmonary disease
            - diabetes_uncomplicated : binary indicator for diabetes, uncomplicated
            - diabetes_complicated : binary indicator for diabetes, complicated
            - hypothyroid : binary indicator for hypothyroidism
            - renal_failure : binary indicator for renal failure
            - liver_disease : binary indicator for liver disease
            - PUD : binary indicator for peptic ulcer disease
            - aids_hiv : binary indicator for AIDS/HIV
            - lymphoma : binary indicator for lymphoma
            - rheumatic : binary indicator for rheumatoid arthritis/collagen vascular diseases
            - coagulopathy : binary indicator for coagulopathy
            - obesity : binary indicator for obesity
            - weight_loss : binary indicator for weight loss
            - fluid : binary indicator for fluid and electrolyte disorders
            - blood_loss_anemia : binary indicator for blood loss anemia
            - deficiency_anemia : binary indicator for deficiency anemia
            - alcohol_abuse : binary indicator for alcohol abuse
            - drug_abuse : binary indicator for drug abuse
            - psychoses : binary indicator for psychoses
            - depression : binary indicator for depression
            - van_walraven_score : weighted composite of the binary Elixhauser comorbidities 
            - lymph_met : binary indicator for lymph node metastasis
            - thoracic_met : binary indicator for thoracic metastasis (eg., lung, pleura, mediastinum, or other respiratory)
            - liver_met : binary indicator for liver metastasis
            - bone_met : binary indicator for bone metastasis 
            - brain_met : binary indicator for brain/CNS metastasis
            - adrenal_met : binary indicator for adrenal metastasis
            - peritoneum_met : binary indicator for peritoneal/retroperitoneal metastasis
            - gi_met : binary indicator for gastrointestinal tract metastasis excluding liver, adrenal, and peritoneum
            - other_met : binary indicator for other sites of metastasis

        Notes
        -----
        Mapping information: 
        - See "Coding algorithms for defining comorbidities in ICD-9-CM and ICD-10 administrative data" by Quan et al for details on ICD mapping to comorbidities. 
        For ICD-9 codes, the Enhanced ICD-9-CM by Quan was used for mapping.
        - See "A modification of the Elixhauser comorbidity measures into a point system for hospital death using administrative data" by van Walraven et al for 
        details on van Walraven score.
        - Metastatic cancer and tumor categories are excluded in the Elxihauser comorbidities and van Walraven score as this is intended for an advanced cancer population
        
        Output handling: 
        - All PatientIDs from index_date_df are included in the output and values will be set to 0 for patients with misisng Elixhauser comorbidities or metastasis sites, but NaN for missing van_walraven_score
        - Duplicate PatientIDs are logged as warnings but retained in output
        - Results are stored in self.diagnoses_df attribute
        """
        # Input validation
        if not isinstance(index_date_df, pd.DataFrame):
            raise ValueError("index_date_df must be a pandas DataFrame")
        if 'PatientID' not in index_date_df.columns:
            raise ValueError("index_date_df must contain a 'PatientID' column")
        if not index_date_column or index_date_column not in index_date_df.columns:
            raise ValueError('index_date_column not found in index_date_df')
        if index_date_df['PatientID'].duplicated().any():
            raise ValueError("index_date_df contains duplicate PatientID values, which is not allowed")
        
        if days_before is not None:
            if not isinstance(days_before, int) or days_before < 0:
                raise ValueError("days_before must be a non-negative integer or None")
        if not isinstance(days_after, int) or days_after < 0:
            raise ValueError("days_after must be a non-negative integer")
        
        index_date_df = index_date_df.copy()
        # Rename all columns from index_date_df except PatientID to avoid conflicts with merging and processing 
        for col in index_date_df.columns:
            if col != 'PatientID':  # Keep PatientID unchanged for merging
                index_date_df.rename(columns={col: f'imported_{col}'}, inplace=True)

        # Update index_date_column name
        index_date_column = f'imported_{index_date_column}'

        try:
            df = pd.read_csv(file_path)
            logging.info(f"Successfully read Diagnosis.csv file with shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")

            df['DiagnosisDate'] = pd.to_datetime(df['DiagnosisDate'])
            index_date_df[index_date_column] = pd.to_datetime(index_date_df[index_date_column])

            # Select PatientIDs that are included in the index_date_df the merge on 'left'
            df = df[df.PatientID.isin(index_date_df.PatientID)]
            df = pd.merge(
                df,
                index_date_df[['PatientID', index_date_column]],
                on = 'PatientID',
                how = 'left'
            )
            logging.info(f"Successfully merged Diagnosis.csv df with index_date_df resulting in shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")

            df['index_to_diagnosis'] = (df['DiagnosisDate'] - df[index_date_column]).dt.days
            
            # Select ICD codes that fall within desired before and after index date
            if days_before is None:
                # Only filter for days after
                df_filtered = df[df['index_to_diagnosis'] <= days_after].copy()
            else:
                # Filter for both before and after
                df_filtered = df[
                    (df['index_to_diagnosis'] <= days_after) & 
                    (df['index_to_diagnosis'] >= -days_before)
                ].copy()

            # Elixhauser comorbidities based on ICD-9 codes
            df9_elix = (
                df_filtered
                .query('DiagnosisCodeSystem == "ICD-9-CM"')
                .assign(diagnosis_code = lambda x: x['DiagnosisCode'].replace(r'\.', '', regex=True)) # Remove decimal points from ICD-9 codes to make mapping easier 
                .drop_duplicates(subset = ['PatientID', 'diagnosis_code'], keep = 'first')
                .assign(comorbidity=lambda x: x['diagnosis_code'].map(
                    lambda code: next((comorb for pattern, comorb in self.ICD_9_EXLIXHAUSER_MAPPING.items() 
                                    if re.match(pattern, code)), 'Other')))
                .query('comorbidity != "Other"') 
                .drop_duplicates(subset=['PatientID', 'comorbidity'], keep = 'first')
                .assign(value=1)  # Add a column of 1s to use for pivot
                .pivot(index = 'PatientID', columns = 'comorbidity', values = 'value')
                .fillna(0) 
                .astype('Int64')  
                .rename_axis(columns = None)
                .reset_index()
            )

            # Elixhauser comorbidities based on ICD-10 codes
            df10_elix = (
                df_filtered
                .query('DiagnosisCodeSystem == "ICD-10-CM"')
                .assign(diagnosis_code = lambda x: x['DiagnosisCode'].replace(r'\.', '', regex=True)) # Remove decimal points from ICD-10 codes to make mapping easier 
                .drop_duplicates(subset = ['PatientID', 'diagnosis_code'], keep = 'first')
                .assign(comorbidity=lambda x: x['diagnosis_code'].map(
                    lambda code: next((comorb for pattern, comorb in self.ICD_10_ELIXHAUSER_MAPPING.items() 
                                    if re.match(pattern, code)), 'Other')))
                .query('comorbidity != "Other"') 
                .drop_duplicates(subset=['PatientID', 'comorbidity'], keep = 'first')
                .assign(value=1)  # Add a column of 1s to use for pivot
                .pivot(index = 'PatientID', columns = 'comorbidity', values = 'value')
                .fillna(0) 
                .astype('Int64')
                .rename_axis(columns = None)
                .reset_index()  
            )

            all_columns_elix = ['PatientID'] + list(self.ICD_9_EXLIXHAUSER_MAPPING.values())
            
            # Reindex both dataframes to have all columns, filling missing ones with 0
            df9_elix_aligned = df9_elix.reindex(columns = all_columns_elix, fill_value = 0)
            df10_elix_aligned = df10_elix.reindex(columns = all_columns_elix, fill_value = 0)

            # Combine Elixhauser comorbidity dataframes for ICD-9 and ICD-10
            df_elix_combined = pd.concat([df9_elix_aligned, df10_elix_aligned]).groupby('PatientID').max().reset_index()

            # Calculate van Walraven score
            van_walraven_score = df_elix_combined.drop('PatientID', axis=1).mul(self.VAN_WALRAVEN_WEIGHTS).sum(axis=1)
            df_elix_combined['van_walraven_score'] = van_walraven_score

            # Metastatic sites based on ICD-9 codes 
            df9_mets = (
                df_filtered
                .query('DiagnosisCodeSystem == "ICD-9-CM"')
                .assign(diagnosis_code = lambda x: x['DiagnosisCode'].replace(r'\.', '', regex=True)) # Remove decimal points from ICD-9 codes to make mapping easier 
                .drop_duplicates(subset = ['PatientID', 'diagnosis_code'], keep = 'first')
                .assign(met_site=lambda x: x['diagnosis_code'].map(
                    lambda code: next((site for pattern, site in self.ICD_9_METS_MAPPING.items()
                                    if re.match(pattern, code)), 'no_met')))
                .query('met_site != "no_met"') 
                .drop_duplicates(subset=['PatientID', 'met_site'], keep = 'first')
                .assign(value=1)  # Add a column of 1s to use for pivot
                .pivot(index = 'PatientID', columns = 'met_site', values = 'value')
                .fillna(0) 
                .astype('Int64')  
                .rename_axis(columns = None)
                .reset_index()
            )

            # Metastatic sites based on ICD-10 codes 
            df10_mets = (
                df_filtered
                .query('DiagnosisCodeSystem == "ICD-10-CM"')
                .assign(diagnosis_code = lambda x: x['DiagnosisCode'].replace(r'\.', '', regex=True)) # Remove decimal points from ICD-9 codes to make mapping easier 
                .drop_duplicates(subset = ['PatientID', 'diagnosis_code'], keep = 'first')
                .assign(met_site=lambda x: x['diagnosis_code'].map(
                    lambda code: next((site for pattern, site in self.ICD_10_METS_MAPPING.items()
                                    if re.match(pattern, code)), 'no_met')))
                .query('met_site != "no_met"') 
                .drop_duplicates(subset=['PatientID', 'met_site'], keep = 'first')
                .assign(value=1)  # Add a column of 1s to use for pivot
                .pivot(index = 'PatientID', columns = 'met_site', values = 'value')
                .fillna(0) 
                .astype('Int64')  
                .rename_axis(columns = None)
                .reset_index()
            )

            all_columns_mets = ['PatientID'] + list(self.ICD_9_METS_MAPPING.values())
            
            # Reindex both dataframes to have all columns, filling missing ones with 0
            df9_mets_aligned = df9_mets.reindex(columns = all_columns_mets, fill_value = 0)
            df10_mets_aligned = df10_mets.reindex(columns = all_columns_mets, fill_value = 0)

            df_mets_combined = pd.concat([df9_mets_aligned, df10_mets_aligned]).groupby('PatientID').max().reset_index()

            # Start with index_date_df to ensure all PatientIDs are included
            final_df = index_date_df[['PatientID']].copy()
            final_df = pd.merge(final_df, df_elix_combined, on = 'PatientID', how = 'left')
            final_df = pd.merge(final_df, df_mets_combined, on = 'PatientID', how = 'left')

            binary_columns = [col for col in final_df.columns 
                    if col not in ['PatientID', 'van_walraven_score']]
            final_df[binary_columns] = final_df[binary_columns].fillna(0).astype('Int64')

            # Check for duplicate PatientIDs
            if len(final_df) > final_df['PatientID'].nunique():
                duplicate_ids = final_df[final_df.duplicated(subset = ['PatientID'], keep = False)]['PatientID'].unique()
                logging.warning(f"Duplicate PatientIDs found: {duplicate_ids}")

            logging.info(f"Successfully processed Diagnosis.csv file with final shape: {final_df.shape} and unique PatientIDs: {(final_df['PatientID'].nunique())}")
            self.diagnosis_df = final_df
            return final_df
        
        except Exception as e:
            logging.error(f"Error processing Diagnosis.csv file: {e}")
            return None
        
    def process_mortality(self,
                          file_path: str,
                          index_date_df: pd.DataFrame,
                          index_date_column: str,
                          visit_path: str = None, 
                          telemedicine_path: str = None, 
                          biomarkers_path: str = None, 
                          oral_path: str = None,
                          drop_dates: bool = True) -> Optional[pd.DataFrame]:
        """
        Processes Enhanced_Mortality_V2.csv by cleaning data types, calculating time from index date to death/censor, and determining mortality events. 

        Parameters
        ----------
        file_path : str
            Path to Enhanced_Mortality_V2.csv file
        index_date_df : pd.DataFrame
            DataFrame containing unique PatientIDs and their corresponding index dates. Only mortality data for PatientIDs present in this DataFrame will be processed
        index_date_column : str
            Column name in index_date_df containing the index date
        visit_path : str
            Path to Visit.csv file, using VisitDate to determine last EHR activity date for censored patients
        telemedicine_path : str
            Path to Telemedicine.csv file, using VisitDate to determine last EHR activity date for censored patients
        biomarkers_path : str
            Path to Enhanced_MetCRCBiomarkers.csv file, using SpecimenCollectedDate to determine last EHR activity date for censored patients
        oral_path : str
            Path to Enhanced_MetCRC_Orals.csv file, using StartDate and EndDate to determine last EHR activity date for censored patients
        drop_dates : bool, default = True
            If True, drops date columns (index_date_column, DateOfDeath, last_ehr_date)   
        
        Returns
        -------
        pd.DataFrame or None
            - PatientID : object
                unique patient identifier
            - duration : float
                days from index date to death or censor 
            - event : Int64
                mortality status (1 = death, 0 = censored)

        If drop_dates=False, the DataFrame will also include:
            - {index_date_column} : datetime64
                The index date for each patient
            - DateOfDeath : datetime64
                Date of death (if available)
            - last_ehr_activity : datetime64
                Most recent EHR activity date (if available from supplementary files)
                
        Notes
        ------
        Death date handling:
        - Known death date: 'event' = 1, 'duration' = days from index to death
        - No death date: 'event' = 0, 'duration' = days from index to last EHR activity
        
        Death date imputation for incomplete dates:
        - Missing day: Imputed to 15th of the month
        - Missing month and day: Imputed to July 1st of the year
    
        Censoring logic:
        - Patients without death dates are censored at their last EHR activity
        - Last EHR activity is determined as the maximum date across all provided
          supplementary files (visit, telemedicine, biomarkers, or oral)
        - If no supplementary files are provided or a patient has no activity in 
          supplementary files, duration may be null for censored patients
        
        Output handling:
        - Duplicate PatientIDs are logged as warnings if found but retained in output
        - Processed DataFrame is stored in self.mortality_df
        """
        # Input validation
        if not isinstance(index_date_df, pd.DataFrame):
            raise ValueError("index_date_df must be a pandas DataFrame")
        if 'PatientID' not in index_date_df.columns:
            raise ValueError("index_date_df must contain a 'PatientID' column")
        if not index_date_column or index_date_column not in index_date_df.columns:
            raise ValueError('index_date_column not found in index_date_df')
        if index_date_df['PatientID'].duplicated().any():
            raise ValueError("index_date_df contains duplicate PatientID values, which is not allowed")
        
        index_date_df = index_date_df.copy()
        # Rename all columns from index_date_df except PatientID to avoid conflicts with merging and processing 
        for col in index_date_df.columns:
            if col != 'PatientID':  # Keep PatientID unchanged for merging
                index_date_df.rename(columns={col: f'imported_{col}'}, inplace=True)

        # Update index_date_column name
        index_date_column = f'imported_{index_date_column}'

        try:
            df = pd.read_csv(file_path)
            logging.info(f"Successfully read Enhanced_Mortality_V2.csv file with shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")

            # When only year is available: Impute to July 1st (mid-year)
            df['DateOfDeath'] = np.where(df['DateOfDeath'].str.len() == 4, df['DateOfDeath'] + '-07-01', df['DateOfDeath'])

            # When only month and year are available: Impute to the 15th day of the month
            df['DateOfDeath'] = np.where(df['DateOfDeath'].str.len() == 7, df['DateOfDeath'] + '-15', df['DateOfDeath'])

            df['DateOfDeath'] = pd.to_datetime(df['DateOfDeath'])

            # Process index dates and merge
            index_date_df[index_date_column] = pd.to_datetime(index_date_df[index_date_column])
            df = pd.merge(
                index_date_df[['PatientID', index_date_column]],
                df,
                on = 'PatientID',
                how = 'left'
            )
            logging.info(f"Successfully merged Enhanced_Mortality_V2.csv df with index_date_df resulting in shape: {df.shape} and unique PatientIDs: {(df['PatientID'].nunique())}")
                
            # Create event column
            df['event'] = df['DateOfDeath'].notna().astype('Int64')

            # Initialize final dataframe
            final_df = df.copy()

            # Create a list to store all last activity date dataframes
            patient_last_dates = []

            # Determine last EHR data
            if all(path is None for path in [visit_path, telemedicine_path, biomarkers_path, oral_path]):
                logging.info("WARNING: At least one of visit_path, telemedicine_path, biomarkers_path, or oral_path must be provided to calculate duration for those with a missing death date")
            else: 
                # Process visit and telemedicine data
                if visit_path is not None or telemedicine_path is not None:
                    visit_dates = []
                    try:
                        if visit_path is not None:
                            df_visit = pd.read_csv(visit_path)
                            df_visit['VisitDate'] = pd.to_datetime(df_visit['VisitDate'])
                            visit_dates.append(df_visit[['PatientID', 'VisitDate']])
                            
                        if telemedicine_path is not None:
                            df_tele = pd.read_csv(telemedicine_path)
                            df_tele['VisitDate'] = pd.to_datetime(df_tele['VisitDate'])
                            visit_dates.append(df_tele[['PatientID', 'VisitDate']])
                        
                        if visit_dates:
                            df_visit_combined = pd.concat(visit_dates)
                            df_visit_max = (
                                df_visit_combined
                                .query("PatientID in @index_date_df.PatientID")
                                .groupby('PatientID')['VisitDate']
                                .max()
                                .to_frame(name='last_visit_date')
                                .reset_index()
                            )
                            patient_last_dates.append(df_visit_max)
                    except Exception as e:
                        logging.error(f"Error processing Visit.csv or Telemedicine.csv: {e}")
                                            
                # Process biomarkers data
                if biomarkers_path is not None:
                    try: 
                        df_biomarkers = pd.read_csv(biomarkers_path)
                        df_biomarkers['SpecimenCollectedDate'] = pd.to_datetime(df_biomarkers['SpecimenCollectedDate'])

                        df_biomarkers_max = (
                            df_biomarkers
                            .query("PatientID in @index_date_df.PatientID")
                            .groupby('PatientID')['SpecimenCollectedDate']
                            .max()
                            .to_frame(name='last_biomarker_date')
                            .reset_index()
                        )
                        patient_last_dates.append(df_biomarkers_max)
                    except Exception as e:
                        logging.error(f"Error reading Enhanced_MetCRCBiomarkers.csv file: {e}")

                # Process oral medication data
                if oral_path is not None:
                    try:
                        df_oral = pd.read_csv(oral_path)
                        df_oral['StartDate'] = pd.to_datetime(df_oral['StartDate'])
                        df_oral['EndDate'] = pd.to_datetime(df_oral['EndDate'])

                        df_oral_max = (
                            df_oral
                            .query("PatientID in @index_date_df.PatientID")
                            .assign(max_date=lambda x: x[['StartDate', 'EndDate']].max(axis=1))
                            .groupby('PatientID')['max_date']
                            .max()
                            .to_frame(name='last_oral_date')
                            .reset_index()
                        )
                        patient_last_dates.append(df_oral_max)
                    except Exception as e:
                        logging.error(f"Error reading Enhanced_MetCRC_Orals.csv file: {e}")

                # Combine all last activity dates
                if patient_last_dates:
                    # Start with the first dataframe
                    combined_dates = patient_last_dates[0]
                    
                    # Merge with any additional dataframes
                    for date_df in patient_last_dates[1:]:
                        combined_dates = pd.merge(combined_dates, date_df, on = 'PatientID', how = 'outer')
                    
                    # Calculate the last activity date across all columns
                    date_columns = [col for col in combined_dates.columns if col != 'PatientID']
                    if date_columns:
                        logging.info(f"The following columns {date_columns} are used to calculate the last EHR date")
                        combined_dates['last_ehr_activity'] = combined_dates[date_columns].max(axis=1)
                        single_date = combined_dates[['PatientID', 'last_ehr_activity']]
                        
                        # Merge with the main dataframe
                        final_df = pd.merge(final_df, single_date, on='PatientID', how='left')
     
            # Calculate duration
            if 'last_ehr_activity' in final_df.columns:
                final_df['duration'] = np.where(
                    final_df['event'] == 0, 
                    (final_df['last_ehr_activity'] - final_df[index_date_column]).dt.days, 
                    (final_df['DateOfDeath'] - final_df[index_date_column]).dt.days
                )
                
                # Drop date variables if specified
                if drop_dates:               
                    final_df = final_df.drop(columns=[index_date_column, 'DateOfDeath', 'last_ehr_activity'])
                       
            else: 
                final_df['duration'] = (final_df['DateOfDeath'] - final_df[index_date_column]).dt.days
            
                # Drop date variables if specified
                if drop_dates:               
                    final_df = final_df.drop(columns=[index_date_column, 'DateOfDeath'])
                
            # Check for duplicate PatientIDs
            if len(final_df) > final_df['PatientID'].nunique():
                duplicate_ids = final_df[final_df.duplicated(subset=['PatientID'], keep=False)]['PatientID'].unique()
                logging.warning(f"Duplicate PatientIDs found: {duplicate_ids}")

            logging.info(f"Successfully processed Enhanced_Mortality_V2.csv file with final shape: {final_df.shape} and unique PatientIDs: {final_df['PatientID'].nunique()}. There are {final_df['duration'].isna().sum()} out of {final_df['PatientID'].nunique()} patients with missing duration values")
            self.mortality_df = final_df
            return final_df

        except Exception as e:
            logging.error(f"Error processing Enhanced_Mortality_V2.csv file: {e}")
            return None