from ._imports import *
from ._pconfig import *
from functools import cache
from dateutil.relativedelta import relativedelta

#TODO  currently life data source is using csv need to change to athena 
#TODO  salary success share currently using dummy data, also need to change to athena

def training_pipeline( runtime):   

    '''
    run_time = create_runtime ('inflow_premiums', 'dev', train_end=None)
    dict_results = training_pipeline(run_time)

    Parameters
    ----------
    runtime : dict
        dictionary contains parameters which will be used monitor pipeline.   

    Returns
    ------- 
    dict
        training results including tuned hyperparametersets, dataset used in training.
    ''' 

    cf_type = runtime['cf_type'] #'claim loss'
    train_begin = runtime['train_begin']   #'2020-05-01'
    train_end = runtime['train_end']  #None 
    test_len = runtime['test_len'] #8, 
    env= runtime['env'] #'dev' 

    #train_end is last month by default
    if not train_end:
        train_end = DtFind.first_day_last_month( dt.datetime.today(), dt_format=None).strftime("%Y-%m-%d")    

    df_origin, df_ts, dict_column_rename = get_train_data(cf_type,env,train_begin, train_end)  

    #get training range data
    train_df_prep, test_df_prep, df_selected_reset_prep = prep_prophet_training_data(df_ts, 
                                                                                    train_begin, 
                                                                                    train_end, 
                                                                                    test_len, 
                                                                                    dict_column_rename
                                                                                    )
    
    best_params, best_loss, winner, results = get_best_prophet_param_both2(train_df_prep, test_df_prep, test_len, cf_type)    
    #best_params['additional_feature_flag'] = best_params.get('additional_feature_flag', 0)  #optuna could skip this one
    dict_results = {
        'best params': best_params,
        'best loss': best_loss,
        'winner' : winner,
        'tuning results': results,
        'dataset inference': df_selected_reset_prep,  
        'original dataset':df_origin
        }
    return dict_results  


def get_train_data (cf_type, env, train_begin, train_end, generic_target_name='Values'):
    #mutual
    if cf_type == 'claim_loss':            
            df = get_claim_loss(env)
            df_ts = (df.pipe(add_mutual_month_number)
                                    .pipe(create_mutual_time_series,'total loss payments', 'Loss', train_begin, train_end, cf_type))
            dict_column_rename = {"year_month": "ds", "Loss": "y"}
    elif cf_type == 'claim_subrogation':
            df =  get_misc_adjust(env, filter='Claims Salvage and Subrogation')    
            df_ts = (df.pipe(add_mutual_month_number)
                                .pipe(create_mutual_time_series,'amount', 'sub_sal', train_begin, train_end, cf_type))
            dict_column_rename = {"year_month": "ds", "sub_sal": "y"}
    elif cf_type == 'vendor':
            df =  get_vendor_payments(env)
            df['amount'] = df['amount'].apply(lambda x : -x if x <0 else x)  #convert to positive value
            df_ts = (df.pipe(add_mutual_month_number)
                                .pipe(create_mutual_time_series,'amount', 'vendor', train_begin, train_end, cf_type))
            dict_column_rename = {"year_month": "ds", "vendor": "y"}
    elif cf_type == 'salary':
            df =  get_salary_payments(env)
            df['amount'] = df['amount'].apply(lambda x : -x if x <0 else x)  #convert to positive value    
            df['Date'] = pd.to_datetime(df['Date'])
            df_ts = df.loc[:, ['Date','amount','payday_count','success_share']].set_index('Date')
            dict_column_rename = {"Date": "ds", "amount": "y"}
    elif cf_type == 'inflow_premiums':
           df_invoice = prep_premium_inflow_invoice(env)
           premium_in = get_premium_inflow_history(env)
           data_with_invoice = premium_in.merge(df_invoice,left_on=['actual_month','actual_year'], right_on=['expected_clear_month','expected_clear_year'])
           #data_with_invoice['Date'] = pd.to_datetime(data_with_invoice['Date'])
           df = data_with_invoice.loc[:, ['Date','Premiums','invoice_due_amt']]
           #add additional feature
           #df_features = pd.read_csv(r"C:\Users\A036898\Downloads\a3539b06-967a-46f7-8506-8abc747b9aaf.csv")
           df_features  = get_additional_features(env)
           df_features['year_month'] = pd.to_datetime(df_features['year_month'])
           df = df.merge(df_features.rename(columns={'year_month':'Date'}), how = 'inner', on='Date').iloc[:,:2] #get original, remove invoice too            
           df_ts = df.set_index('Date')
           dict_column_rename = {"Date": "ds", "Premiums": "y"}
           save_additional_features_dataset (df_features, cf_type, env)
    elif cf_type == 'outflow_premiums':           
           df = get_premium_outflow_history(env).loc[:, ['Date','Dividend']]
           df['Dividend'] = df['Dividend'].apply(lambda x : -x if x <0 else x)  #convert to positive value
           #add additional feature from a query
           #df_features = pd.read_csv(r"C:\Users\A036898\Downloads\a3539b06-967a-46f7-8506-8abc747b9aaf.csv")
           df_features  = get_additional_features(env)
           df_features = df_features[['year_month', 'account_count', 'policy_count', 'total_premium',       
                            'unearned__premium_since_policy_start']]  #select highly correlated feature based on outflow reason
           df = df.merge(df_features.rename(columns={'year_month':'Date'}), how = 'inner', on='Date').iloc[:,:2]
           df['Date'] = pd.to_datetime(df['Date'])
           df_ts = df.set_index('Date')
           dict_column_rename = {"Date": "ds", "Dividend": "y"} 
           save_additional_features_dataset (df_features, cf_type, env)       
    
    elif cf_type =='inflow_receipts':  
           df_misc =  get_misc_adjust(env, 'Misc. Receipts')
           df_adj =  get_misc_adjust(env, 'Adjustments')
           df  =  pd.concat([df_misc, df_adj], ignore_index= True)
           df  =            (          #sort by year month number
                df.assign(mon_num = df['trans_month'].apply(lambda x: list(calendar.month_name).index(x)))
                  .sort_values(by = ['trans_year', 'mon_num'])
                  .drop(columns=['mon_num'])        
                )           
           df_ts = (df.pipe(add_mutual_month_number)
                    .pipe(create_mutual_time_series,'amount', 'misc_receipts', train_begin, train_end, cf_type))           
           df_ts['misc_receipts'] = cap_input_data(df_ts['misc_receipts'] )
           dict_column_rename = {"year_month": "ds", "misc_receipts": "y"}

    #life
    if cf_type[:4] == 'life':
        if cf_type == 'life_inflow_premiums':            
                df = get_life_inflow_premiums(env)
        elif cf_type == 'life_outflow_claims':            
                df = get_life_outflow_claims(env)
        elif cf_type in [ 'life_outflow_vendor', 'life_outflow_payroll', 'life_inflow_reinsurance'] :
                df = get_life_data(env, cf_type)
                if cf_type == 'life_outflow_payroll':  #salary model need additional features
                    df = get_life_salary_additional_features(df)                
        elif cf_type == 'life_receipts_adjustments':
                df = get_life_receipts_adjustments(env)
        df_ts = (df.pipe(add_life_month_number)
                                .pipe(create_life_time_series,generic_target_name, train_begin, train_end, cf_type))
        dict_column_rename = {"year_month": "ds", generic_target_name: "y"}     

    return df, df_ts, dict_column_rename


def get_claim_loss(env):
    query_claim_loss = 'SELECT * FROM silver_cashflow_forecast.claim_loss_summary'
    df_claim_loss = aws_athena_query('silver_cashflow_forecast', query_claim_loss, env)
    assert df_claim_loss.values.any(),  f"claim loss has missing value"
    df_claim_loss['total loss payments'] = df_claim_loss.apply(lambda r:r.loss_amt + r.exp_amt, axis =1)
    df_claim_loss = df_claim_loss[['trans_year','trans_month','total loss payments']]
    df_claim_loss['trans_month'] = df_claim_loss['trans_month'].str.strip()
    return df_claim_loss

def get_misc_adjust(env, filter='Claims Salvage and Subrogation'):
    query_sub = f'''SELECT * FROM 
            "silver_cashflow_forecast"."misc_receipt_adj"
            where cash_flow_typ = '{filter}'
            order by date
            '''
    df = aws_athena_query('silver_cashflow_forecast', query_sub, env)
    #make name consistent between dataset
    df['trans_year'] = df['date'].apply(pd.to_datetime).dt.year
    df['trans_month'] = df['date'].apply(pd.to_datetime).dt.month_name()
    df = df[['trans_year','trans_month','amount']]
    assert df.values.any(), f"{filter} has missing value"
    df['trans_month'] = df['trans_month'].str.strip()
    return df 




def add_mutual_month_number(df):
    df = df.copy()
    dict_month_mapping ={month: index for index, month in enumerate(calendar.month_name) if month}
    df['trans_month'] = df['trans_month'].str.strip().map(dict_month_mapping)
    df['trans_month_str'] = [ ("0" + str(i))[-2:]  for i in df['trans_month']]
    df['year_month'] = df['trans_year'].astype('str') + '-' +  df['trans_month_str'] 
    df = df.sort_values(by=['trans_year','trans_month'])
    return df 

def create_mutual_time_series(df, column, new_column_name, month_begin, month_end, cf_type):    
    df1 = df[['year_month', column]]
    df1.loc[:,'year_month'] = pd.to_datetime(df1['year_month'])
    df1.set_index('year_month', inplace=True)  #create time series index
    #month_begin, month_end =pd.to_datetime(month_begin), pd.to_datetime(month_end)
    df1 = df1.loc[month_begin:month_end]  #remove Apr 2024 outlier and last partial month
    if cf_type !='inflow_receipts':  #this type could have both postive and negative
        df1[column] = df1[column].apply(lambda x: -x if x< 0 else x)  #in athena table these are positive value, just in case
    #column_name = 'Loss' if column == 'total loss payments' else 'sal_sub' 
    df1.columns = [new_column_name]   #use in time series, short name
    return df1

#choose data range to use for training
def select_data_range (df_ts, begin, end=None):  
    '''
    select_data_range(df_claim_ts, begin = '2020-04-01')
    '''
    if not begin:
        begin=df_ts.index[0]

    if not end:
        end = df_ts.index[-1]

    return df_ts.loc[begin:end]

def prep_prophet_training_data (df, Begin_dt, end_dt, test_len,dict_column_rename ):
    '''
    This function is used to create train and test data for prophet hyperparameter tuning
    test_len is month number hold for test
    train_df_prep, test_df_prep = prep_prophet_training_data(df_subrogation_ts, '2020-05-01', '2024-09-01', 8, {"year_month": "ds", "Loss": "y"})
    '''
    df = df.copy()
    _df_selected =select_data_range(df, Begin_dt, end_dt)
    # if check_missing:
    #     missing_flag, monthlist = check_ts_missing_month(_df_selected, end_dt)
    #     assert not missing_flag, f'These month(s) data are missing {','.join(monthlist)}'    
    _df_selected_reset = _df_selected.reset_index().rename(columns=dict_column_rename)
    train_len = len(_df_selected_reset)- test_len
    _train_df = _df_selected_reset[0 : train_len]
    _test_df = _df_selected_reset[train_len : ]
    return _train_df, _test_df, _df_selected_reset


#########################add premium inflow code  by wcao
@cache
def get_clear_date(start_date, days_to_add):
    '''
    cache start date and clear date, save run time from 2 min to 2 sec
    '''
    current_date = pd.to_datetime(start_date)      
    while days_to_add > 0:
        current_date += dt.timedelta(days=1)
        if current_date.weekday() < 5 and (not us_holidays.get(current_date)):
            days_to_add -= 1
    return current_date


def calculate_expected_clear_date(row,dt_col='payment_due_dt',method_col='policy_payment_method'):
    business_days_to_add = {
    "ACH/EFT": 2.00,
    "Apply Back to Policy": 1.00,
    "AutoPay": 2.00,
    "AutoPay Bank": 2.00,
    "AutoPay Credit Card": 2.00,
    "Business Check/Money Order": 0.00,
    "Cash": 0.00,
    "Check": 0.00,
    "Credit Card": 2.00,
    "Foreign Check": 0.00,
    "Payment": 2.00,
    "Remote Scanner Deposit": 0.00
    }
    start_date = row[dt_col]
    days_to_add = business_days_to_add.get(row[method_col], 0)
    return get_clear_date(start_date, days_to_add)

def prep_premium_inflow_invoice(env):
    '''
    use lambda date to get the lastest record
    '''
    query_invoice = '''with payment_inflow_rnk as (
                select 
                *,
                row_number() over (partition by "rundate", "uw_company", "policy_payment_method", "payment_due_dt" order by "lambda_run_ts_utc" desc ) as rnk
                from silver_cashflow_forecast.payments_inflow_predictor
    )
            SELECT * FROM payment_inflow_rnk
            where payment_due_dt>date'2021-12-31'
            and rnk =1
    '''
    finserv_results = aws_athena_query('silver_cashflow_forecast', query_invoice, env)
    global us_holidays
    us_holidays = holidays.US()    
    finserv_results['expected_clear_date'] = finserv_results.apply(lambda x:calculate_expected_clear_date(x), axis=1).apply(pd.to_datetime)
    finserv_results['expected_clear_month'] = finserv_results['expected_clear_date'].dt.month
    finserv_results['expected_clear_year'] = finserv_results['expected_clear_date'].dt.year
    finserv_results_agg = finserv_results.groupby(['expected_clear_year','expected_clear_month']).agg({'invoice_due_amt':'sum'}).reset_index()
    return finserv_results_agg
	

def get_premium_inflow_history(env):  #prevent upstream duplicate
    query_premium = '''with cte_rnk as (
                SELECT year_month_day as "Date",year(year_month_day) as "Year",month(year_month_day) as "Month", payment_method,payment 
                    ,row_number() over (partition by year_month_day, payment_method, payment order by year_month_day) as rnk
                    FROM silver_cashflow_forecast.payments_inflow_1_month
                    where year_month_day>date'2021-12-31'
                )    
                select 
                "Date", 
                "Year", 
                "Month",
                payment_method,
                payment
                from cte_rnk where rnk =1'''
    premium_in = aws_athena_query('silver_cashflow_forecast', query_premium, env)
    premium_in['Date'] = premium_in.apply(lambda x: calculate_expected_clear_date(x,dt_col='Date',method_col='payment_method'),axis=1).apply(pd.to_datetime)
    premium_in['Date'] = pd.to_datetime(premium_in[['Year','Month']].assign(DAY=1))
    premium_in['Premiums'] = premium_in['payment'].astype('float')
    premium_in.dropna(axis = 0, inplace= True)
    premium_in['actual_month'] = premium_in['Date'].dt.month.astype(int)
    premium_in['actual_year'] = premium_in['Date'].dt.year.astype(int)
    premium_in_agg = premium_in.groupby(['actual_year','actual_month']).agg({'Premiums':'sum'}).reset_index()
    premium_in_agg['Date'] = pd.to_datetime(premium_in_agg[['actual_year', 'actual_month']].assign(day=1).rename(columns={'actual_year': 'year', 'actual_month': 'month'}))
    return premium_in_agg


def get_premium_outflow_history(env):
    query_premium = '''SELECT date as "Date", amount as Dividend, cash_flow_typ, adjustment_typ, filename, year, month, day   
	FROM "silver_cashflow_forecast"."misc_receipt_adj"
	WHERE cash_flow_typ like '%Premium/Dividend%'
	order by date'''
    premium_out = aws_athena_query('amica-eds-dev-datascience-db', query_premium, env)
    premium_out['Date'] = pd.to_datetime(premium_out['Date'],errors='coerce').apply(lambda x: x.replace(day=1))
    premium_out['Date'] = premium_out['Date'].dt.date
    premium_out['Date'] = premium_out['Date'].astype(str)
    return premium_out

def get_vendor_payments(env):
    query_vendor = '''SELECT date as "Date",Year(date) as trans_year,date_format(date, '%M') as trans_month, amount   
	FROM "silver_cashflow_forecast"."misc_receipt_adj"
	WHERE cash_flow_typ ='Vendor'
	order by date;'''
    df_vendor = aws_athena_query('silver_cashflow_forecast', query_vendor, env)
    assert df_vendor.values.any(),  f"vendor has missing value"
    df_vendor['Date'] = pd.to_datetime(df_vendor['Date'],errors='coerce').apply(lambda x: x.replace(day=1))
    df_vendor['Date'] = df_vendor['Date'].dt.date
    df_vendor['Date'] = df_vendor['Date'].astype(str)
    df_vendor['trans_month'] = df_vendor['trans_month'].str.strip()
    return df_vendor

def calculate_paydays(start_date,end_date,payday_weekday=4, start_payday=dt.datetime(2025,2,21)):
    paydays = []
    start_payday = dt.datetime(2025,2,21) #this represents a a
    current_date = start_date

    while current_date <= end_date:
        if current_date.weekday() == payday_weekday and (current_date - start_payday).days % 14 == 0: #Is Friday and biweekly
            paydays.append(current_date)
        current_date += relativedelta(days=1)

    monthly_counts = {}
    for payday in paydays:
        year = payday.year
        month = payday.strftime('%B')

        if (year, month) not in monthly_counts:
            monthly_counts[(year,month)] = 0
        monthly_counts[(year,month)] += 1
    
    data = [{'trans_year':year, 'trans_month':month, 'payday_count':count} for (year,month), count in monthly_counts.items()]
    data = pd.DataFrame(data)
    return data

def get_payday_datataframe():
    
    current_date = dt.datetime.now()
    start_date = dt.datetime(2020,1,1)
    end_date = current_date + relativedelta(months=4)
    end_date = end_date.replace(day=1) + relativedelta(days=-1)
    
    df = calculate_paydays(start_date,end_date)
    df['success_share'] = np.where(df['trans_month']=='February',1,0)
    return df

def get_salary_payments(env):
    query_salary = '''SELECT date as "Date",Year(date) as trans_year,date_format(date, '%M') as trans_month, amount   
	FROM "silver_cashflow_forecast"."misc_receipt_adj"
	WHERE cash_flow_typ ='Salary and Benefits'
	order by date;'''
    df_salary = aws_athena_query('silver_cashflow_forecast', query_salary, env)
    payday_df = get_payday_datataframe()
    
    #This needs to be replaced with misc_adjustment table^M
    success_sharing_ath = aws_athena_query('silver_cashflow_forecast',
                                       '''select * FROM "silver_cashflow_forecast"."misc_receipt_adj" where cash_flow_typ = 'Success Share';''',env)    
    #success_sharing_ath['date'] = pd.to_datetime(success_sharing_ath['date']).dt.to_period('M').dt.to_timestamp().astype(str)^M    
    df_salary = df_salary.merge(payday_df,on=['trans_year','trans_month'])[['Date','trans_year','trans_month','amount','payday_count','success_share']]
    assert df_salary.values.any(),  f"salary has missing value"
    df_salary['Date'] = pd.to_datetime(df_salary['Date'])+pd.offsets.MonthEnd(0)
    success_sharing_ath['date'] = pd.to_datetime(success_sharing_ath['date'])+pd.offsets.MonthEnd(0)
    for i,row in success_sharing_ath.iterrows():          
          df_salary['amount'] = np.where(df_salary['Date']==row['date'], df_salary['amount']-row['amount'], df_salary['amount'])
    df_salary['Date'] = pd.to_datetime(df_salary['Date'],errors='coerce').apply(lambda x: x.replace(day=1))
    df_salary['Date'] = df_salary['Date'].dt.date    
    df_salary['trans_month'] = df_salary['trans_month'].str.strip()
    return df_salary

#########################add misc receipts code  by Wcao
def cap_input_data (s:pd.Series):
        mean = s.mean()
        std_dev = s.std()
        upper_bound = mean + 2 * std_dev
        lower_bound = mean - 2 * std_dev

        s = np.where(s>upper_bound,upper_bound, 
                                        np.where(s< lower_bound,lower_bound,s))
        s = np.where(s==0,mean,s)
        return s


########################## additional features by athena
def get_additional_features(env):        
    if env ==  'dev':
        db = 's3tablescatalog_cda_iceberg_pc_perf_merge_public_link'
    elif env =='prod':
        db = 's3tablescatalog_cda_iceberg_pc_prod_merge_public_link'
    query = '''
    with date_series AS (
    SELECT
        CAST(t.date AS DATE) AS full_date
    FROM
        UNNEST(SEQUENCE(DATE '2020-04-01', date_add('month', 3, current_date), INTERVAL '1' DAY)) AS t(date)
    ),
    
    date_range as (
    SELECT
        DATE_TRUNC('month', full_date) AS datebegin,
            date_trunc('month',  full_date) + interval '1' month - interval '1' day  AS dateend
    FROM
        date_series
    GROUP BY
        DATE_TRUNC('month', full_date)
    -- ORDER BY
    --     month_begin_date
        )    ,
    temp_policy as (
    select 
            distinct
            d.datebegin,
            c.AccountNumber,
            ((SUBSTRING(a.policynumber,1,1) || SUBSTRING(a.policynumber,5,9))) as PolicyNumber, 
            PeriodStart, 
            PeriodEnd, 
            a.TotalCostRPT,
            a.TotalCostRPT/DATE_DIFF('day', PeriodStart, PeriodEnd) AS DailyPremium,
            DATE_DIFF('day',
                CASE WHEN PeriodStart > d.datebegin THEN PeriodStart ELSE d.datebegin END,
                CASE WHEN PeriodEnd < d.dateend THEN PeriodEnd ELSE d.dateend END) + 1 AS DaysInMonthOverlap,
            DATE_DIFF('day',
                PeriodStart,
                CASE WHEN PeriodEnd < d.dateend THEN PeriodEnd ELSE d.dateend END) + 1 AS accum_days
            from pc_policyperiod a 
                join pc_policy b on a.PolicyID = b.ID 
                join pc_account c  on b.AccountID = c.ID
                join date_range d  on 1=1
            where 
            a.MostRecentModel =true  
            and TotalCostRPT > 0
            and cast(a.Periodstart as date) <= d.dateend AND cast(a.PeriodEnd as date) >= d.datebegin
    )

    select 
    --date_format(datebegin, '%Y-%m') as year_month,
    datebegin as year_month, 
    count(distinct AccountNumber) as account_count,
    count(distinct PolicyNumber) as policy_count,
    sum(TotalCostRPT ) as total_premium,   --given inflight policy all premium
    sum(DaysInMonthOverlap * DailyPremium ) as earned_premium_this_month,
    sum(accum_days * DailyPremium) as earned_premium_since_policy_start,
    sum(TotalCostRPT ) - sum(accum_days * DailyPremium) as unearned__premium_since_policy_start
    from temp_policy
    group by datebegin
    order by datebegin
    '''
    df_features =aws_athena_query(db, query, env)
    #save the additional feature for future reference 

    return df_features
    



###########################################################
#life inflow premium

def get_life_inflow_premiums(env):
    #TODO use athena query
    df_inflow_premiums  = pd.read_csv(r"\\hofile01\Enterprise Data Solutions\EDS Initiatives\Cash Flow Forecasting\Cashflow for life\Premiums\Data\cashflow_life_premium_txns_agg.csv" )
    df_inflow_premiums  = df_inflow_premiums[['Processed_Year', 'Processed_Month','Tre_AMTPROCESSED']]
    return df_inflow_premiums 

# life outlfow claims
def get_life_outflow_claims(env):
    #TODO use athena query
    df_outflow_claims  = pd.read_csv(r"\\hofile01\Enterprise Data Solutions\EDS Initiatives\Cash Flow Forecasting\Cashflow for life\Claims\Data\cashflow_life_claims_txns_agg.csv" ) 
    df_outflow_claims  = df_outflow_claims[['Processed_Year', 'Processed_Month','Tre_AMTDISBURSED']]
    return df_outflow_claims 


def get_life_receipts_adjustments(env):
    df_receipts = get_life_data(env, cf_type = 'life_inflow_misc_receipts')
    df_adjustments = get_life_data(env, cf_type = 'life_outflow_adjustments')
    df_receipts_adjustments = pd.concat([df_receipts, df_adjustments], ignore_index=True).sort_values(by=['Processed_Year', 'Processed_Month']).reset_index(drop=True)
    return df_receipts_adjustments


def get_life_data(env, cf_type = 'life_outflow_vendor'):
    ''''
    get data from life athena table, need to change hard code csv    '''
    
    dict_cf_type = {'life_outflow_vendor':'Vendor',
                    'life_inflow_misc_receipts':'Misc. Receipts',
                    'life_outflow_adjustments':'Adjustments',
                    'life_outflow_payroll':'Salary and Benefits',
                    'life_inflow_reinsurance':'Reinsurance'                  
                    } 
    #df = pd.read_csv(r"\\hofile01\Enterprise Data Solutions\EDS Initiatives\Cash Flow Forecasting\Cashflow for life\Misc Receipts_Vendor_Salary and Benefits\misc_receipts_vendor_salary_benefits.csv")
    df = pd.read_csv(r"\\hofile01\Enterprise Data Solutions\EDS Initiatives\Cash Flow Forecasting\Cashflow for life\Actuals\Cashflow_life_actuals.csv")
    df['Date'] = pd.to_datetime(df['Date'] )
    df= df[df['Cash_flow_type']==dict_cf_type[cf_type]].sort_values(by= 'Date')
    df =  df[df['Amount'].str.strip()!='-']  #remove - row
    df['Processed_Year'] = df['Date'].dt.year
    df['Processed_Month'] = df['Date'].dt.month
    df ['Amount'] = [ -1*float(i) if (float(i)<0) and dict_cf_type[cf_type] not in ['Misc. Receipts', 'Adjustments'] else float(i)  for i in df ['Amount'].apply(lambda x: x.replace(',', '')) ]  
    df = df [['Processed_Year', 'Processed_Month', 'Amount' ]]
    return df



#general function

def add_life_month_number(df):
    df = df.copy()
    dict_rename = dict(zip(df.columns[:2], ['Year', 'Month']))
    if str(df.iloc[:, 1].dtype)[:3]  == 'int':  #check if month column is number     
        df['year_month'] = pd.to_datetime(df.rename(columns=dict_rename)[['Year', 'Month']].assign(DAY=1))
        
    else: 
        raise ValueError ('convert month to number in the previous step')
    return df

def create_life_time_series(df, column, month_begin, month_end, cf_type):     #here name predict column generic
    df = df.copy()
    df.set_index('year_month', inplace=True, drop=True)  #create time series index
    df = df.iloc[:, 2:]  #get the predict column, remove year month
    df.columns = [column if i==0 else c for i, c in  enumerate(df.columns)]  #give generic name
    #month_begin, month_end =pd.to_datetime(month_begin), pd.to_datetime(month_end)
    df = df.loc[month_begin:month_end]  #remove Apr 2024 outlier and last partial month
    ### convert negative value, modify in life
    if cf_type !='life_receipts_adjustments':  #this type could have both postive and negative
        df[column] = df[column].apply(lambda x: -x if x< 0 else x)  #in athena table these are positive value, just in case     
       #use in time series, short name
    return df

def get_life_salary_additional_features(df_salary):
    payday_df = get_payday_datataframe()
    payday_df['trans_month'] =  pd.to_datetime(payday_df['trans_month'], format='%B').dt.month  #convert to month number
    df_salary = (df_salary.merge(payday_df, how = 'left', left_on=['Processed_Year','Processed_Month'], right_on = ['trans_year','trans_month'])
            .drop(['trans_year','trans_month'], axis=1)   #remove left side join column
            )
    #TODO this is dummy data, need to replace with real data
    dict_success_share = {
        'date': [ '2022-02-28', '2023-02-28', '2024-02-28', '2025-02-28'],  #will this automatically loaded?
        'amount':[1219213.75,
                1010019.08,
                3199405.16,
                4295804.42,
            ]
        }
    df_success_share = pd.DataFrame(dict_success_share)
    #add join column
    df_success_share['date'] = pd.to_datetime(df_success_share['date'])
    df_success_share['year'] = df_success_share['date'].dt.year
    df_success_share['month'] = df_success_share['date'].dt.month
    #rename column
    df_success_share.columns =['date', 'deduct_amount', 'year', 'month']  

    #merge two dataset and deduct the amt
    df_salary = df_salary.merge(df_success_share, how = 'left', left_on = ['Processed_Year', 'Processed_Month'], right_on = ['year', 'month'])
    df_salary['Amount'] = df_salary.apply(lambda x: x['Amount'] if pd.isna(x['deduct_amount']) else x['Amount'] - x['deduct_amount'], axis =1)
    df_salary = df_salary.drop(['date',	'deduct_amount', 'year','month'], axis =1)
    return df_salary 

def save_additional_features_dataset (df, cf_type, env):
    '''
    Save earlist additional feature dataset for troubleshooting 
    Parameters
    ----------
    df : DataFrame 
                The additonal feature dataframe.
    cf_type : str 
                cashflow type.
    env : str 
                environment you are using.
    Examples
    ----------
    save_additional_features_dataset (df_dummy, cf_type, env)
    '''
    datestamp = dt.datetime.now().strftime("%Y%m")
    s3_file_path = os.path.join('cash_flow_forecasting/model_misc/additional_data_hx/',f'cash_flow_model_{cf_type}_additional_features_{datestamp}.csv' )
    if s3_check_file_exists(s3_file_path, env):
        pass  #if file exist, not save
    else :  s3_save_and_upload_file(
            df,
            s3_file_path = s3_file_path,
            overwrite=False,
            env=env,
        )
    return 


# def prep_prophet_training_data (df, Begin_dt, end_dt, test_len,dict_column_rename ):
#     '''
#     This function is used to create train and test data for prophet hyperparameter tuning
#     test_len is month number hold for test
#     train_df_prep, test_df_prep = prep_prophet_training_data(df_subrogation_ts, '2020-05-01', '2024-09-01', 8, {"year_month": "ds", "Loss": "y"})
#     '''
#     df = df.copy()
#     _df_selected =select_data_range(df, Begin_dt, end_dt)    
#     _df_selected_reset = _df_selected.reset_index().rename(columns=dict_column_rename)
#     train_len = len(_df_selected_reset)- test_len
#     _train_df = _df_selected_reset[0 : train_len]
#     _test_df = _df_selected_reset[train_len : ]
#     return _train_df, _test_df, _df_selected_reset
    
