from ._get_geo_dataset import *
from ._wind import *
from ._imports import *
from ._pconfig import pconfig

data_folder, output_folder = pconfig['data_folder'], pconfig['output_folder']

def get_claim_data(cat_no, cat_date):

    sql_claim = f'''
        declare @lossdate date = '{cat_date }'

    drop table if exists #tmp_claim     
    ;with cte as (
    select 
    CAT_NO, 
    POLICY_NO,
    [FILE_NO],
    [FILETYPE],
    LOSS_DT,
    POL_1ST_INS_DATE,
    RES_PREM_ADDRESS,
    CITY,
    STATE, 
    ZIP5,
    COUNTY, 
    EXP_AMT,   --what is this 
    COVA,          --  what is this value? 21718.15 in thsi view but show      limit 706000  select  COV_A_LIMIT from POLICY_HISTORY_VIEW where policy_no = '66024221VG'
    COVA_DED,  --what is this
    COVB,
    COVB_DED,  --what is this
    --COVC,
    --COVC_DED,
    --COVD,
    --COVD_DED,
    GEO_SCORE,
    GEO_MATCH_ADDR,
    GEO_X,
    GEO_Y,
    GEO_ADDR_TYPE,
    GEO_SPATIAL_GEOM,  --is this geo code?
    FILE_HDLR_BRANCH,
    row_number() over (partition by policy_no order by cova desc ) as rnk  --some record policy dup  '6512421122' 
    from 
    CLAIMS_HO_HISTPOLICY 
    where LOB = 'Home'
    and COV = 'HAIL'
    and CAT_NO  = '{cat_no}'
    and loss_dt = @lossdate
    )

    select
    CAT_NO, 
    POLICY_NO,
    [FILE_NO],
    [FILETYPE],
    LOSS_DT,
    POL_1ST_INS_DATE,
    RES_PREM_ADDRESS,
    CITY,
    STATE, 
    ZIP5,
    COUNTY, 
    EXP_AMT,   --what is this 
    COVA,          --  what is this value? 21718.15 in thsi view but show      limit 706000  select  COV_A_LIMIT from POLICY_HISTORY_VIEW where policy_no = '66024221VG'
    COVA_DED,  --what is this
    COVB,
    COVB_DED,  --what is this
    GEO_SCORE,
    GEO_MATCH_ADDR,
    GEO_X,
    GEO_Y,
    GEO_ADDR_TYPE,
    GEO_SPATIAL_GEOM,  --is this geo code?
    FILE_HDLR_BRANCH
    into #tmp_claim
    from cte
    where rnk =1 


    --select * from #tmp_claim

    --get account number
    drop table if exists #tmp_claim_with_account

    select 
    b.ACCOUNT_NO,
    b.POL_1ST_INS_DATE as POL_1ST_INS_DATE_pol, 
    b.GEO_X as geo_x_pol,
    b.GEO_Y as geo_y_pol,
    a.*
    into #tmp_claim_with_account
    from  #tmp_claim a      
    join POLICY_HISTORY_VIEW b  on a.POLICY_NO = b.POLICY_NO  and a.GEO_X =b.GEO_X and a.GEO_Y = b.GEO_Y     
    where 
    b.INFORCE_BEGIN_DT <= @lossdate AND b.INFORCE_END_DT >@lossdate

    select * from #tmp_claim_with_account
    '''
    df_claim = gis_query(sql_claim)
    return df_claim


def get_all_policies_span_over_date(cat_date):
    policy_query= f'''
        select 
        ACCOUNT_NO, 
        POLICY_NO,
        POLICY_EFF_DATE,
        INFORCE_BEGIN_DT,
        INFORCE_END_DT,
        RES_PREM_ADDRESS,
        [CITY],
        [STATE],
        [ZIP5],
        COV_A_LIMIT,
        COV_A_LIMIT,
        LINE_OF_BUSINESS,
        [GEO_X],
        [GEO_Y]
        from 
        POLICY_HISTORY_VIEW 
        Where INFORCE_BEGIN_DT <= '{cat_date}' AND INFORCE_END_DT >'{cat_date}'     --the latest is 2023-10-31 not updated after that
        and LINE_OF_BUSINESS = 'HOMEOWNERS'          
        order by INFORCE_BEGIN_DT desc
'''
    df_policy = gis_query(policy_query)
    return df_policy


# def get_policy_outside_swath (gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, df_policy, gdf_has_no_claim_in_hailswath):
#     cat_zipcode  = set(list(gdf_has_claim_inside_swath['ZIP5'].unique()) + list(gdf_has_claim_not_in_swath['ZIP5'].unique()) )
#     df_policy_outside_swath = df_policy[df_policy['ZIP5'].isin(cat_zipcode) ]  #this minimize the policy number to be matched with claim data, and also get the policy data outside swath but in the same zipcode, which can be used as control group for later modeling.
#     df_has_no_claim_policy_not_in_swath = df_policy_outside_swath[~df_policy_outside_swath['POLICY_NO'].isin(gdf_has_no_claim_in_hailswath['POLICY_NO'])]
#     return df_policy_outside_swath, df_has_no_claim_policy_not_in_swath


# def get_outside_swath_nearest_polygon(df_has_no_claim_policy_not_in_swath, gdf):
#     geometry = [Point(xy) for xy in zip(df_has_no_claim_policy_not_in_swath['GEO_X'], df_has_no_claim_policy_not_in_swath['GEO_Y'])]
#     # Convert to GeoDataFrame
#     gdf_has_no_claim_policy_not_in_swath = gpd.GeoDataFrame(df_has_no_claim_policy_not_in_swath, geometry=geometry, crs="EPSG:4326") 
#     gdf_has_no_claim_policy_not_in_swath = get_nearest_distance_polygon(gdf_has_no_claim_policy_not_in_swath, gdf)    
#     gdf_has_no_claim_policy_not_in_swath_at_8k = gdf_has_no_claim_policy_not_in_swath[gdf_has_no_claim_policy_not_in_swath['distance'] < 8000]
#     return gdf_has_no_claim_policy_not_in_swath, gdf_has_no_claim_policy_not_in_swath_at_8k

def get_policy_outside_swath (gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, df_policy, gdf_has_no_claim_in_hailswath):
    cat_zipcode  = set(list(gdf_has_claim_inside_swath['ZIP5'].unique()) + list(gdf_has_claim_not_in_swath['ZIP5'].unique()) )
    df_policy_in_zipcode = df_policy[df_policy['ZIP5'].isin(cat_zipcode) ]
    policy_no_in_swath = set(list(gdf_has_claim_inside_swath['POLICY_NO'].unique()) + list(gdf_has_no_claim_in_hailswath['POLICY_NO'].unique()) )
    df_policy_outside_swath = df_policy_in_zipcode[~df_policy_in_zipcode['POLICY_NO'].isin(policy_no_in_swath) ]
    df_has_no_claim_policy_not_in_swath = df_policy_outside_swath[~df_policy_outside_swath['POLICY_NO'].isin(gdf_has_claim_not_in_swath['POLICY_NO'])]
    return df_policy_outside_swath, df_has_no_claim_policy_not_in_swath

def convert_df_to_gdf(df_has_no_claim_policy_not_in_swath):
    geometry = [Point(xy) for xy in zip(df_has_no_claim_policy_not_in_swath['GEO_X'], df_has_no_claim_policy_not_in_swath['GEO_Y'])]
    gdf_has_no_claim_policy_not_in_swath = gpd.GeoDataFrame(df_has_no_claim_policy_not_in_swath, geometry=geometry, crs="EPSG:4326") 
    return gdf_has_no_claim_policy_not_in_swath

def get_outside_swath_nearest_polygon(df_has_no_claim_policy_not_in_swath, gdf):
    gdf_has_no_claim_policy_not_in_swath = convert_df_to_gdf(df_has_no_claim_policy_not_in_swath) 
    gdf_has_no_claim_policy_not_in_swath = get_nearest_distance_polygon(gdf_has_no_claim_policy_not_in_swath, gdf)    
    gdf_has_no_claim_policy_not_in_swath_at_8k = gdf_has_no_claim_policy_not_in_swath[gdf_has_no_claim_policy_not_in_swath['distance'] < 8000]
    return gdf_has_no_claim_policy_not_in_swath, gdf_has_no_claim_policy_not_in_swath_at_8k

def get_summary_level_data(cat_no, cat_date,gdf, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath
        ,df_policy_outside_swath , gdf_has_no_claim_policy_not_in_swath_at_8k
    ):
    df_cat_summary = pd.DataFrame( columns = ['cat', 'cat_date', 'total_shape_count', 'total_Shape_Area',
        'avg_Shape_Area', 'avg_hail_diameter_in', 'max_hail_diameter_in',
        'min_hail_diameter_in', 'std_hail_diameter_in',
        'total_hail_diameter_shape_area', 'avg_hail_diameter_shape_area',
        'avg_fill_opacity', 'total_claim_count_inside_swath',
        'total_policy_count_inside_swath', 'total_claim_count_outside_swatch',
        'total_claim_count_outside_swatch_distance_50_percentile',
        'total_claim_count_outside_swatch_distance_75_percentile',
        'total_claim_count_outside_swatch_distance_90_percentile',
        'total_policy_count_outside_swath_at_8k',
        'total_policy_count_outside_swath'
        ]
        )
    dict_summary = {
        'cat': cat_no,
        'cat_date':cat_date,        
        'total_shape_count': len(gdf),
        'total_Shape_Area' : gdf['Shape_Area'].sum(),
        'avg_Shape_Area': gdf['Shape_Area'].mean(),
        'avg_hail_diameter_in':gdf['diameter_in'].mean(),
        'max_hail_diameter_in':gdf['diameter_in'].max(),
        'min_hail_diameter_in': gdf['diameter_in'].min(),
        'std_hail_diameter_in': gdf['diameter_in'].std(),
        'total_hail_diameter_shape_area': (gdf['diameter_in']*gdf['Shape_Area']).sum(),
        'avg_hail_diameter_shape_area': (gdf['diameter_in']*gdf['Shape_Area']).mean(), 
        'avg_fill_opacity':gdf['fill_opacity'].mean(),
        'total_claim_count_inside_swath': len(gdf_has_claim_inside_swath),    #this will not available, preditor 
        'total_policy_count_inside_swath': len(gdf_has_claim_inside_swath) + len(gdf_has_no_claim_in_hailswath) , 
        'total_claim_count_outside_swath': len(gdf_has_claim_not_in_swath),  #this will not available  predictor
        'total_claim_count_outside_swath_distance_50_percentile': gdf_has_claim_not_in_swath['distance'].quantile(0.5),
        'total_claim_count_outside_swath_distance_75_percentile': gdf_has_claim_not_in_swath['distance'].quantile(0.75),
        'total_claim_count_outside_swath_distance_90_percentile': gdf_has_claim_not_in_swath['distance'].quantile(0.9),
        'total_policy_count_outside_swath_at_8k': len(gdf_has_no_claim_policy_not_in_swath_at_8k),
        'total_policy_count_outside_swath': len(df_policy_outside_swath)  
        }

    df_cat_summary.loc[len(df_cat_summary)] = dict_summary.values()
    return df_cat_summary

def save_raw_gdf (cat_no,cat_date,**kargs):  #use name : gd format
    for  n, gd in kargs.items():
        if n == 'gdf':
            gd.to_file(f'{data_folder}/{cat_no}_{cat_date}_hail_swath__all.gpkg', driver='GPKG')
        elif n == 'summary':
            gd.to_csv(f"{data_folder}/{cat_no}_{cat_date}_summary.csv", index=False)
        else:
            gd['geometry_wkt'] = gd['geometry'].apply(lambda p: p.wkt)
            df_no_geo = pd.DataFrame(gd.drop(columns='geometry'))
            df_no_geo.to_csv(f"{data_folder}/{cat_no}_{cat_date}_{n}.csv", index=False)
    return

def save_raw_gdf_test (cat_no,cat_date,**kargs):  #use name : gd format
    '''
    This is use to save to a specific test folder, not to mix with other data
    '''
    for  n, gd in kargs.items():
        if n == 'gdf':
            gd.to_file(f'{data_folder}/{cat_no}_{cat_date}_hail_swath__all.gpkg', driver='GPKG')
        elif n == 'summary':
            gd.to_csv(f"{data_folder}/{cat_no}_{cat_date}_summary.csv", index=False)
        else:
            gd['geometry_wkt'] = gd['geometry'].apply(lambda p: p.wkt)
            df_no_geo = pd.DataFrame(gd.drop(columns='geometry'))
            df_no_geo.to_csv(f"{data_folder}/{cat_no}_{cat_date}_{n}.csv", index=False)
    return

def stratified_sample_fixed_n(df, strata_cols, n_samples, random_state=None):
    """
    Performs stratified sampling on a pandas DataFrame based on multiple columns
    to achieve a fixed total number of samples.

    :param df: The input DataFrame.
    :param strata_cols: A list of column names to stratify by.
    :param n_samples: The fixed total number of samples to retrieve.
    :param random_state: Seed for the random number generator (for reproducibility).
    :return: A new DataFrame with the stratified sample.
    """
    # 1. Calculate the proportion of each stratum in the original data
    strata_counts = df.groupby(strata_cols).size()
    strata_proportions = strata_counts / len(df)
    
    # 2. Calculate the number of samples needed from each stratum to meet the total sample size
    # Rounding can cause the total number of samples to be slightly off.
    strata_sample_n = np.round(strata_proportions * n_samples).astype(int)
    
    # Adjust to ensure the total sample size matches exactly n_samples
    total_sampled = strata_sample_n.sum()
    if total_sampled != n_samples:
        diff = n_samples - total_sampled
        # Adjust the largest strata to make up the difference
        largest_strata = strata_sample_n.nlargest(abs(diff)).index
        strata_sample_n.loc[largest_strata[:abs(diff)]] += np.sign(diff).astype(int)
    
    # Check total samples after adjustment
    assert strata_sample_n.sum() == n_samples

    # 3. Apply the sampling within each group
    # The function below takes the calculated number of samples for each group
    def sample_group(group, n):
        # Ensure we don't try to sample more rows than available in the group
        n_actual = min(len(group), n)
        return group.sample(n=n_actual, random_state=random_state)

    # Convert the Series of counts to a dictionary for easier application
    sample_counts_dict = strata_sample_n.to_dict()

    # Apply the sampling to the original DataFrame using the pre-calculated sample counts
    sampled_df = df.groupby(strata_cols, group_keys=False).apply(
        lambda x: sample_group(x, sample_counts_dict[tuple(x.name) if isinstance(x.name, tuple) else x.name])
    )
    
    return sampled_df.sample(frac=1, random_state=random_state) 


def merge_gdf_data (cat_no, cat_date, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k):
    df_data = pd.DataFrame()

    for df in [gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath,  gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k]:    
        temp_df = pd.DataFrame()
        #join key    
        temp_df['ACCOUNT_NO'] = df['ACCOUNT_NO']
        temp_df['POLICY_NO'] = df['POLICY_NO']
        #cat info
        temp_df['LOSS_DT'] = cat_date
        temp_df['CAT_NO'] = cat_no
        #geo info
        temp_df['RES_PREM_ADDRESS'] = df['RES_PREM_ADDRESS']
        temp_df['CITY'] = df['CITY']    
        temp_df['STATE'] = df['STATE']
        temp_df['ZIP5'] = df['ZIP5']
        temp_df['GEO_X'] = df['GEO_X']
        temp_df['GEO_Y'] = df['GEO_Y']

        if (df is gdf_has_claim_inside_swath) or (df is gdf_has_no_claim_in_hailswath):
                temp_df['diameter_in'] = df['diameter_in']
        elif (df is gdf_has_claim_not_in_swath) or (df is gdf_has_no_claim_policy_not_in_swath_at_8k):
            temp_df['diameter_in'] = df['nearest_polygon_diameter_in']

        #group
        if (df is gdf_has_no_claim_in_hailswath) or (df is gdf_has_no_claim_policy_not_in_swath_at_8k):
            temp_df['TYPE'] = 'No_Claim_in_HailSwath' if df is gdf_has_no_claim_in_hailswath else 'No_Claim_outside_HailSwath'
            temp_df['Claim_Status'] = 'No_Claim'
        else:
            temp_df['TYPE'] = 'Claim_in_HailSwath' if df is gdf_has_claim_inside_swath else 'Claim_outside_HailSwath'
            temp_df['Claim_Status'] = 'Has_Claim'

        df_data = pd.concat([df_data, temp_df], ignore_index=True)
    return df_data

def prep_model_data(cat_no, cat_date, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k): 
    '''
    this is for training to get balanced dataset,  use stratified sample method to downsample
    '''    
    def gdf_drop_duplicate(gdf):
        gdf.sort_values(by=['ACCOUNT_NO', 'POLICY_NO','diameter_in'], ascending=False, inplace=True)
        gdf.drop_duplicates(subset=['ACCOUNT_NO', 'POLICY_NO'], keep='first', inplace=True)
        return 
    
    for g in [gdf_has_claim_inside_swath,  gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k]:
        g = g.copy()
        #print(len(g))

    for g in [gdf_has_claim_inside_swath,  gdf_has_no_claim_in_hailswath]:
        gdf_drop_duplicate(g)
        
    #under sampling stratify by hail size 
    gdf_has_no_claim_in_hailswath = stratified_sample_fixed_n(gdf_has_no_claim_in_hailswath, ['diameter_in', 'ZIP5'], len(gdf_has_claim_inside_swath), random_state=42)
    #print(len(gdf_has_no_claim_in_hailswath))
    gdf_has_no_claim_policy_not_in_swath_at_8k = stratified_sample_fixed_n(gdf_has_no_claim_policy_not_in_swath_at_8k, ['nearest_polygon_diameter_in', 'ZIP5'], len(gdf_has_claim_not_in_swath), random_state=42)
    #print(len(gdf_has_no_claim_policy_not_in_swath_at_8k))
    
    #merge data
    df_data = merge_gdf_data(cat_no, cat_date,  gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k)
    
    return df_data

def prep_model_data_full(cat_no, cat_date, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k): 
    '''
    this is for actual data for test and predicton to get all dataset,  not use stratified sample method to downsample
    '''  
    def gdf_drop_duplicate(gdf):
        gdf.sort_values(by=['ACCOUNT_NO', 'POLICY_NO','diameter_in'], ascending=False, inplace=True)
        gdf.drop_duplicates(subset=['ACCOUNT_NO', 'POLICY_NO'], keep='first', inplace=True)
        return 
    
    for g in [gdf_has_claim_inside_swath,  gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k]:
        g = g.copy()
        #print(len(g))

    for g in [gdf_has_claim_inside_swath,  gdf_has_no_claim_in_hailswath]:
        gdf_drop_duplicate(g)
        
    #under sampling stratify by hail size 
    #gdf_has_no_claim_in_hailswath = stratified_sample_fixed_n(gdf_has_no_claim_in_hailswath, ['diameter_in', 'ZIP5'], len(gdf_has_claim_inside_swath), random_state=42)
    #print(len(gdf_has_no_claim_in_hailswath))
    #gdf_has_no_claim_policy_not_in_swath_at_8k = stratified_sample_fixed_n(gdf_has_no_claim_policy_not_in_swath_at_8k, ['nearest_polygon_diameter_in', 'ZIP5'], len(gdf_has_claim_not_in_swath), random_state=42)
    #print(len(gdf_has_no_claim_policy_not_in_swath_at_8k))
    
    #merge data
    df_data = merge_gdf_data(cat_no, cat_date,  gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k)
    
    return df_data


def get_claim_history(cat_date):
    sql_claim_history = f'''
    declare @lossdate date = '{cat_date}'
    drop table if exists #tmp_account
    select 
    ACCOUNT_NO,
    Policy_No,
    POL_1ST_INS_DATE,
    GEO_X, 
    GEO_Y
    into #tmp_account
    from POLICY_HISTORY_VIEW b
    where 
    b.INFORCE_BEGIN_DT <= @lossdate AND b.INFORCE_END_DT >@lossdate

    --join with claim table to get claim filed

    select 
    ACCOUNT_NO, 
    count(*) as claim_count,
    sum(COVA) as claim_amount
    from #tmp_account a
    join CLAIMS_HO_HISTPOLICY b on  b.POL_1ST_INS_DATE = a.POL_1ST_INS_DATE and a.GEO_X = b.GEO_X and a.GEO_Y = b.GEO_Y
    where b.LOB = 'Home'
    and b.COV = 'HAIL'
    and b.LOSS_DT < @lossdate
    group by ACCOUNT_NO
    '''
    df_claim_history = gis_query(sql_claim_history)
    return df_claim_history
   

def get_pc_roof_data(df_data, cat_date):
    df_data1 = df_data.copy()
    pc_all_policy = f'''
    select 
    ((SUBSTRING(b.policynumber,1,1) + SUBSTRING(b.policynumber,5,9)))as PolicyNumber,
    c.RoofYearExt,
    b.periodstart,
    b.periodend
    from pc_policy  a  
    join pc_policyperiod b on a.id = b.PolicyID
    join pcx_dwelling_hoe  c on b.id = c.BranchID
    where  a.ProductCode = 'Homeowners' 
    and 
    b.periodstart <= '{cat_date}' and b.periodend > '{cat_date}'
    and ((SUBSTRING(b.policynumber,1,1) + SUBSTRING(b.policynumber,5,9))) is not null
    and RoofYearExt is not null
    '''
    df_all_policy_roof =  pc_query(pc_all_policy)
    df_all_policy_roof = df_all_policy_roof.drop_duplicates()
    df_all_policy_roof.rename(columns={'PolicyNumber': 'POLICY_NO'}, inplace=True)
    df_data1 = df_data1.merge(df_all_policy_roof, on='POLICY_NO', how='left')
    df_data1 = df_data1[pd.to_datetime(df_data1['LOSS_DT']).dt.year >= df_data1['RoofYearExt']]
    #get row number ranked by policy no, roof year desc 
    df_data1 = df_data1.sort_values(by=['POLICY_NO', 'RoofYearExt'], ascending=[True, False])
    df_data1['rank'] = df_data1.groupby('POLICY_NO').cumcount()+1
    #df_data1[df_data1['POLICY_NO'] == '6203421329']
    df_data1 = df_data1[df_data1['rank'] == 1]
    df_data1 = df_data1.drop(columns=['rank', 'periodstart', 'periodend'])
    #join back due some record not found in the database
    df_data = df_data.merge(df_data1[['POLICY_NO', 'RoofYearExt']], on='POLICY_NO', how='left') 
    return df_data


async def get_hail_storm_data(cat_no, cat_date, save_gdf = False):
    '''
    get training balanced dataset
    df = pd.DataFrame(columns=['Account_No', 'Policy_No', 'Loss_Dt', 'Cat_No', 'Address',
       'City', 'State', 'Zip5', 'Geo_X', 'Geo_Y', 'Hail_Diameter_In', 'Type',
       'Claim_Status', 'Wind_Speed', 'Historic_Claim_Count', 'Claim_Amount','Roof_Year'])
    for index, row in df_hail_storm_selected.iterrows():    
        cat_no = row['cat_no']
        cat_date = str(row['LOSS_DT'])[:10]
        print(f"Processing cat_no: {cat_no}, cat_date: {cat_date}")
        df_data = await get_hail_storm_data(cat_no, cat_date, save_gdf = True)
        df_data.columns = df.columns
        df = pd.concat([df, df_data], ignore_index=True)
    df.head()
    
    '''
    gdf = get_hail_storm_polygon(cat_no, cat_date, save_gdf = save_gdf)
    #get all claim related with the cat
    df_claim = get_claim_data(cat_no, cat_date)
    #get all policy span over cat date
    df_policy = get_all_policies_span_over_date(cat_date)
    #get spatial join between polygon and claim
    gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath = get_cat_geodataframe (gdf, df_claim, df_policy)
    gdf_has_claim_not_in_swath = get_nearest_distance_polygon(gdf_has_claim_not_in_swath, gdf)
    #policy outside the swath
    df_policy_outside_swath, df_has_no_claim_policy_not_in_swath = get_policy_outside_swath (gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, df_policy, gdf_has_no_claim_in_hailswath)
    #get nearest polygon for policy outside swath
    gdf_has_no_claim_policy_not_in_swath, gdf_has_no_claim_policy_not_in_swath_at_8k = get_outside_swath_nearest_polygon(df_has_no_claim_policy_not_in_swath, gdf)
    #get summary level data
    df_cat_summary = get_summary_level_data(cat_no, cat_date,gdf, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, df_policy_outside_swath, gdf_has_no_claim_policy_not_in_swath_at_8k)
    #save raw dataset
    dict_gdf = {
        'gdf': gdf,
        'summary': df_cat_summary, 
        'gdf_has_claim_inside_swath':gdf_has_claim_inside_swath, 
        'gdf_has_claim_not_in_swath':gdf_has_claim_not_in_swath, 
        'gdf_has_no_claim_in_hailswath':gdf_has_no_claim_in_hailswath,
        'gdf_has_no_claim_policy_not_in_swath_at_8k': gdf_has_no_claim_policy_not_in_swath_at_8k
    }
    save_raw_gdf (cat_no, cat_date, **dict_gdf)
    #combine all data for model, downsample negative class
    df_data = prep_model_data(cat_no, cat_date, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k)
    #add wind data
    #df_data['wind_speed'] = df_data.apply(lambda row: get_wind_data(row['GEO_Y'], row['GEO_X'], row['LOSS_DT'], row['LOSS_DT'], row['LOSS_DT']), axis=1)
    df_data = await get_wind_dataframe(df_data)
    #add claim history
    df_claim_history = get_claim_history(cat_date)
    df_data = df_data.merge(df_claim_history, on='ACCOUNT_NO', how='left')
    #add pc roof data
    df_data = get_pc_roof_data(df_data, cat_date)
    return df_data


async def get_hail_storm_data_full(cat_no, cat_date, save_gdf = False):
    '''
    get full unbalanced dataset
    df = pd.DataFrame(columns=['Account_No', 'Policy_No', 'Loss_Dt', 'Cat_No', 'Address',
       'City', 'State', 'Zip5', 'Geo_X', 'Geo_Y', 'Hail_Diameter_In', 'Type',
       'Claim_Status', 'Wind_Speed', 'Historic_Claim_Count', 'Claim_Amount','Roof_Year'])
    for index, row in df_hail_storm_selected.iterrows():    
        cat_no = row['cat_no']
        cat_date = str(row['LOSS_DT'])[:10]
        print(f"Processing cat_no: {cat_no}, cat_date: {cat_date}")
        df_data = await get_hail_storm_data_full(cat_no, cat_date, save_gdf = True)
        df_data.columns = df.columns
        df = pd.concat([df, df_data], ignore_index=True)
    df.head()
    
    '''
    gdf = get_hail_storm_polygon(cat_no, cat_date, save_gdf = save_gdf)
    #get all claim related with the cat
    df_claim = get_claim_data(cat_no, cat_date)
    #get all policy span over cat date
    df_policy = get_all_policies_span_over_date(cat_date)
    #get spatial join between polygon and claim
    gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath = get_cat_geodataframe (gdf, df_claim, df_policy)
    gdf_has_claim_not_in_swath = get_nearest_distance_polygon(gdf_has_claim_not_in_swath, gdf)
    #policy outside the swath
    df_policy_outside_swath, df_has_no_claim_policy_not_in_swath = get_policy_outside_swath (gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, df_policy, gdf_has_no_claim_in_hailswath)
    #get nearest polygon for policy outside swath
    gdf_has_no_claim_policy_not_in_swath, gdf_has_no_claim_policy_not_in_swath_at_8k = get_outside_swath_nearest_polygon(df_has_no_claim_policy_not_in_swath, gdf)
    #get summary level data
    df_cat_summary = get_summary_level_data(cat_no, cat_date,gdf, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, df_policy_outside_swath, gdf_has_no_claim_policy_not_in_swath_at_8k)
    #save raw dataset
    dict_gdf = {
        'gdf': gdf,
        'summary': df_cat_summary, 
        'gdf_has_claim_inside_swath':gdf_has_claim_inside_swath, 
        'gdf_has_claim_not_in_swath':gdf_has_claim_not_in_swath, 
        'gdf_has_no_claim_in_hailswath':gdf_has_no_claim_in_hailswath,
        'gdf_has_no_claim_policy_not_in_swath_at_8k': gdf_has_no_claim_policy_not_in_swath_at_8k
    }
    save_raw_gdf_test (cat_no, cat_date, **dict_gdf)
    #combine all data for model, downsample negative class
    df_data = prep_model_data_full(cat_no, cat_date, gdf_has_claim_inside_swath, gdf_has_claim_not_in_swath, gdf_has_no_claim_in_hailswath, gdf_has_no_claim_policy_not_in_swath_at_8k)
    #add wind data
    #df_data['wind_speed'] = df_data.apply(lambda row: get_wind_data(row['GEO_Y'], row['GEO_X'], row['LOSS_DT'], row['LOSS_DT'], row['LOSS_DT']), axis=1)
    df_data = await get_wind_dataframe(df_data)
    #add claim history
    df_claim_history = get_claim_history(cat_date)
    df_data = df_data.merge(df_claim_history, on='ACCOUNT_NO', how='left')
    #add pc roof data
    df_data = get_pc_roof_data(df_data, cat_date)
    return df_data


