import pandas
from sqlalchemy import create_engine
import s3fs
from datetime import datetime
import os, sys
currentDirectory = os.path.dirname(os.path.realpath(__file__))
directoryOneLevelAbove = os.path.dirname(currentDirectory)
sys.path.append(directoryOneLevelAbove)
directoryTwoLevelsAbove = os.path.dirname(directoryOneLevelAbove)
sys.path.append(directoryTwoLevelsAbove)
from helper import helper_methods
#from com.flourish.production.helper import helper_methods
from helper import environment
from helper import get_master_data_file_from_s3_as_data_frame
from helper import db_operations
from pandas.api.types import is_integer_dtype
from pandas.api.types import is_bool_dtype

def execute_for_file_name_pattern(source, schemaName, rawTableName, sourceAlignedTableName, bucketName, destinationFolderInsideBucket, destinationFileNameWithoutExtension, fileNamePattern):
    updateRunDetails = {}
    try:
        condition = {'table_name': rawTableName, 'name_pattern': fileNamePattern}
        master_ingestion_raw_source_file_details = get_master_data_file_from_s3_as_data_frame.get_master_ingestion_raw_source_file_details_from_s3_for_source_with_condition(
            source,
            condition
        )

        if(len(master_ingestion_raw_source_file_details) == 1):
            master_ingestion_raw_source_file_detail = master_ingestion_raw_source_file_details.iloc[0]
            execute_common(source, schemaName, rawTableName, sourceAlignedTableName, bucketName, destinationFolderInsideBucket, destinationFileNameWithoutExtension, master_ingestion_raw_source_file_detail)

    except Exception as e:
        print(e)
        updateRunDetails["status_code"] = 'SAF'
        updateRunDetails["status_description"] = 'Source alignment unsuccessful'
    finally:
        if "id" in updateRunDetails:
            db_operations.update_run_details(
                schemaName,
                updateRunDetails
            )

def execute(source, schemaName, rawTableName, sourceAlignedTableName, bucketName, destinationFolderInsideBucket, destinationFileNameWithoutExtension):
    updateRunDetails = {}
    try:
        master_ingestion_raw_source_file_details = get_master_data_file_from_s3_as_data_frame.get_master_ingestion_raw_source_file_details_from_s3_for_source_with_condition(
            source,
            "table_name",
            rawTableName
        )

        if(len(master_ingestion_raw_source_file_details) == 1):

            master_ingestion_raw_source_file_detail = master_ingestion_raw_source_file_details.iloc[0]
            execute_common(source, schemaName, rawTableName, sourceAlignedTableName, bucketName, destinationFolderInsideBucket, destinationFileNameWithoutExtension, master_ingestion_raw_source_file_detail)

    except Exception as e:
        print(e)
        updateRunDetails["status_code"] = 'SAF'
        updateRunDetails["status_description"] = 'Source alignment unsuccessful'
    finally:    
        if "id" in updateRunDetails:
            db_operations.update_run_details(
                schemaName,
                updateRunDetails
            )

def execute_common(source, schemaName, rawTableName, sourceAlignedTableName, bucketName, destinationFolderInsideBucket, destinationFileNameWithoutExtension, master_ingestion_raw_source_file_detail):
    executionDate = datetime.now().strftime("%Y-%m-%d")
    executionDateWithTimeWithDashes = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    updateRunDetails = {}
    try:
        runDetails = db_operations.get_run_details_with_given_status_using_file_details_id(schemaName, master_ingestion_raw_source_file_detail.id, 'RTLS')

        if(len(runDetails) > 0):
            runDetail = runDetails.iloc[0]
            
            updateRunDetails["id"] = runDetail.id

            dBConnectionEngine = create_engine(environment.database_connection_string)
            dBConnection = dBConnectionEngine.connect()

            source_file_column_to_table_column_mapping = helper_methods.create_data_frame_from_csv('s3://'+bucketName+'/processed/'+schemaName+'/master_ingestion_source_file_column_to_table_column_mapping.csv', '', master_ingestion_raw_source_file_detail.file_encoding, 0)
            #source_file_column_to_table_column_mapping = source_file_column_to_table_column_mapping.replace(numpy.nan, '', regex=True)
            source_file_column_to_table_column_mapping = source_file_column_to_table_column_mapping.query("file_details_id == "+str(master_ingestion_raw_source_file_detail.id))
            source_file_column_to_table_column_mapping.sort_values('source_file_column_order')
            key_columns_in_table = source_file_column_to_table_column_mapping.query("file_details_id == "+str(master_ingestion_raw_source_file_detail.id)+" and is_key_column == 'Y'")
            print('********** key columns for table: ', rawTableName, ' is/are: ', key_columns_in_table.table_column_name.tolist())

            data_from_raw_table = pandas.read_sql('select * from '+schemaName+'.'+rawTableName, dBConnection)
            data_from_raw_table = helper_methods.remove_database_audit_columns_from_dataframe(data_from_raw_table)
            data_from_raw_table = helper_methods.change_datatype_of_columns_based_on_master_configuration(data_from_raw_table, source_file_column_to_table_column_mapping)
            #data_from_raw_table.set_index(key_columns_in_table.table_column_name.tolist(), inplace=True, verify_integrity=True)
            #data_from_raw_table = data_from_raw_table.replace(numpy.nan, '', regex=True)
            print('********** data_from_raw_table: ', len(data_from_raw_table))

            for column in data_from_raw_table.columns:
                if is_integer_dtype(data_from_raw_table[column]):
                    data_from_raw_table[column] = data_from_raw_table[column].astype('Int64')
                elif is_bool_dtype(data_from_raw_table[column]):
                    data_from_raw_table[column] = data_from_raw_table[column].astype('boolean')

            if(len(key_columns_in_table) > 0):
                data_from_source_aligned_table = pandas.read_sql('select * from '+schemaName+'.'+sourceAlignedTableName+' where row_status_code = 1', dBConnection)
                #data_from_source_aligned_table = helper_methods.remove_database_audit_columns_from_dataframe(data_from_source_aligned_table)
                data_from_source_aligned_table = helper_methods.convert_float_column_with_integers_and_nans_to_integer_column(data_from_source_aligned_table)
                #data_from_source_aligned_table.set_index(key_columns_in_table.table_column_name.tolist(), inplace=True, verify_integrity=True)
                #data_from_source_aligned_table = data_from_source_aligned_table.replace(numpy.nan, '', regex=True)
                print('********** data_from_source_aligned_table: ', len(data_from_source_aligned_table))

                for column in data_from_source_aligned_table.columns:
                    if is_integer_dtype(data_from_source_aligned_table[column]):
                        data_from_source_aligned_table[column] = data_from_source_aligned_table[column].astype('Int64')
                    elif is_bool_dtype(data_from_source_aligned_table[column]):
                        data_from_source_aligned_table[column] = data_from_source_aligned_table[column].astype('boolean')

                mergedDf1 = data_from_raw_table.merge(data_from_source_aligned_table.drop_duplicates(), on=key_columns_in_table.table_column_name.tolist(), how='outer', indicator=True)

                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned = pandas.DataFrame(mergedDf1[mergedDf1['_merge'] == 'left_only'])
                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned.drop(helper_methods.drop_columns_ending_with_suffix(to_insert_df_since_present_in_raw_but_not_present_in_source_aligned.columns, '_y'), axis=1, inplace=True)
                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned.drop(['_merge'], axis=1, inplace=True)
                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned.rename(columns=helper_methods.change_column_names_by_removing_suffix(to_insert_df_since_present_in_raw_but_not_present_in_source_aligned.columns, '_x'), inplace=True)
                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned["row_start_date"] = executionDateWithTimeWithDashes
                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned["row_end_date"] = environment.platform_maximum_date_value
                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned["row_status_code"] = 1
                to_insert_df_since_present_in_raw_but_not_present_in_source_aligned["load_timestamp"] = executionDateWithTimeWithDashes
                print('Rows to insert since they are present only in raw and not in source aligned')
                print(len(to_insert_df_since_present_in_raw_but_not_present_in_source_aligned))

                to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw = pandas.DataFrame(mergedDf1[mergedDf1['_merge'] == 'right_only'])
                to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw.drop(helper_methods.drop_columns_ending_with_suffix(to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw.columns, '_x'), axis=1, inplace=True)
                to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw.drop(['_merge'], axis=1, inplace=True)
                to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw.rename(columns=helper_methods.change_column_names_by_removing_suffix(to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw.columns, '_y'), inplace=True)
                print('Rows where nothing needs to be done since they are present only in source aligned and not in raw')
                print(len(to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw))

                concatenated_df_raw_and_source_aligned = pandas.concat([data_from_source_aligned_table, data_from_raw_table])
                grouped_df_on_all_columns_from_concatenated_df = concatenated_df_raw_and_source_aligned.fillna(0).groupby(list(data_from_raw_table.columns))
                to_do_nothing_df_idx = [x[0] for x in grouped_df_on_all_columns_from_concatenated_df.groups.values() if len(x) > 1]
                #print(to_do_nothing_df_idx)
                to_do_nothing_df_as_no_difference_between_raw_and_source_aligned_records = pandas.DataFrame(data_from_source_aligned_table.iloc[to_do_nothing_df_idx])
                print('Rows where nothing needs to be done since nothing is different between raw and source aligned.')
                print(len(to_do_nothing_df_as_no_difference_between_raw_and_source_aligned_records))

                to_update_status_to_inactive_df_since_newer_row_present_in_raw_table = pandas.DataFrame(mergedDf1[mergedDf1['_merge'] == 'both'])
                to_update_status_to_inactive_df_since_newer_row_present_in_raw_table.drop(helper_methods.drop_columns_ending_with_suffix(to_update_status_to_inactive_df_since_newer_row_present_in_raw_table.columns, '_x'), axis=1, inplace=True)
                to_update_status_to_inactive_df_since_newer_row_present_in_raw_table.drop(['_merge'], axis=1, inplace=True)
                to_update_status_to_inactive_df_since_newer_row_present_in_raw_table.rename(columns=helper_methods.change_column_names_by_removing_suffix(to_update_status_to_inactive_df_since_newer_row_present_in_raw_table.columns, '_y'), inplace=True)
                #print(to_update_status_to_inactive_df_since_newer_row_present_in_raw_table)
                to_update_status_to_inactive_df_since_newer_row_present_in_raw_table.set_index(key_columns_in_table.table_column_name.tolist(), inplace=True, verify_integrity=True)
                to_update_status_to_inactive_df_since_newer_row_present_in_raw_table_without_audit_columns = helper_methods.remove_database_audit_columns_from_dataframe(to_update_status_to_inactive_df_since_newer_row_present_in_raw_table)

                to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table = pandas.DataFrame(mergedDf1[mergedDf1['_merge'] == 'both'])
                to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table.drop(helper_methods.drop_columns_ending_with_suffix(to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table.columns, '_y'), axis=1, inplace=True)
                to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table.drop(['_merge'], axis=1, inplace=True)
                to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table.rename(columns=helper_methods.change_column_names_by_removing_suffix(to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table.columns, '_x'), inplace=True)
                #print(to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table)
                to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table.set_index(key_columns_in_table.table_column_name.tolist(), inplace=True, verify_integrity=True)
                to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table_without_audit_columns = helper_methods.remove_database_audit_columns_from_dataframe(to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table)

                comparison_df = (to_update_status_to_inactive_df_since_newer_row_present_in_raw_table_without_audit_columns.drop('check_sum', axis=1, errors='ignore').fillna(0)==to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table_without_audit_columns.drop('check_sum', axis=1, errors='ignore').fillna(0)).all(axis=1)

                to_update_status_to_inactive_df = pandas.DataFrame(to_update_status_to_inactive_df_since_newer_row_present_in_raw_table[~comparison_df])
                to_update_status_to_inactive_df.reset_index(inplace=True)
                to_update_status_to_inactive_df["row_end_date"] = executionDateWithTimeWithDashes
                to_update_status_to_inactive_df["row_status_code"] = 0
                print('Rows to update status to inactive. These are records that are present in source aligned but we have got a newer record in the raw file.')
                print(len(to_update_status_to_inactive_df))

                to_insert_with_active_status_df = pandas.DataFrame(to_insert_with_active_status_df_since_old_row_present_in_source_aligned_table[~comparison_df])
                to_insert_with_active_status_df.reset_index(inplace=True)
                to_insert_with_active_status_df["row_start_date"] = executionDateWithTimeWithDashes
                to_insert_with_active_status_df["row_end_date"] = environment.platform_maximum_date_value
                to_insert_with_active_status_df["row_status_code"] = 1
                to_insert_with_active_status_df["load_timestamp"] = executionDateWithTimeWithDashes
                print('Rows to insert with active status. These are records that have come in the raw file but where an older version is present in the source aligned table.')
                print(len(to_insert_with_active_status_df))

                final_data_to_insert = pandas.concat([to_insert_df_since_present_in_raw_but_not_present_in_source_aligned, to_do_nothing_df_since_present_in_source_aligned_but_not_present_in_raw, to_insert_with_active_status_df, to_update_status_to_inactive_df, to_do_nothing_df_as_no_difference_between_raw_and_source_aligned_records])

                if(len(final_data_to_insert) > 0):
                    s3 = s3fs.S3FileSystem(anon=False)
                    dataFileName = bucketName + '/' + destinationFolderInsideBucket + '/' + executionDate + '/' + destinationFileNameWithoutExtension + '_' + datetime.now().strftime('%Y%m%d%H%M%S') +'.csv'
                    with s3.open(dataFileName, 'w', encoding="utf-8") as f:
                        final_data_to_insert.to_csv(f, index=False, header=False, line_terminator='\n', encoding=('utf-8' if (pandas.isna(master_ingestion_raw_source_file_detail.file_encoding) or not master_ingestion_raw_source_file_detail.file_encoding) else master_ingestion_raw_source_file_detail.file_encoding))

                    dBConnection.execute("""
                        insert into %s select a.*, current_timestamp from %s a;
                    """ % (schemaName+'.'+sourceAlignedTableName+'_history', schemaName+'.'+sourceAlignedTableName))

                    metadataObject = MetaData(bind=dBConnectionEngine, reflect=True, schema=schemaName)
                    metadataObject.reflect()
                    source_aligned_table_object = Table(sourceAlignedTableName, metadataObject)
                    delete_from_source_aligned_table = source_aligned_table_object.delete().where(source_aligned_table_object.c.row_status_code == 1)
                    dBConnection.execute(delete_from_source_aligned_table)
                    
                    final_data_to_insert['load_timestamp'] = final_data_to_insert['load_timestamp'].astype(str)
                    final_data_to_insert['row_start_date'] = final_data_to_insert['row_start_date'].astype(str)
                    final_data_to_insert['row_end_date'] = final_data_to_insert['row_end_date'].astype(str)
                    final_data_to_insert.to_sql(name=sourceAlignedTableName, con=dBConnectionEngine, schema=schemaName, if_exists='append', index=False)
            else:
                print("No key columns so all the rows in the source aligned table will be kept as is and all the rows from the raw table will be inserted into source aligned table with ACTIVE status.")
                data_from_raw_table["row_start_date"] = executionDateWithTimeWithDashes
                data_from_raw_table["row_end_date"] = environment.platform_maximum_date_value
                data_from_raw_table["row_status_code"] = 1
                data_from_raw_table.insert(len(data_from_raw_table.columns)-3, "load_timestamp", executionDateWithTimeWithDashes)
                
                #print(data_from_raw_table.columns)

                final_data_to_insert = pandas.concat([data_from_raw_table])
                final_data_to_insert["load_timestamp"] = executionDateWithTimeWithDashes

                #print(final_data_to_insert.columns)

                if(len(final_data_to_insert) > 0):
                    s3 = s3fs.S3FileSystem(anon=False)
                    dataFileName = bucketName + '/' + destinationFolderInsideBucket + '/' + executionDate + '/' + destinationFileNameWithoutExtension + '_' + datetime.now().strftime('%Y%m%d%H%M%S') +'.csv'
                    with s3.open(dataFileName, 'w', encoding="utf-8") as f:
                        final_data_to_insert.to_csv(f, index=False, header=False, line_terminator='\n', encoding=('utf-8' if (pandas.isna(master_ingestion_raw_source_file_detail.file_encoding) or not master_ingestion_raw_source_file_detail.file_encoding) else master_ingestion_raw_source_file_detail.file_encoding))

                    dBConnection.execute("""
                        insert into %s select a.*, current_timestamp from %s a;
                    """ % (schemaName+'.'+sourceAlignedTableName+'_history', schemaName+'.'+sourceAlignedTableName))
                    final_data_to_insert.to_sql(name=sourceAlignedTableName, con=dBConnectionEngine, schema=schemaName, if_exists='append', index=False)

            dBConnection.close()
            del dBConnection
            dBConnectionEngine.dispose()
            del dBConnectionEngine

            updateRunDetails["status_code"] = 'SAS'
            updateRunDetails["status_description"] = 'Source alignment successful'
    except Exception as e:
        print(e)
        updateRunDetails["status_code"] = 'SAF'
        updateRunDetails["status_description"] = 'Source alignment unsuccessful'
    finally:
        if "id" in updateRunDetails:
            db_operations.update_run_details(
                schemaName,
                updateRunDetails
            )