#!/usr/bin/env python3
# coding: utf-8

from multiprocessing import Pool

from napoleontoolbox.file_saver import dropbox_file_saver
from napoleontoolbox.signal import signal_generator
from napoleontoolbox.connector import napoleon_connector
from napoleontoolbox.signal import signal_utility
import json

from binance.websockets import BinanceSocketManager
from functools import partial
from binance.client import Client
import numpy as np
from napoleontoolbox.connector import napoleon_s3_connector
import pandas as pd

from datetime import datetime, timedelta

import hashlib

from napoleontoolbox.connector import napoleon_connector
from napoleontoolbox.bot import binance_bot

def pair_handling_message(fetcher, msg, compute_minute_signal=True, compute_day_signal=True, compute_parallel = False):
    high_dict = fetcher.pair_dict['high']
    low_dict = fetcher.pair_dict['low']
    volu_dict = fetcher.pair_dict['volu']
    close_dict = fetcher.pair_dict['close']
    open_dict = fetcher.pair_dict['open']

    hour_high_dict = fetcher.pair_dict['hour_high']
    hour_low_dict = fetcher.pair_dict['hour_low']
    hour_volu_dict = fetcher.pair_dict['hour_volu']
    hour_close_dict = fetcher.pair_dict['hour_close']
    hour_open_dict = fetcher.pair_dict['hour_open']

    day_high_dict = fetcher.pair_dict['day_high']
    day_low_dict = fetcher.pair_dict['day_low']
    day_volu_dict = fetcher.pair_dict['day_volu']
    day_close_dict = fetcher.pair_dict['day_close']
    day_open_dict = fetcher.pair_dict['day_open']

    trade_time_unix_timestamp = msg['T']/1000
    trade_time_unix_timestamp_minute = trade_time_unix_timestamp - (trade_time_unix_timestamp % 60)
    trade_time_unix_timestamp_hour = trade_time_unix_timestamp - (trade_time_unix_timestamp % 3600)
    trade_time_unix_timestamp_day = trade_time_unix_timestamp - (trade_time_unix_timestamp % (3600*24))

    price = float(msg['p'])

    actual_day_open = day_open_dict.get(trade_time_unix_timestamp_day, np.nan)
    if np.isnan(actual_day_open):
        # we open a new hour
        if len(day_high_dict) > 1 or len(day_low_dict) > 1 or len(day_volu_dict) > 1 or len(
                day_close_dict) > 1 or len(day_open_dict) > 1:
            raise Exception('Trouble in paradise we enter a new day with no empty dictionaries')
        if len(day_high_dict) > 0 and len(day_low_dict) > 0 and len(day_volu_dict) > 0 and len(day_close_dict) > 0 and len(day_open_dict) > 0:
            key_day_ts = next(iter(day_high_dict))
            day_dt_object = datetime.utcfromtimestamp(key_day_ts)
            day_name = fetcher.pair + '_' + day_dt_object.strftime('%d_%b_%Y_%H_%M')
            print('appending one day row ' + day_name)

            day_new_data_lst = [key_day_ts, day_open_dict[key_day_ts], day_high_dict[key_day_ts],
                                 day_low_dict[key_day_ts],
                                 day_close_dict[key_day_ts], day_volu_dict[key_day_ts]]
            day_new_data = np.array(day_new_data_lst)
            #print('new daily data')
            #print(day_new_data)
            if fetcher.day_np is None:
                fetcher.day_np = day_new_data
            else:
                fetcher.day_np = np.vstack((fetcher.day_np, day_new_data))

            if compute_day_signal:
                if compute_parallel:
                    ia_day_signal = fetcher.launchDayParallelPool()
                else:
                    ia_day_signal = fetcher.launchDaySequential()

            if (len(fetcher.day_np.shape) > 1 and len(fetcher.day_np) > 3):
                #print('last three days')
                #print(fetcher.day_np[-3:, :])
                print('day ia new signal computation for the bot ' + str(ia_day_signal))
            print('updating bot position for pair '+ str(fetcher.pair) + ' and signal '+str(ia_day_signal))
            fetcher.bot.update_position(fetcher.pair, ia_day_signal)

            if fetcher.hour_np is not None and len(fetcher.hour_np)>0:
                print('persisting hours dataframe for the previous day')
                #fetcher.upload_last_hour_60_minutes_dataframe()
                fetcher.upload_last_day_24_hours_signals_dataframe()
                fetcher.reset_hour_np_to_lookback_window()

            if fetcher.day_np is not None and len(fetcher.day_np)>0:
                print('persisting days dataframe')
                fetcher.upload_days_signals_dataframe()


        # we clear every former minutes
        day_high_dict.clear()
        day_low_dict.clear()
        day_volu_dict.clear()
        day_close_dict.clear()
        day_open_dict.clear()
        day_open_dict[trade_time_unix_timestamp_day] = price

    day_close_dict[trade_time_unix_timestamp_day] = price
    day_actual_high = day_high_dict.get(trade_time_unix_timestamp_day, 0)
    if price > day_actual_high:
        day_high_dict[trade_time_unix_timestamp_day]=price
    day_actual_low = day_low_dict.get(trade_time_unix_timestamp_day, np.inf)
    if price < day_actual_low:
        day_low_dict[trade_time_unix_timestamp_day]=price
    day_actual_volu = day_volu_dict.get(trade_time_unix_timestamp_day, 0)
    day_volu_dict[trade_time_unix_timestamp_day] = day_actual_volu + float(msg['q'])

    actual_hour_open = hour_open_dict.get(trade_time_unix_timestamp_hour, np.nan)
    if np.isnan(actual_hour_open):
        # we open a new hour
        if len(hour_high_dict)>1 or len(hour_low_dict)>1 or len(hour_volu_dict)>1 or len(hour_close_dict)>1 or len(hour_open_dict)>1:
            raise Exception('Trouble in paradise we enter a new hour with no empty dictionaries')
        if len(hour_high_dict) > 0 and len(hour_low_dict) > 0 and len(hour_volu_dict) > 0 and len(hour_close_dict) > 0 and len(hour_open_dict) > 0:
            key_hour_ts = next(iter(hour_high_dict))
            hour_dt_object = datetime.utcfromtimestamp(key_hour_ts)
            hour_name = fetcher.pair + '_' + hour_dt_object.strftime('%d_%b_%Y_%H_%M')
            print('appending one hour row '+hour_name)

            hour_new_data_lst = [key_hour_ts, hour_open_dict[key_hour_ts], hour_high_dict[key_hour_ts], hour_low_dict[key_hour_ts],
                            hour_close_dict[key_hour_ts], hour_volu_dict[key_hour_ts]]
            hour_new_data = np.array(hour_new_data_lst)
            #print('new hour data')
            #print(hour_new_data)
            if fetcher.hour_np is None:
                fetcher.hour_np = hour_new_data
            else:
                fetcher.hour_np = np.vstack((fetcher.hour_np, hour_new_data))

            if (len(fetcher.hour_np.shape)>1 and len(fetcher.hour_np)>3):
                #print('last three hours')
                #print(fetcher.hour_np[-3:,:])
                print('no hourly ia signal yet')
            #fetcher.minute_df.loc[len(fetcher.minute_df)] = new_data
            if fetcher.minute_np is not None and len(fetcher.minute_np)>0:
                print('persisting minutes dataframe for the previous hour')
                fetcher.upload_last_hour_60_minutes_signals_dataframe()
                fetcher.reset_minute_np_to_lookback_window()
        # we clear every former hours
        hour_high_dict.clear()
        hour_low_dict.clear()
        hour_volu_dict.clear()
        hour_close_dict.clear()
        hour_open_dict.clear()
        hour_open_dict[trade_time_unix_timestamp_hour] = price


    hour_close_dict[trade_time_unix_timestamp_hour] = price
    hour_actual_high = hour_high_dict.get(trade_time_unix_timestamp_hour, 0)
    if price > hour_actual_high:
        hour_high_dict[trade_time_unix_timestamp_hour]=price
    hour_actual_low = hour_low_dict.get(trade_time_unix_timestamp_hour, np.inf)
    if price < hour_actual_low:
        hour_low_dict[trade_time_unix_timestamp_hour]=price
    hour_actual_volu = hour_volu_dict.get(trade_time_unix_timestamp_hour, 0)
    hour_volu_dict[trade_time_unix_timestamp_hour] = hour_actual_volu + float(msg['q'])


    actual_open = open_dict.get(trade_time_unix_timestamp_minute, np.nan)
    if np.isnan(actual_open):
        # we open a new minute
        if len(high_dict)>1 or len(low_dict)>1 or len(volu_dict)>1 or len(close_dict)>1 or len(open_dict)>1:
            raise Exception('Trouble in paradise we enter a new minute with no empty dictionaries')
        if len(high_dict) > 0 and len(low_dict) > 0 and len(volu_dict) > 0 and len(close_dict) > 0 and len(open_dict) > 0:
            key_ts = next(iter(high_dict))
            minute_dt_object = datetime.utcfromtimestamp(key_ts)
            minute_name = fetcher.pair + '_' + minute_dt_object.strftime('%d_%b_%Y_%H_%M')
            print('appending one minute row '+minute_name)

            new_data_lst = [key_ts, open_dict[key_ts], high_dict[key_ts], low_dict[key_ts],
                            close_dict[key_ts], volu_dict[key_ts]]
            new_data = np.array(new_data_lst)
            #print('new minute data')
            #print(new_data)
            if fetcher.minute_np is None:
                fetcher.minute_np = new_data
            else:
                fetcher.minute_np = np.vstack((fetcher.minute_np, new_data))
            if compute_minute_signal:
                if compute_parallel:
                    ia_minute_signal = fetcher.launchMinuteParallelPool()
                else:
                    ia_minute_signal = fetcher.launchMinuteSequential()
            if (len(fetcher.minute_np.shape)>1 and len(fetcher.minute_np)>3):
                #print('last three minutes '+minute_name)
                #print(fetcher.minute_np[-3:,:])
                print('minute ia signal ' + str(ia_minute_signal))
        # we clear every former minutes
        high_dict.clear()
        low_dict.clear()
        volu_dict.clear()
        close_dict.clear()
        open_dict.clear()
        open_dict[trade_time_unix_timestamp_minute] = price
    close_dict[trade_time_unix_timestamp_minute] = price
    actual_high = high_dict.get(trade_time_unix_timestamp_minute, 0)
    if price > actual_high:
        high_dict[trade_time_unix_timestamp_minute]=price
    actual_low = low_dict.get(trade_time_unix_timestamp_minute, np.inf)
    if price < actual_low:
        low_dict[trade_time_unix_timestamp_minute]=price
    actual_volu = volu_dict.get(trade_time_unix_timestamp_minute, 0)
    volu_dict[trade_time_unix_timestamp_minute] = actual_volu + float(msg['q'])

class RealTimeSignalEnsemblingParalleLauncher():
    def __init__(self, min_calib_starting_date=None, min_calib_running_date=None, day_calib_starting_date=None, day_calib_running_date=None, drop_token='', dropbox_backup=True, BINANCE_PUBLIC='', BINANCE_SECRET='', AWS_PUBLIC='', AWS_SECRET='', AWS_REGION='',bucket='', local_root_directory='../data',underlying = None, pair = None, min_calib_selected_algo='', day_calib_selected_algo='', user='napoleon',  db_path_suffix = '_run.sqlite', list_pkl_file_suffix = 'my_list.pkl', backed_up_signal_histo_suffix='backed_up_signal_histo.pkl', load_daily_histo = True, from_cryptocompare=False):
        self.min_calib_starting_date = min_calib_starting_date
        self.min_calib_running_date = min_calib_running_date

        self.day_calib_starting_date = day_calib_starting_date
        self.day_calib_running_date = day_calib_running_date

        self.pair = pair
        self.BINANCE_PUBLIC = BINANCE_PUBLIC
        self.BINANCE_SECRET = BINANCE_SECRET

        self.bot = binance_bot.NaPoleonBinanceFutureBot(BINANCE_PUBLIC, BINANCE_SECRET)

        self.dropbox_backup =dropbox_backup
        self.drop_token =drop_token
        self.dbx = dropbox_file_saver.NaPoleonDropboxConnector(drop_token=drop_token,dropbox_backup=dropbox_backup)

        self.AWS_PUBLIC = AWS_PUBLIC
        self.AWS_SECRET = AWS_SECRET
        self.AWS_REGION = AWS_REGION
        self.bucket = bucket

        self.client = Client(BINANCE_PUBLIC, BINANCE_SECRET)
        self.aws_client = napoleon_s3_connector.NapoleonS3Connector(AWS_PUBLIC,AWS_SECRET,region=AWS_REGION)
        self.pair = pair
        self.generic_columns = ['ts', 'open', 'high', 'low', 'close', 'volumefrom']
        self.minute_df = pd.DataFrame(columns=self.generic_columns)

        self.hour_df = pd.DataFrame(columns=self.generic_columns)
        self.day_df = pd.DataFrame(columns=self.generic_columns)

        self.minute_np = None
        self.minute_signal_np = None
        self.hour_np = None
        self.day_np = None
        self.day_signal_np = None

        self.pair_dict = {}
        self.pair_dict['high'] = {}
        self.pair_dict['low'] = {}
        self.pair_dict['volu'] = {}
        self.pair_dict['close'] = {}
        self.pair_dict['open'] = {}
        self.pair_dict['hour_high'] = {}
        self.pair_dict['hour_low'] = {}
        self.pair_dict['hour_volu'] = {}
        self.pair_dict['hour_close'] = {}
        self.pair_dict['hour_open'] = {}

        self.pair_dict['day_high'] = {}
        self.pair_dict['day_low'] = {}
        self.pair_dict['day_volu'] = {}
        self.pair_dict['day_close'] = {}
        self.pair_dict['day_open'] = {}

        self.seed = 0
        self.local_root_directory = local_root_directory

        self.underlying = underlying

        self.list_pkl_file_suffix = list_pkl_file_suffix

        self.min_calib_selected_algo=min_calib_selected_algo
        self.day_calib_selected_algo = day_calib_selected_algo

        self.min_calib_dates_stub = self.min_calib_starting_date.strftime('%d_%b_%Y') + '_' + self.min_calib_running_date.strftime('%d_%b_%Y')
        self.min_calib_list_pkl_file_name = self.min_calib_dates_stub + '_' + self.underlying + '_minutely_' + self.min_calib_selected_algo + self.list_pkl_file_suffix

        self.day_calib_dates_stub = self.day_calib_starting_date.strftime('%d_%b_%Y') + '_' + self.day_calib_running_date.strftime('%d_%b_%Y')
        self.day_calib_list_pkl_file_name = self.day_calib_dates_stub + '_' + self.underlying + '_daily_' + self.day_calib_selected_algo + self.list_pkl_file_suffix

        print('minutes_selected algos')
        print(self.min_calib_list_pkl_file_name)

        print('days_selected algos')
        print(self.day_calib_list_pkl_file_name)

        self.backed_up_signal_histo_suffix = backed_up_signal_histo_suffix


        if self.dropbox_backup:
            self.min_calib_signals_list =  self.dbx.download_pkl(self.min_calib_list_pkl_file_name)
            self.day_calib_signals_list =  self.dbx.download_pkl(self.day_calib_list_pkl_file_name)

        else:
            self.min_calib_signals_list = napoleon_connector.load_pickled_list(local_root_directory=self.local_root_directory,
                                                                               list_pkl_file_name=self.min_calib_list_pkl_file_name)
            self.day_calib_signals_list = napoleon_connector.load_pickled_list(local_root_directory=self.local_root_directory,
                                                                               list_pkl_file_name=self.day_calib_list_pkl_file_name)
        self.min_calib_signals_list = list(self.min_calib_signals_list )
        self.min_calib_signals_list.sort()

        self.day_calib_signals_list = list(self.day_calib_signals_list )
        self.day_calib_signals_list.sort()

        self.minute_signal_mapping={}
        minute_max_lookback_window = 0
        for me_signal in self.min_calib_signals_list:
            run_json_string = signal_utility.recover_to_sql_column_format(me_signal)
            params = json.loads(run_json_string)
            salty = str(int(hashlib.sha1(run_json_string.encode('utf-8')).hexdigest(), 16) % (10 ** 8))
            params = json.loads(run_json_string)
            readable_label = params['signal_type'] + str(params['trigger']) + salty
            self.minute_signal_mapping[me_signal]=readable_label
            if params['lookback_window']>minute_max_lookback_window:
                minute_max_lookback_window=params['lookback_window']
        self.minute_max_lookback_window = minute_max_lookback_window

        self.day_signal_mapping={}
        day_max_lookback_window = 0
        self.day_lookback_windows = []
        for me_signal in self.day_calib_signals_list:
            run_json_string = signal_utility.recover_to_sql_column_format(me_signal)
            params = json.loads(run_json_string)
            salty = str(int(hashlib.sha1(run_json_string.encode('utf-8')).hexdigest(), 16) % (10 ** 8))
            params = json.loads(run_json_string)
            readable_label = params['signal_type'] + str(params['trigger']) + salty
            self.day_signal_mapping[me_signal]=readable_label
            self.day_lookback_windows.append(params['lookback_window'])
            if params['lookback_window']>day_max_lookback_window:
                day_max_lookback_window=params['lookback_window']
        self.day_max_lookback_window = day_max_lookback_window

        self.user = user
        self.db_path_suffix = db_path_suffix
        self.filename =  user + db_path_suffix
        self.db_path = self.local_root_directory + self.filename
        self.runs = []
        self.totalRow = 0
        self.empty_runs_to_investigate = []
        self.dbx = dropbox_file_saver.NaPoleonDropboxConnector(drop_token=drop_token,dropbox_backup=dropbox_backup)
        self.last_minute = None
        self.last_day = None
        self.load_daily_histo = load_daily_histo
        self.from_cryptocompare = from_cryptocompare
        if self.load_daily_histo:
            print('loading daily historical signals')
            self.reload_daily_histo(from_cryptocompare = self.from_cryptocompare)


    def get_aws_last_past_minutes(self, nb_last_hours = 24):
        date_now = datetime.utcnow()
        files_to_fetch = []
        #minute_name = self.pair + '_' + date_now.strftime('%d_%b_%Y_%H_%M')
        #files_to_fetch.append(minute_name)
        d = date_now
        for i in range(nb_last_hours):
            d = d - timedelta(hours=1)
            minute_name = self.pair + '_' + d.strftime('%d_%b_%Y_%H')
            files_to_fetch.append(minute_name)
        aggregated_df = None
        for me_file in files_to_fetch:
            df = None
            try:
                df = self.aws_client.download_dataframe_from_csv(self.bucket, me_file)
            except Exception as e:
                print(e)
                df = None
            if aggregated_df is not None:
                if df is not None:
                    print(me_file)
                    print(df.shape)
                    aggregated_df = pd.concat([aggregated_df, df])
            if aggregated_df is None:
                if df is not None:
                    print(df.shape)
                    aggregated_df = df

        print('size fetched')
        print(aggregated_df.shape)

        aggregated_df = aggregated_df.sort_values(by=['date'])
        aggregated_df['date'] = pd.to_datetime(aggregated_df['date'])
        aggregated_df = aggregated_df.set_index(aggregated_df['date'])
        aggregated_df = aggregated_df.drop(columns=['date'])
        test = aggregated_df.pivot_table(index=['date'], aggfunc='size')
        print('max duplication')
        print(test.max())

        aggregated_df = aggregated_df.drop_duplicates()
        test = aggregated_df.pivot_table(index=['date'], aggfunc='size')
        print('max duplication  after dropping duplicates')
        print(test.max())

        print('size fetched after dropping duplicates')
        print(aggregated_df.shape)
        return aggregated_df

    def get_aws_last_past_hours(self, nb_last_days = 2):
        date_now = datetime.utcnow()
        files_to_fetch = []

        d = date_now
        for i in range(nb_last_days):
            d = d - timedelta(days=1)
            hour_name = self.pair + '_' + d.strftime('%d_%b_%Y')
            files_to_fetch.append(hour_name)
        aggregated_df = None
        for me_file in files_to_fetch:
            df = None
            try:
                df = self.aws_client.download_dataframe_from_csv(self.bucket, me_file)
            except Exception as e:
                print(e)
                df = None
            if aggregated_df is not None:
                if df is not None:
                    print(me_file)
                    print(df.shape)
                    aggregated_df = pd.concat([aggregated_df, df])
            if aggregated_df is None:
                if df is not None:
                    print(df.shape)
                    aggregated_df = df

        print('size fetched')
        print(aggregated_df.shape)

        aggregated_df = aggregated_df.sort_values(by=['date'])
        aggregated_df['date'] = pd.to_datetime(aggregated_df['date'])
        aggregated_df = aggregated_df.set_index(aggregated_df['date'])
        aggregated_df = aggregated_df.drop(columns=['date'])
        test = aggregated_df.pivot_table(index=['date'], aggfunc='size')
        print('max duplication')
        print(test.max())

        aggregated_df = aggregated_df.drop_duplicates()
        test = aggregated_df.pivot_table(index=['date'], aggfunc='size')
        print('max duplication  after dropping duplicates')
        print(test.max())

        print('size fetched after dropping duplicates')
        print(aggregated_df.shape)
        return aggregated_df

    def get_aws_last_past_days(self, nb_last_days = 1):
        date_now = datetime.utcnow()
        files_to_fetch = []

        d = date_now
        for i in range(nb_last_days):
            d = d - timedelta(days=1)
            day_name = self.pair + '_' + d.strftime('%d_%b_%Y') + '_dayohlc'
            files_to_fetch.append(day_name)
        aggregated_df = None
        for me_file in files_to_fetch:
            df = None
            try:
                df = self.aws_client.download_dataframe_from_csv(self.bucket, me_file)
            except Exception as e:
                print(e)
                df = None
            if aggregated_df is not None:
                if df is not None:
                    print(me_file)
                    print(df.shape)
                    aggregated_df = pd.concat([aggregated_df, df])
            if aggregated_df is None:
                if df is not None:
                    print(df.shape)
                    aggregated_df = df

        print('size fetched')
        print(aggregated_df.shape)

        aggregated_df = aggregated_df.sort_values(by=['date'])
        aggregated_df['date'] = pd.to_datetime(aggregated_df['date'])
        aggregated_df = aggregated_df.set_index(aggregated_df['date'])
        aggregated_df = aggregated_df.drop(columns=['date'])
        test = aggregated_df.pivot_table(index=['date'], aggfunc='size')
        print('max duplication')
        print(test.max())

        aggregated_df = aggregated_df.drop_duplicates()
        test = aggregated_df.pivot_table(index=['date'], aggfunc='size')
        print('max duplication  after dropping duplicates')
        print(test.max())

        print('size fetched after dropping duplicates')
        print(aggregated_df.shape)
        return aggregated_df

    def recompute_daily_histo(self, from_cryptocompare = False):
        if from_cryptocompare:
            print('recomputing historical up to '+str(self.day_max_lookback_window))
            running_date = datetime.now()
            starting_date = running_date - timedelta(days=2*self.day_max_lookback_window)
            daily_df = napoleon_connector.fetch_crypto_daily_data(ssj=self.underlying, local_root_directory=self.local_root_directory,
                                                                   daily_return_pkl_filename_suffix='_daily_temp.pkl',
                                                                   refetch_all=True,
                                                                   daily_crypto_starting_day=starting_date,
                                                                  daily_crypto_ending_day=running_date)
            daily_df = daily_df.sort_index()
            print('time range for daily recomputation')
            print(max(daily_df.index))
            print(min(daily_df.index))
            print(daily_df.shape)
            print('data refetched')
            self.recreate_daily_histo(daily_df,running_date)
        else :
            print('not enough ohlc binance history')


    def recreate_daily_histo(self, daily_df, up_to_date):
        cumulated_signals = None
        print('Number of signals to aggregate '+str(len(self.day_calib_signals_list)))
        skipping_size = self.day_max_lookback_window
        mapping_dictionary = {}
        counter = 1
        for me_signal in self.day_calib_signals_list:
            print('backtesting '+str(counter)+' over '+str(len(self.day_calib_signals_list)))
            # we have to reload the file each time
            freqly_df = daily_df.copy()
            ## idiosyncratic run itself
            run_json_string = signal_utility.recover_to_sql_column_format(me_signal)
            params = json.loads(run_json_string)
            normalization = params['normalization']
            transaction_costs = params['transaction_costs']
            trigger = params['trigger']
            signal_type = params['signal_type']
            if normalization and not signal_generator.is_signal_continuum(signal_type):
                return
            params = json.loads(run_json_string)
            salty = str(int(hashlib.sha1(run_json_string.encode('utf-8')).hexdigest(), 16) % (10 ** 8))
            params = json.loads(run_json_string)
            signal_column = params['signal_type'] + str(params['trigger']) + salty
            mapping_dictionary[me_signal]=signal_column
            if signal_type == 'long_only':
                freqly_df['signal']=1.
            else:
                lookback_window = params['lookback_window']
                if skipping_size < lookback_window:
                    raise Exception('The skipping size must be greater than the lookback window')
                # kwargs = {**generics, **idios}
                signal_generation_method_to_call = getattr(signal_generator, signal_type)
                freqly_df = signal_utility.roll_wrapper(freqly_df, lookback_window, skipping_size,
                                          lambda x: signal_generation_method_to_call(data=x, **params), trigger)
                freqly_df, _ = signal_utility.reconstitute_signal_perf(data = freqly_df, transaction_cost = transaction_costs, normalization = normalization)

            if cumulated_signals is None:
                freqly_df['signal_not_shifted'] = freqly_df['signal'].shift(-1)
                cumulated_signals = freqly_df[['close','high','low','open','volumefrom','signal_not_shifted']].copy()
                cumulated_signals = cumulated_signals.rename(columns={'signal_not_shifted': signal_column})
            else :
                freqly_df['signal_not_shifted'] = freqly_df['signal'].shift(-1)
                cumulated_signals[signal_column]=freqly_df['signal_not_shifted'].copy()
            counter=counter+1

        print('recomputation done')
        print(cumulated_signals.shape)
        cumulated_signals['date'] = pd.to_datetime(cumulated_signals.index)
        cumulated_signals['ts'] = cumulated_signals['date'].values.astype(np.int64) // 10 ** 9

        up_to_date = up_to_date - timedelta(days=1)
        cumulated_signals=cumulated_signals[cumulated_signals.index<=up_to_date]

        backed_up_signal_histo_name =  up_to_date.strftime('%d_%b_%Y')+'_'+self.underlying+'_'+self.backed_up_signal_histo_suffix
        saving_path = self.local_root_directory+backed_up_signal_histo_name
        print('saving histo recomputation to '+backed_up_signal_histo_name)
        cumulated_signals.to_pickle(saving_path)

    def reload_daily_histo(self, from_cryptocompare = False):
        upto_date = datetime.now() - timedelta(days=1)
        if from_cryptocompare :
            backed_up_signal_histo_name =  upto_date.strftime('%d_%b_%Y')+'_'+self.underlying+'_'+self.backed_up_signal_histo_suffix
            saving_path = self.local_root_directory+backed_up_signal_histo_name
            print('loading histo recomputation to '+backed_up_signal_histo_name)
            updated_df = pd.read_pickle(saving_path)
            updated_df=updated_df.sort_index()
            signal_columns = [self.day_signal_mapping[me_sig] for me_sig in self.day_calib_signals_list]
            me_columns = self.generic_columns + signal_columns
            updated_df = updated_df[me_columns].copy()
            day_df = updated_df[self.generic_columns].copy()
            self.day_signal_np = updated_df.values
            self.day_np = day_df.values
            self.day_df = day_df
        else:
            updated_df =self.get_aws_last_past_days()
            updated_df['date'] = pd.to_datetime(updated_df.index)
            updated_df['ts'] = updated_df['date'].values.astype(np.int64) // 10 ** 9
            updated_df = updated_df[updated_df.index <= upto_date]
            updated_df=updated_df.sort_index()
            signal_columns = [self.day_signal_mapping[me_sig] for me_sig in self.day_calib_signals_list]
            me_columns = self.generic_columns + signal_columns
            updated_df = updated_df[me_columns].copy()
            day_df = updated_df[self.generic_columns].copy()
            self.day_signal_np = updated_df.values
            self.day_np = day_df.values
            self.day_df = day_df
        return updated_df


    def start_fetching_pair(self):
        bm = BinanceSocketManager(self.client)
        pair_callback = partial(pair_handling_message, self)
        bm.start_aggtrade_socket(self.pair, pair_callback)
        bm.start()

    def reset_minute_np_to_lookback_window(self, loolback_window = 70):
        print('minutes shape before hour reset')
        print(self.minute_np.shape)
        print(self.minute_signal_np.shape)
        self.minute_np = self.minute_np[-loolback_window:]
        self.minute_signal_np = self.minute_signal_np[-loolback_window:]
        print('minutes shape afteer hour reset')
        print(self.minute_np.shape)
        print(self.minute_signal_np.shape)

    def reset_hour_np_to_lookback_window(self, loolback_window = 70):
        print('hours shape before day reset')
        print(self.hour_np.shape)
        self.hour_np = self.hour_np[-loolback_window:]
        print('hours shape after day reset')
        print(self.hour_np.shape)

    def update_day_dataframe(self):
        print('updating day df')
        if len(self.day_np.shape)>1:
            updated_df=pd.DataFrame(data=self.day_np, columns = self.generic_columns, index=range(len(self.day_np)))
        else:
            updated_df=pd.DataFrame(data=self.day_np.reshape(1,len(self.day_np)), columns = self.generic_columns, index=[0])
        updated_df['ts'] = updated_df['ts'].astype(int)
        updated_df['ts'] = pd.to_datetime(updated_df['ts'], unit='s')
        updated_df = updated_df.sort_values(by=['ts'])
        updated_df = updated_df.rename(columns={"ts": "date"}, errors="raise")
        updated_df = updated_df.set_index(updated_df['date'])
        updated_df = updated_df.drop(columns=['date'])
        self.day_df = updated_df
        self.last_day = max(self.day_df.index)

    def update_minute_dataframe(self):
        print('updating minute df')
        if len(self.minute_np.shape)>1:
            updated_df=pd.DataFrame(data=self.minute_np, columns = self.generic_columns, index=range(len(self.minute_np)))
        else:
            updated_df=pd.DataFrame(data=self.minute_np.reshape(1,len(self.minute_np)), columns = self.generic_columns, index=[0])
        updated_df['ts'] = updated_df['ts'].astype(int)
        updated_df['ts'] = pd.to_datetime(updated_df['ts'], unit='s')
        updated_df = updated_df.sort_values(by=['ts'])
        updated_df = updated_df.rename(columns={"ts": "date"}, errors="raise")
        updated_df = updated_df.set_index(updated_df['date'])
        updated_df = updated_df.drop(columns=['date'])
        self.minute_df = updated_df
        self.last_minute = max(self.minute_df.index)



    def upload_last_day_24_hours_signals_dataframe(self):
        print('updating 24 hours day df')
        updated_df=pd.DataFrame(data=self.hour_np, columns = self.generic_columns, index=range(len(self.hour_np)))
        # updated_df = updated_df.astype(
        #     {'ts': 'float64', 'open': 'float64', 'high': 'float64', 'low': 'float64', 'close': 'float64','volumefrom': 'float64'}
        #                                )
        # updated_df['ts']=updated_df['ts'].astype(int)
        # updated_df['hour_ts'] = updated_df['ts']-updated_df['ts']%3600
        # updated_df['day_ts'] = updated_df['ts']-updated_df['ts']%(3600*24)
        # updated_df = updated_df.astype({'hour_ts': 'int64'})
        # updated_df = updated_df.astype({'day_ts': 'int64'})
        # if updated_df['day_ts'].nunique()>1:
        #     print('too many days saved together : investigate')
        # day_timestamp = updated_df['day_ts'].max()

        updated_df['ts'] = updated_df['ts'].astype(int)
        me_ts =updated_df['ts']
        me_day_ts = me_ts-me_ts%(3600*24)
        me_day_ts = me_day_ts.astype({'ts': 'int64'})
        if me_day_ts.nunique()>1:
            print('too many days saved together : investigate')
        day_timestamp = me_day_ts.max()

        day_dt_object = datetime.utcfromtimestamp(day_timestamp)
        filename = self.pair+'_'+day_dt_object.strftime('%d_%b_%Y') #str(hourr_timestamp)
        updated_df['ts'] = pd.to_datetime(updated_df['ts'], unit='s')
        updated_df = updated_df.sort_values(by=['ts'])
        updated_df = updated_df.rename(columns={"ts": "date"}, errors="raise")
        updated_df = updated_df.set_index(updated_df['date'])
        updated_df = updated_df.drop(columns=['date'])
        self.hour_df = updated_df
        local_path = self.local_root_directory + filename
        self.hour_df.to_csv(local_path)
        print('uploading last day file '+local_path)
        self.aws_client.upload_file(self.bucket,local_path)

    def upload_last_hour_60_minutes_signals_dataframe(self):
        print('updating 60 minutes hour df')
        signal_columns = [self.minute_signal_mapping[me_sig] for me_sig in self.min_calib_signals_list]
        me_columns = self.generic_columns + signal_columns
        updated_df=pd.DataFrame(data=self.minute_signal_np, columns = me_columns, index=range(len(self.minute_signal_np)))
        #updated_df['ts']=updated_df['ts'].astype(int)
        # updated_df['hour_ts'] = updated_df['ts']-updated_df['ts']%3600
        # updated_df = updated_df.astype({'hour_ts': 'int64'})
        # if updated_df['hour_ts'].nunique()>1:
        #     print('too many hours saved together : investigate')
        # hour_timestamp = updated_df['hour_ts'].max()

        updated_df['ts'] = updated_df['ts'].astype(int)
        me_ts =updated_df['ts']
        me_hour_ts = me_ts-me_ts%3600
        me_hour_ts = me_hour_ts.astype({'ts': 'int64'})
        if me_hour_ts.nunique()>1:
            print('too many days saved together : investigate')
        hour_timestamp = me_hour_ts.max()

        hour_dt_object = datetime.utcfromtimestamp(hour_timestamp)
        filename = self.pair+'_'+hour_dt_object.strftime('%d_%b_%Y_%H') #str(hourr_timestamp)
        updated_df['ts'] = pd.to_datetime(updated_df['ts'], unit='s')
        updated_df = updated_df.sort_values(by=['ts'])
        updated_df = updated_df.rename(columns={"ts": "date"}, errors="raise")
        updated_df = updated_df.set_index(updated_df['date'])
        updated_df = updated_df.drop(columns=['date'])
        self.minute_df = updated_df
        local_path = self.local_root_directory + filename
        self.minute_df.to_csv(local_path)
        print('uploading last hour file '+local_path)
        self.aws_client.upload_file(self.bucket,local_path)

    def upload_days_signals_dataframe(self):
        print('updating day df')
        signal_columns = [self.day_signal_mapping[me_sig] for me_sig in self.day_calib_signals_list]
        me_columns = self.generic_columns + signal_columns
        updated_df=pd.DataFrame(data=self.day_signal_np, columns = me_columns, index=range(len(self.day_signal_np)))
        # updated_df['ts']=updated_df['ts'].astype(int)
        # updated_df['hour_ts'] = updated_df['ts']-updated_df['ts']%3600
        # updated_df['day_ts'] = updated_df['ts']-updated_df['ts']%(3600*24)
        # updated_df = updated_df.astype({'day_ts': 'int64'})
        # if updated_df['day_ts'].nunique()>1:
        #     print('too many days saved together : investigate')
        # day_timestamp = updated_df['day_ts'].max()
        updated_df['ts'] = updated_df['ts'].astype(int)
        me_ts =updated_df['ts']
        me_day_ts = me_ts-me_ts%(3600*24)
        me_day_ts = me_day_ts.astype({'ts': 'int64'})
        if me_day_ts.nunique()>1:
            print('too many days saved together : investigate')
        day_timestamp = me_day_ts.max()

        day_dt_object = datetime.utcfromtimestamp(day_timestamp)
        filename = self.pair+'_'+day_dt_object.strftime('%d_%b_%Y')+'_dayohlc' #str(hourr_timestamp)
        updated_df['ts'] = pd.to_datetime(updated_df['ts'], unit='s')
        updated_df = updated_df.sort_values(by=['ts'])
        updated_df = updated_df.rename(columns={"ts": "date"}, errors="raise")
        updated_df = updated_df.set_index(updated_df['date'])
        updated_df = updated_df.drop(columns=['date'])
        self.day_df = updated_df
        local_path = self.local_root_directory + filename
        self.day_df.to_csv(local_path)
        self.aws_client.upload_file(self.bucket,local_path)
        return

    def download_minute_dataframe(self, filename):
        dataframe = self.aws_client.download_dataframe_from_csv(self.bucket, filename)
        return dataframe

    def launchDayParallelPool(self, use_num_cpu):
        self.update_day_dataframe()
        print('launching parallel daily computation for alphas at '+str(self.last_day))
        with Pool(processes=use_num_cpu) as pool:
            run_results = pool.starmap(self.runDaySignal, self.day_calib_signals_list)
        print('parallel computation done')
        print('results length')
        print(len(run_results))
        raise Exception('plug trigger')


    def launchDaySequential(self):
        self.update_day_dataframe()
        print('launching sequential daily computation for alphas at '+str(self.last_day))
        run_results = []
        trigger_results = []
        for meArg in self.day_calib_signals_list:
            signal_value, trigger_value = self.runDaySignal(meArg)
            run_results.append(signal_value)
            trigger_results.append(trigger_value)
        print('results length')
        print(len(run_results))
        new_data_sig = np.array(run_results)
        if len(self.day_np.shape)>1:
            new_data_sig = np.hstack((self.day_np[-1,:].reshape(1, len(self.generic_columns)), new_data_sig.reshape(1, len(new_data_sig))))
        else:
            new_data_sig = np.hstack((self.day_np.reshape(1, len(self.generic_columns)), new_data_sig.reshape(1, len(new_data_sig))))
        #print('new daily data signal')
        #print(new_data_sig)
        if self.day_signal_np is None:
            self.day_signal_np = new_data_sig
        else:
            self.day_signal_np = np.vstack((self.day_signal_np, new_data_sig))
        if len(self.day_signal_np.shape)>1 and len(self.day_signal_np)>3:
            #print('last three daily signals')
            #print(self.day_signal_np[-3:])
            print('forward filling the triggered daily signals ')
            counter = 0
            for meArg in self.day_calib_signals_list:
                triggered = trigger_results[counter]
                if triggered:
                    signal_counter = len(self.generic_columns)+counter
                    for row_idx in range(1, self.day_signal_np.shape[0]):
                        if np.isnan(self.day_signal_np[row_idx, signal_counter]):
                            self.day_signal_np[row_idx, signal_counter] = self.day_signal_np[row_idx-1, signal_counter]
                else :
                    if np.isnan(self.day_signal_np[-1, signal_counter]):
                        self.day_signal_np[-1, signal_counter] = 0
                counter = counter + 1
            last_signals = self.day_signal_np[-1, len(self.generic_columns) :]
            return self.day_signals_ia_mixing(last_signals)
        return np.nan

    def launchMinuteParallelPool(self, use_num_cpu):
        self.update_minute_dataframe()
        print('launching parallel computation for alphas at '+str(self.last_minute))
        with Pool(processes=use_num_cpu) as pool:
            run_results = pool.starmap(self.runMinuteSignal, self.min_calib_signals_list)
        raise Exception('pluggg trigger')


    def launchMinuteSequential(self):
        self.update_minute_dataframe()
        print('launching sequential minute computation for alphas at '+str(self.last_minute))
        run_results = []
        trigger_results = []
        for meArg in self.min_calib_signals_list:
            signal_value, trigger_value = self.runMinuteSignal(meArg)
            run_results.append(signal_value)
            trigger_results.append(trigger_value)
        print('minute results length')
        print(len(run_results))
        #print('triggered results')
        #print(trigger_results)
        new_data_sig = np.array(run_results)
        if len(self.minute_np.shape)>1:
            new_data_sig = np.hstack((self.minute_np[-1,:].reshape(1, len(self.generic_columns)), new_data_sig.reshape(1, len(new_data_sig))))
        else:
            new_data_sig = np.hstack((self.minute_np.reshape(1, len(self.generic_columns)), new_data_sig.reshape(1, len(new_data_sig))))
        #print('new minute signal')
        #print(new_data_sig)
        if self.minute_signal_np is None:
            self.minute_signal_np = new_data_sig
        else:
            self.minute_signal_np = np.vstack((self.minute_signal_np, new_data_sig))

        if len(self.minute_signal_np.shape)>1 and len(self.minute_signal_np)>3:
            #print('last three signals')
            #print(self.minute_signal_np[-3:])
            print('forward filling the triggered minute signals ')
            counter = 0
            for meArg in self.min_calib_signals_list:
                triggered = trigger_results[counter]
                signal_counter = len(self.generic_columns) + counter
                if triggered:
                    for row_idx in range(1, self.minute_signal_np.shape[0]):
                        if np.isnan(self.minute_signal_np[row_idx, signal_counter]):
                            self.minute_signal_np[row_idx, signal_counter] = self.minute_signal_np[row_idx-1, signal_counter]
                else :
                    if np.isnan(self.minute_signal_np[-1, signal_counter]):
                        self.minute_signal_np[-1, signal_counter] = 0
                counter = counter + 1
            last_signals = self.minute_signal_np[-1, len(self.generic_columns) :]
            return self.minute_signals_ia_mixing(last_signals)
        return np.nan

    def day_signals_ia_mixing(self, signals):
        nb_nan_signals = np.isnan(signals).sum()
        print('still '+str(nb_nan_signals)+' not triggered day signals')
        return np.nanmean(signals)

    def minute_signals_ia_mixing(self, signals):
        nb_nan_signals = np.isnan(signals).sum()
        print('still '+str(nb_nan_signals)+' not triggered minute signals')
        return np.nanmean(signals)


    def runMinuteSignal(self, me_signal):
        ## idiosyncratic run itself
        run_json_string = signal_utility.recover_to_sql_column_format(me_signal)
        params = json.loads(run_json_string)
        signal_type = params['signal_type']
        normalization = params['normalization']
        trigger = params['trigger']
        transaction_costs = params['transaction_costs']
        if normalization and not signal_generator.is_signal_continuum(signal_type):
            return np.nan, trigger
        lookback_window = params['lookback_window']
        if lookback_window>self.minute_df.shape[0]:
            return np.nan, trigger
        freqly = self.minute_df.iloc[-lookback_window:,:].copy()
        signal_generation_method_to_call = getattr(signal_generator, signal_type)
        last_generated_signal = signal_utility.compute_last_signal(freqly, lookback_window,
                                                                   lambda x: signal_generation_method_to_call(
                                                                       data=x, **params))
        return last_generated_signal, trigger

    def runDaySignal(self, me_signal):
        ## idiosyncratic run itself
        run_json_string = signal_utility.recover_to_sql_column_format(me_signal)
        params = json.loads(run_json_string)
        signal_type = params['signal_type']
        normalization = params['normalization']
        trigger = params['trigger']
        transaction_costs = params['transaction_costs']
        if normalization and not signal_generator.is_signal_continuum(signal_type):
            return np.nan, trigger
        lookback_window = params['lookback_window']
        if lookback_window>self.day_df.shape[0]:
            return np.nan, trigger
        freqly = self.day_df.iloc[-lookback_window:,:].copy()
        signal_generation_method_to_call = getattr(signal_generator, signal_type)

        #last_generated_signal = np.nan
        #try:
        last_generated_signal = signal_utility.compute_last_signal(freqly, lookback_window,
                                                                       lambda x: signal_generation_method_to_call(
                                                                           data=x, **params))
        #except Exception as e:
        #    print('Trouble computing signal ', e)

        return last_generated_signal, trigger



    def hour_day_fetcher(self):
        dfs_list = self.get_aws_last_past_minutes(nb_last_hours = 24)
        dfs_list['ts'] = dfs_list.index
        # dfs_list['day_ts'] = dfs_list.ts.apply(lambda x: datetime.fromtimestamp(x).date())
        dfs_list['day_ts'] = dfs_list.ts.apply(lambda x: x.date())
        # dfs_list.ts = dfs_list.ts.apply(lambda x: datetime.fromtimestamp(x))
        dfs_list = dfs_list.sort_values('ts')
        current_day_df = dfs_list[dfs_list.day_ts>=datetime.today().date()]

        available_hours = current_day_df.hour_ts.unique()
        hourly_df = pd.DataFrame()

        ##Reconstitute hourly ohlc

        for hour in available_hours:
            current_df_hour = current_day_df[current_day_df.hour_ts == hour]
            approx_open_hour = current_df_hour['open'].values[0]
            approx_close_hour = current_df_hour['close'].values[-1]
            approx_high_hour = max(current_df_hour['high'].values)
            approx_low_hour = min(current_df_hour['low'].values)
            approx_ohlc_hour =  pd.DataFrame(np.array([[approx_open_hour,approx_high_hour,approx_low_hour, approx_close_hour, hour]]), columns = ['open', 'high', 'low', 'close', 'hour_ts'])
            if hourly_df.empty:
                hourly_df = approx_ohlc_hour
            else:
                hourly_df = pd.concat([hourly_df, approx_ohlc_hour], ignore_index=True)

        hourly_df.hour_ts = hourly_df.hour_ts.apply(lambda x: datetime.fromtimestamp(x))
        hourly_df.index = hourly_df.hour_ts
        hourly_df = hourly_df.loc[:,hourly_df.columns != 'hour_ts']

        ##Reconstitute last day ohlc

        approx_open_day = hourly_df['open'].values[0]
        approx_close_day = hourly_df['close'].values[-1]
        approx_high_day = max(hourly_df['high'].values)
        approx_low_day = min(hourly_df['low'].values)
        last_day_df = pd.DataFrame(np.array([[approx_open_day,approx_high_day,approx_low_day, approx_close_day]]), columns = ['open', 'high', 'low','close'])
        return hourly_df, last_day_df



