import time
import threading
import numpy as np
import pandas as pd
from pytz import timezone
from scipy.stats import norm
from datetime import datetime
from stockstats import StockDataFrame as Sdf
from rltrade.ibkr import IBapi, api_connect,stock_contract,future_contract

class IBKRDownloader:

    def __init__(self,
        start_date:str,end_date:str,
        ticker_list:list,
        sec_types:list,
        exchanges:list,
        start_time="09:30:00",
        end_time="15:59:00",
        demo=True):
        self.start_date = start_date.replace('-','')
        self.end_date = end_date.replace('-','')
        self.ticker_list = ticker_list
        self.sec_types = sec_types
        self.exchanges = exchanges
        self.start_date_dt = datetime.strptime(self.start_date,"%Y%m%d")
        self.end_date_dt = datetime.strptime(self.end_date,"%Y%m%d")
        self.date_delta =  (self.end_date_dt-self.start_date_dt)
        self.days = self.date_delta.days
        self.years = int(self.days / 365)
        self.dates = pd.date_range(start=self.start_date_dt,end =self.end_date_dt,freq='2D')
        self.dates = [''.join(str(x).split('-'))[:8] for x in self.dates]
        self.start_time = start_time
        self.end_time = end_time
        self.demo = demo
        self.count = 0
            
    def download_data(self,app:IBapi,id:int,ticker:str,sectype:str,exchange:str):
        def run_loop():
            app.run()
        thread = threading.Thread(target=run_loop,daemon=True)
        thread.start()
        while True:
            if isinstance(app.nextorderId, int):
                break
            else:
                print('waiting for connection')
                time.sleep(1)
        duration = f"{self.years} Y" if self.years > 0 else f"{self.days} D"
        if sectype == 'STK':
            cnt = stock_contract(ticker,secType=sectype,exchange=exchange)
        elif sectype == 'FUT':
            cnt = future_contract(ticker,secType=sectype,exchange=exchange)
        app.reqHistoricalData(id,cnt,self.end_date+" "+ self.end_time + " est", duration,'1 day','TRADES',1,2,False,[])
        app.nextorderId += 1
        return app

    def fetch_data(self):
        df = pd.DataFrame()
        not_downloaded = list()
        print("connecting to server...")
        app = api_connect(demo=self.demo)
        for i,tic in enumerate(self.ticker_list):
            print("Trying to download: ",tic)
            sec = self.sec_types[i]
            exchange = self.exchanges[i]
            try:
                temp_df = self.download_data(app,i,tic,sec,exchange)
                time.sleep(10)
                temp_df = app.get_df()
                temp_df['tic'] = tic
                temp_df['sec'] = sec
                temp_df['exchange'] = exchange
                app.reset_df()
                df = df.append(temp_df)
            except:
                print("Not able to download",tic)
                not_downloaded.append(tic)
        app.disconnect()
        time.sleep(5)
        if len(not_downloaded) > 0:
            print("IB was not able to download this ticker",not_downloaded)
        
        df = df.reset_index()
        df["date"] = pd.to_datetime(df['date'],format='%Y%m%d')
        df["day"] = df["date"].dt.dayofweek
        df["date"] = df["date"].apply(lambda x: x.strftime("%Y-%m-%d"))
        return df
    
    def download_min_data(self,app:IBapi,id:int,ticker:str,sectype:str,exchange:str,end:str):
        def run_loop():
            app.run()
        thread = threading.Thread(target=run_loop,daemon=True)
        thread.start()
        while True:
            if isinstance(app.nextorderId, int):
                break
            else:
                print('waiting for connection')
                time.sleep(1)
        if sectype == 'STK':
            cnt = stock_contract(ticker,secType=sectype,exchange=exchange)
        elif sectype == 'FUT':
            cnt = future_contract(ticker,secType=sectype,exchange=exchange)

        app.reqHistoricalData(id,cnt ,end+" "+ self.end_time + " est","2 D",'1 min','TRADES',1,1,False,[])
        app.nextorderId += 1
        return app
    
    def fetch_min_data(self):
        df = pd.DataFrame()
        not_downloaded = list()
        print("connecting to server...")
        app = api_connect(demo=self.demo)
        for i,tic in enumerate(self.ticker_list):
            print("Trying to download: ",tic)
            sec = self.sec_types[i]
            exchange  = self.exchanges[i]
            for end in self.dates[1:]:
                self.count += 1
                try:
                    app = self.download_min_data(app,self.count,tic,sec,exchange,end)
                    time.sleep(10)
                    temp_df = app.get_df()
                    temp_df['tic'] = tic
                    temp_df['sec'] = sec
                    temp_df['exchange'] = exchange
                    app.reset_df()
                    df = df.append(temp_df)
                except:
                    print("Not able to download",tic)
                    not_downloaded.append(tic)
        app.disconnect()
        time.sleep(5)
        if len(not_downloaded) > 0:
            print("IB was not able to download this ticker",not_downloaded)
        
        df["date"] = pd.to_datetime(df['date'],format='%Y%m%d  %H:%M:%S')
        df['time'] = df['date'].dt.time
        df['only date'] = df['date'].dt.date
        df = df[(df['time']>=datetime.strptime(self.start_time,"%H:%M:%S").time()) & 
                (df['time']<=datetime.strptime(self.end_time,"%H:%M:%S").time())]
        
        df['market closed'] = df.groupby('only date')['time'].transform(lambda x:(x==x.max()))
        df['market closed'] = df['market closed'] | df['market closed'].shift(-1).fillna(True)
        return df
    
    def download_todays_min_data(self,app:IBapi,id:int,ticker:str,sectype:str,exchange:str,end:str,duration:str):
        def run_loop():
            app.run()
        thread = threading.Thread(target=run_loop,daemon=True)
        thread.start()
        while True:
            if isinstance(app.nextorderId, int):
                break
            else:
                print('waiting for connection')
                time.sleep(1)
        if sectype == 'STK':
            cnt = stock_contract(ticker,secType=sectype,exchange=exchange)
        elif sectype == 'FUT':
            cnt = future_contract(ticker,secType=sectype,exchange=exchange)

        app.reqHistoricalData(id,cnt ,"", duration,'1 min','TRADES',1,1,True,[])
        time.sleep(5)
        app.nextorderId += 1
        return app
    
    def fetch_todays_min_data(self,end,duration):
        df = pd.DataFrame()
        not_downloaded = list()
        print("connecting to server...")
        app = api_connect(demo=self.demo)
        for i,tic in enumerate(self.ticker_list):
            self.count +=1
            sec = self.sec_types[i]
            exchange = self.exchanges[i]
            try:
                temp_df = self.download_todays_min_data(app,self.count,tic,sec,exchange,end,duration)
                time.sleep(1)
                temp_df = app.get_df()
                temp_df['tic'] = tic
                temp_df['sec'] = sec
                temp_df['exchange'] = exchange
                app.reset_df()
                df = df.append(temp_df)
            except:
                print("Not able to download",tic)
                not_downloaded.append(tic)   
        app.disconnect()
        time.sleep(5)
        if len(not_downloaded) > 0:
            print("IB was not able to download this ticker",not_downloaded)
        
        df["date"] = pd.to_datetime(df['date'],format='%Y%m%d  %H:%M:%S')
        df['time'] = df['date'].dt.time
        df['only date'] = df['date'].dt.date
        df = df[(df['time']>=datetime.strptime(self.start_time,"%H:%M:%S").time()) & 
                (df['time']<=datetime.strptime(self.end_time,"%H:%M:%S").time())]
        df['market closed'] = False
        return df

class OandaDownloader:
    def __init__(self,
    account,
    time_frame:str,
    start_date:str,
    end_date:str,
    ticker_list:list,
    start_time="09:30:00",
    end_time="15:59:00",
    ):
        self.account = account
        self.time_frame = time_frame
        self.start_date = start_date
        self.end_date = end_date
        self.start_time = start_time
        self.end_time = end_time
        self.ticker_list = ticker_list
        tz = timezone('EST')
        self.start_date_st = tz.localize(datetime.strptime(self.start_date+self.start_time,"%Y-%m-%d%H:%M:%S")).astimezone(tz)
        self.start_date_et = tz.localize(datetime.strptime(self.start_date+self.end_time,"%Y-%m-%d%H:%M:%S")).astimezone(tz)
        self.end_date_st = tz.localize(datetime.strptime(self.end_date+self.start_time,"%Y-%m-%d%H:%M:%S")).astimezone(tz)
        self.end_date_et = tz.localize(datetime.strptime(self.end_date+self.end_time,"%Y-%m-%d%H:%M:%S")).astimezone(tz)
        
        self.time_value = int(self.time_frame[:-1])
        self.time_format = self.time_frame[-1]

        if self.time_format == 'm':
            freq = f'{self.time_value * 1000} min' 
        elif self.time_format == 'h':
            freq = f'{self.time_value * 1000} H'
        elif self.time_format == 'd':
            freq = f'{self.time_value * 1000} D'

        self.date_range = pd.date_range(start=self.start_date_st,end=self.end_date_et,freq=freq)
        self.date_range = [tz.localize(datetime.strptime(str(x)[:-6],"%Y-%m-%d %H:%M:%S")).astimezone(tz) for x in self.date_range]
        self.date_range.append(self.end_date_et)
        self.date_range = self.date_range[1:]
        self.time_delta = (self.start_date_et-self.start_date_st)

        
    async def fetch_data(self):
            
        df = pd.DataFrame()
        for tic in self.ticker_list:
            for date in self.date_range:
                # if date.weekday() < 5:
                new_data = await self.account.get_historical_candles(tic,self.time_frame,date,limit=1000)
                print(f'Downloaded {len(new_data) if new_data else 0} historical data for {tic}')
                temp_df = pd.DataFrame(new_data)
                temp_df.columns = ['date','open','high','low','close','volume']
                temp_df['date'] = temp_df['date'].dt.tz_convert('EST')
                temp_df["date"] = pd.to_datetime(temp_df['date'],format='%Y%m%d  %H:%M:%S')
                temp_df['tic'] = tic
                temp_df['sec'] = '-'
                temp_df['exchange'] = '-'
                # if date.weekday() == 0:
                #     temp_df['time'] = temp_df['date'].dt.time
                #     temp_df = temp_df[(temp_df['time']>=datetime.strptime("01:00:00","%H:%M:%S").time())]
                #     temp_df.drop('time',inplace=True,axis=1)
                # elif date.weekday() == 4:
                #     temp_df['time'] = temp_df['date'].dt.time
                #     temp_df = temp_df[(temp_df['time']<=datetime.strptime("03:30:00","%H:%M:%S").time())]
                #     temp_df.drop('time',inplace=True,axis=1)
                df = df.append(temp_df)
        
        df['only date'] = df['date'].dt.date
        df['time'] = df['date'].dt.time
        df = df[(df['time']>=datetime.strptime(self.start_time,"%H:%M:%S").time()) & 
                (df['time']<=datetime.strptime(self.end_time,"%H:%M:%S").time())]
        # df['market closed'] = df.groupby('only date')['time'].transform(lambda x:(x==x.max()))
        # df['market closed'] = df['market closed'] | df['market closed'].shift(-1).fillna(True)
        df = df.reset_index(drop=True)
        return df
    
    async def fetch_todays_data(self,time_now,duration):
        df = pd.DataFrame()
        if self.time_format == 'm':
            duration = duration / (60 * self.time_value)
        elif self.time_format == 'h':
            duration = duration / (3600 * self.time_value)

        for tic in self.ticker_list:
            # if time_now.weekday() < 5:
            new_data = await self.account.get_historical_candles(tic,self.time_frame,time_now,limit=duration)
            print(f'Downloaded {len(new_data) if new_data else 0} historical data for {tic}')
            temp_df = pd.DataFrame(new_data)
            temp_df.columns = ['date','open','high','low','close','volume']
            temp_df['date'] = temp_df['date'].dt.tz_convert('EST')
            temp_df["date"] = pd.to_datetime(temp_df['date'],format='%Y%m%d  %H:%M:%S')
            temp_df['tic'] = tic
            temp_df['sec'] = '-'
            temp_df['exchange'] = '-'
            # if time_now.weekday() == 0:
            #     temp_df['time'] = temp_df['date'].dt.time
            #     temp_df = temp_df[(temp_df['time']>=datetime.strptime("01:00:00","%H:%M:%S").time())]
            #     temp_df.drop('time',inplace=True,axis=1)
            # elif time_now.weekday() == 4:
            #     temp_df['time'] = temp_df['date'].dt.time
            #     temp_df = temp_df[(temp_df['time']<=datetime.strptime("03:30:00","%H:%M:%S").time())]
            #     temp_df.drop('time',inplace=True,axis=1)
            df = df.append(temp_df)

        df['only date'] = df['date'].dt.date
        df['time'] = df['date'].dt.time
        df = df[(df['time']>=datetime.strptime(self.start_time,"%H:%M:%S").time()) & 
                (df['time']<=datetime.strptime(self.end_time,"%H:%M:%S").time())]
        # df['market closed'] = False
        df = df.reset_index(drop=True)

        return df

    async def fetch_daily_data(self):
            
        df = pd.DataFrame()
        for tic in self.ticker_list:
            for date in self.date_range:
                # if date.weekday() < 5:
                new_data = await self.account.get_historical_candles(tic,self.time_frame,date,limit=1000)
                print(f'Downloaded {len(new_data) if new_data else 0} historical data for {tic}')
                temp_df = pd.DataFrame(new_data)
                temp_df.columns = ['date','open','high','low','close','volume']
                temp_df['date'] = temp_df['date'].dt.tz_convert('EST')
                temp_df["date"] = pd.to_datetime(temp_df['date'],format='%Y%m%d  %H:%M:%S')
                temp_df['tic'] = tic
                temp_df['sec'] = '-'
                temp_df['exchange'] = '-'
                # if date.weekday() == 0:
                #     temp_df['time'] = temp_df['date'].dt.time
                #     temp_df = temp_df[(temp_df['time']>=datetime.strptime("01:00:00","%H:%M:%S").time())]
                #     temp_df.drop('time',inplace=True,axis=1)
                # elif date.weekday() == 4:
                #     temp_df['time'] = temp_df['date'].dt.time
                #     temp_df = temp_df[(temp_df['time']<=datetime.strptime("03:30:00","%H:%M:%S").time())]
                #     temp_df.drop('time',inplace=True,axis=1)
                df = df.append(temp_df)
        
        df['only date'] = df['date'].dt.date
        df = df.reset_index(drop=True)
        return df


class FeatureEngineer:

    def __init__(self,
                stock_indicator_list = [],
                additional_indicators = [],
                cov_matrix = False):
        self.stock_indicator_list = stock_indicator_list
        self.additional_indicators = additional_indicators
        self.indicators = self.stock_indicator_list + self.additional_indicators
        self.cov_matrix = cov_matrix
    
    def create_data(self,df):
        df = self.clean_data(df)

        if self.cov_matrix:
            df = self.add_cov_matrix(df)

        if 'hurst_exp' in self.additional_indicators:
            df = self.add_hurst_exponent(df)
        if 'half_hour_time' in self.additional_indicators:
            df = self.add_half_hour_time(df)

        if 'vix_fix_5' in self.additional_indicators:
            df = self.add_vix_fix(df,5,'vix_fix_5')
        if 'sharpe_5' in self.additional_indicators:
            df = self.add_sharpe(df,5,'sharpe_5')
        if 'sortino_5' in self.additional_indicators:
            df = self.add_sortino(df,5,'sortino_5')
        if 'calamar_5' in self.additional_indicators:
            df = self.add_clamar(df,5,'calamar_5')
        if 'vpin_5' in self.additional_indicators:
            df = self.add_vpin(df,5,'vpin_5')
        if 'max_value_price_5' in self.additional_indicators:
            df = self.add_max_value(df,5,'max_value_price_5','close')
        if 'min_value_price_5' in self.additional_indicators:
            df = self.add_min_value(df,5,'min_value_price_5','close')
        if 'max_value_volume_5' in self.additional_indicators:
            df = self.add_max_value(df,5,'max_value_volume_5','volume')
        if 'min_value_volume_5' in self.additional_indicators:
            df = self.add_min_value(df,5,'min_value_volume_5','volume')
        
        if 'vix_fix_22' in self.additional_indicators:
            df = self.add_vix_fix(df,22,'vix_fix_22')
        if 'sharpe_22' in self.additional_indicators:
            df = self.add_sharpe(df,22,'sharpe_22')
        if 'sortino_22' in self.additional_indicators:
            df = self.add_sortino(df,22,'sortino_22')
        if 'calamar_22' in self.additional_indicators:
            df = self.add_clamar(df,22,'calamar_22')
        if 'vpin_22' in self.additional_indicators:
            df = self.add_vpin(df,22,'vpin_22')
        if 'max_value_price_22' in self.additional_indicators:
            df = self.add_max_value(df,22,'max_value_price_22','close')
        if 'min_value_price_22' in self.additional_indicators:
            df = self.add_min_value(df,22,'min_value_price_22','close')
        if 'max_value_volume_22' in self.additional_indicators:
            df = self.add_max_value(df,22,'max_value_volume_22','volume')
        if 'min_value_volume_22' in self.additional_indicators:
            df = self.add_min_value(df,22,'min_value_volume_22','volume')


        if 'vix_fix_66' in self.additional_indicators:
            df = self.add_vix_fix(df,66,'vix_fix_66')
        if 'sharpe_66' in self.additional_indicators:
            df = self.add_sharpe(df,66,'sharpe_66')
        if 'sortino_66' in self.additional_indicators:
            df = self.add_sortino(df,66,'sortino_66')
        if 'calamar_66' in self.additional_indicators:
            df = self.add_clamar(df,66,'calamar_66')
        if 'vpin_66' in self.additional_indicators:
            df = self.add_vpin(df,66,'vpin_66')
        if 'max_value_price_66' in self.additional_indicators:
            df = self.add_max_value(df,66,'max_value_price_66','close')
        if 'min_value_price_66' in self.additional_indicators:
            df = self.add_min_value(df,66,'min_value_price_66','close')
        if 'max_value_volume_66' in self.additional_indicators:
            df = self.add_max_value(df,66,'max_value_volume_66','volume')
        if 'min_value_volume_66' in self.additional_indicators:
            df = self.add_min_value(df,66,'min_value_volume_66','volume')

        if 'vix_fix_1year' in self.additional_indicators:
            df = self.add_vix_fix(df,252,'vix_fix_1year')
        if 'sharpe_1year' in self.additional_indicators:
            df = self.add_sharpe(df,252,'sharpe_1year')
        if 'sortino_1year' in self.additional_indicators:
            df = self.add_sortino(df,252,'sortino_1year')
        if 'calamar_1year' in self.additional_indicators:
            df = self.add_clamar(df,252,'calamar_1year')
        if 'vpin_1year' in self.additional_indicators:
            df = self.add_vpin(df,252,'vpin_1year')
        if 'max_value_price_1year' in self.additional_indicators:
            df = self.add_max_value(df,252,'max_value_price_1year','close')
        if 'min_value_price_1year' in self.additional_indicators:
            df = self.add_min_value(df,252,'min_value_price_1year','close')
        if 'max_value_volume_1year' in self.additional_indicators:
            df = self.add_max_value(df,252,'max_value_volume_1year','volume')
        if 'min_value_volume_1year' in self.additional_indicators:
            df = self.add_min_value(df,252,'min_value_volume_1year','volume')
        
        if 'vix_fix_3year' in self.additional_indicators:
            df = self.add_vix_fix(df,3*252,'vix_fix_3year')
        if 'sharpe_3year' in self.additional_indicators:
            df = self.add_sharpe(df,3*252,'sharpe_3year')
        if 'sortino_3year' in self.additional_indicators:
            df = self.add_sortino(df,3*252,'sortino_3year')
        if 'calamar_3year' in self.additional_indicators:
            df = self.add_clamar(df,3*252,'calamar_3year')
        if 'vpin_3year' in self.additional_indicators:
            df = self.add_vpin(df,3*252,'vpin_3year')
        if 'max_value_price_3year' in self.additional_indicators:
            df = self.add_max_value(df,3*252,'max_value_price_3year','close')
        if 'min_value_price_3year' in self.additional_indicators:
            df = self.add_min_value(df,3*252,'min_value_price_3year','close')
        if 'max_value_volume_3year' in self.additional_indicators:
            df = self.add_max_value(df,3*252,'max_value_volume_3year','volume')
        if 'min_value_volume_3year' in self.additional_indicators:
            df = self.add_min_value(df,3*252,'min_value_volume_3year','volume')
        
        if 'vix_fix_5year' in self.additional_indicators:
            df = self.add_vix_fix(df,5*252,'vix_fix_5year')
        if 'sharpe_5year' in self.additional_indicators:
            df = self.add_sharpe(df,5*252,'sharpe_5year')
        if 'sortino_5year' in self.additional_indicators:
            df = self.add_sortino(df,5*252,'sortino_5year')
        if 'calamar_5year' in self.additional_indicators:
            df = self.add_clamar(df,5*252,'calamar_5year')
        if 'vpin_5year' in self.additional_indicators:
            df = self.add_vpin(df,5*252,'vpin_5year')
        if 'max_value_price_5year' in self.additional_indicators:
            df = self.add_max_value(df,5*252,'max_value_price_5year','close')
        if 'min_value_price_5year' in self.additional_indicators:
            df = self.add_min_value(df,5*252,'min_value_price_5year','close')
        if 'max_value_volume_5year' in self.additional_indicators:
            df = self.add_max_value(df,5*252,'max_value_volume_5year','volume')
        if 'min_value_volume_5year' in self.additional_indicators:
            df = self.add_min_value(df,5*252,'min_value_volume_5year','volume')

        if len(self.stock_indicator_list)>0:
            df = self.add_stock_indicators(df)

        df.loc[:,self.indicators] = df[self.indicators].replace([np.inf, -np.inf], np.nan)
        df = df.fillna(method="ffill").fillna(method="bfill")
        df = df.sort_values(["date", "tic"], ignore_index=True)
        df.index = df["date"].factorize()[0]
        return df
    
    def time_series_split(self,df, start, end, target_date_col="date"):
        df = df.copy()
        data = df[(df[target_date_col] >= start) & (df[target_date_col] < end)]
        data.index = data[target_date_col].factorize()[0]
        return data
        
    def train_test_split(self,df,train_period,test_period):
        df = self.create_data(df)
        train = self.time_series_split(df, start = train_period[0], end = train_period[1])
        test = self.time_series_split(df, start = test_period[0], end = test_period[1])
        return train,test
    
    def clean_data(self,data):
        df = data.copy()
        df = df.replace([np.inf, -np.inf], np.nan)
        df = df.fillna(method="ffill").fillna(method="bfill")
        df = df.drop_duplicates(subset=['date','tic'])
        df = df.reset_index(drop=True)
        df = self.skip_missing_dates(df)
        df = self.remove_corrupt_ticker(df)
        df.index = df.date.factorize()[0]
        df=df.sort_values(['date','tic'],ignore_index=True)
        return df
    
    def remove_corrupt_ticker(self,df:pd.DataFrame):
        a = df.groupby('tic')['close'].apply(lambda x:sum(x==0))
        invalid_ticker = a[a>0].index.tolist()
        df = df[~df['tic'].isin(invalid_ticker)]
        df = df.reset_index(drop=True)
        print("Tickers with corrupt Data",invalid_ticker)
        print("Remaining ticker",df.tic.unique().tolist())
        return df
    
    def skip_missing_dates(self,df:pd.DataFrame):
        n_ticker = df['tic'].nunique()
        a = df.groupby('date')['tic'].count()
        invalid_dates = a[a<n_ticker].index.tolist()
        df = df[~df['date'].isin(invalid_dates)]
        df = df.reset_index(drop=True)
        return df
    
    def add_half_hour_time(self,df:pd.DataFrame):
        df['half_hour_time'] = [x.hour*2 +2 if int((x.minute//30)*30) else x.hour*2+1 for x in df['date']]
        return df
    
    def add_cov_matrix(self,df,lookback=252):
        df.index = df.date.factorize()[0]

        cov_list = []
        for i in range(lookback,len(df.index.unique())):
            data_lookback = df.loc[i-lookback:i,:]
            price_lookback=data_lookback.pivot_table(index = 'date',columns = 'tic', values = 'close')
            return_lookback = price_lookback.pct_change().dropna()
            return_lookback = return_lookback.replace([np.inf, -np.inf], np.nan)
            return_lookback = return_lookback.fillna(method="ffill").fillna(method="bfill")
            covs = return_lookback.cov().values 
            cov_list.append(covs)
        
        df_cov = pd.DataFrame({'date':df.date.unique()[lookback:],'cov_list':cov_list})
        df = df.merge(df_cov, on='date',how='left')
        return df
    
    def add_hurst_exponent(self,data,max_lag=20):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['hurst_exp'] = temp['close'].rolling(max_lag*2).apply(lambda x:self.get_hurst_exponent(x.values))
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date", f'hurst_exp']], on=["tic", "date"], how="left")
        return df

    def get_hurst_exponent(self,time_series, max_lag=20):
        lags = range(2, max_lag)
        tau = [np.std(np.subtract(time_series[lag:], time_series[:-lag])) for lag in lags]
        reg = np.polyfit(np.log(lags), np.log(tau), 1)
        return reg[0]
    
    def add_max_value(self,data,days,name,column):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp[name] = temp[column].rolling(days,min_periods=1).max()
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df
    
    def add_min_value(self,data,days,name,column):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp[name] = temp[column].rolling(days,min_periods=1).min()
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df

    def add_sharpe(self,data,days,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['daily_return'] = temp['close'].pct_change(1)
            temp['daily_return'].fillna(0,inplace=True)
            temp[name] = temp['daily_return'].rolling(days,min_periods=1).mean() / temp['daily_return'].rolling(days,min_periods=1).std()
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date",name]], on=["tic", "date"], how="left")
        return df
    
    def add_sortino(self,data,days,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['daily_return'] = temp['close'].pct_change(1)
            temp['daily_return'].fillna(0,inplace=True) 
            temp['daily_negative_return'] = temp['daily_return'] 
            temp.loc[(temp['daily_negative_return']>0),'daily_negative_return'] = 0
            temp[name] = temp['daily_negative_return'].rolling(days,min_periods=1).mean() / temp['daily_negative_return'].rolling(days,min_periods=1).std()
            indicator_df = indicator_df.append(temp, ignore_index=True)
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df
    
    def add_clamar(self,data,days,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['daily_return'] = temp['close'].pct_change(1)
            temp['daily_drawndown'] = temp['daily_return'].diff(1)
            temp['daily_return'].fillna(0,inplace=True)
            temp[name] = temp['daily_return'].rolling(days,min_periods=1).mean()/temp['daily_drawndown'].rolling(days,min_periods=1).min()
            indicator_df = indicator_df.append(temp, ignore_index=True)
        df = df.merge(indicator_df[["tic", "date",name]], on=["tic", "date"], how="left")
        return df
    
    def add_vix_fix(self,data,days,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp[name] = ((temp['close'].rolling(days,min_periods=1).max() \
                                         - temp['low'])/temp['close'].rolling(days,min_periods=1).max()) * 100
            indicator_df = indicator_df.append(temp, ignore_index=True)
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df
    
    def get_buckets(self,df,bucketSize):
        volumeBuckets = pd.DataFrame(columns=['buy','sell','date'])
        count = 0
        BV = 0
        SV = 0
        for index,row in df.iterrows():
            newVolume = row['volume']
            z = row['z']
            if bucketSize < count + newVolume:
                BV = BV + (bucketSize-count)*z
                SV = SV + (bucketSize-count)*(1-z)
                volumeBuckets = volumeBuckets.append({'buy':BV,'sell':SV,'date':index},ignore_index=True)
                count = newVolume - (bucketSize-count)
                if int(count/bucketSize) > 0:
                    for i in range(0,int(count/bucketSize)):
                        BV = (bucketSize)*z
                        SV = (bucketSize)*(1-z)
                        volumeBuckets = volumeBuckets.append({'buy':BV,'sell':SV,'date':index},ignore_index=True)
                count = count%bucketSize
                BV = (count)*z
                SV = (count)*(1-z)
            else:
                BV = BV + (newVolume)*z
                SV = SV + (newVolume)*(1-z)
                count = count + newVolume
        
        volumeBuckets = volumeBuckets.drop_duplicates('date',keep='last')
        return volumeBuckets

    def calc_vpin(self,data,bucketSize,window):
        volume = (data['volume'])
        trades = (data['close'])
        trades = trades.diff(1).dropna()
        volume = volume.dropna()
        sigma = trades.std()
        z = trades.apply(lambda x: norm.cdf(x/sigma))
        df = pd.DataFrame({'z':z,'volume':volume}).dropna()
        volumeBuckets = self.get_buckets(df,bucketSize)
        volumeBuckets['vpin'] = abs(volumeBuckets['buy']-volumeBuckets['sell']).rolling(window,min_periods=1).mean()/bucketSize
        return volumeBuckets[['date','vpin']]
    
    def add_vpin(self,data,days,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic']==ticker)].copy()
            temp = temp.set_index('date')
            bucketsize = temp['volume'].mean()
            temp = self.calc_vpin(temp,bucketsize,days)
            temp[name] = temp['vpin']
            temp['tic'] = ticker
            indicator_df = indicator_df.append(temp,ignore_index=True)
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df

    def add_stock_indicators(self,data):
        df = data.copy()
        stock = Sdf.retype(df.copy())
        unique_ticker = stock.tic.unique()
        for indicator in self.stock_indicator_list:
            indicator_df = pd.DataFrame()
            for i in range(len(unique_ticker)):
                try:
                    temp_indicator = stock[stock.tic == unique_ticker[i]][indicator]
                    temp_indicator = pd.DataFrame(temp_indicator)
                    temp_indicator["tic"] = unique_ticker[i]
                    temp_indicator["date"] = df[df.tic == unique_ticker[i]][
                        "date"
                    ].to_list()
                    indicator_df = indicator_df.append(temp_indicator, ignore_index=True)
                except Exception as e:
                    print(e)
            df = df.merge(indicator_df[["tic", "date", indicator]], on=["tic", "date"], how="left")
        return df

class DayTradeFeatureEngineer:

    def __init__(self,
                stock_indicator_list = [],
                additional_indicators = [],
                cov_matrix = False):
        self.stock_indicator_list = stock_indicator_list
        self.additional_indicators = additional_indicators
        self.indicators = self.stock_indicator_list + self.additional_indicators
        self.cov_matrix = cov_matrix
    
    def create_data(self,df):
        df = self.clean_data(df)

        if self.cov_matrix:
            df = self.add_cov_matrix(df)

        if 'hurst_exp' in self.additional_indicators:
            df = self.add_hurst_exponent(df)
        if 'half_hour_time' in self.additional_indicators:
            df = self.add_half_hour_time(df)

        if 'vix_fix_10' in self.additional_indicators:
            df = self.add_vix_fix(df,10,'vix_fix_10')
        if 'sharpe_10' in self.additional_indicators:
            df = self.add_sharpe(df,10,'sharpe_10')
        if 'sortino_10' in self.additional_indicators:
            df = self.add_sortino(df,10,'sortino_10')
        if 'calamar_10' in self.additional_indicators:
            df = self.add_clamar(df,10,'calamar_10')
        if 'vpin_10' in self.additional_indicators:
            df = self.add_clamar(df,10,'vpin_10')
        if 'max_value_price_10' in self.additional_indicators:
            df = self.add_max_value(df,10,'max_value_price_10','close')
        if 'min_value_price_10' in self.additional_indicators:
            df = self.add_min_value(df,10,'min_value_price_10','close')
        if 'max_value_volume_10' in self.additional_indicators:
            df = self.add_max_value(df,10,'max_value_volume_10','volume')
        if 'min_value_volume_10' in self.additional_indicators:
            df = self.add_min_value(df,10,'min_value_volume_10','volume')
        
        if 'vix_fix_50' in self.additional_indicators:
            df = self.add_vix_fix(df,50,'vix_fix_50')
        if 'sharpe_50' in self.additional_indicators:
            df = self.add_sharpe(df,50,'sharpe_50')
        if 'sortino_50' in self.additional_indicators:
            df = self.add_sortino(df,50,'sortino_50')
        if 'calamar_50' in self.additional_indicators:
            df = self.add_clamar(df,50,'calamar_50')
        if 'vpin_50' in self.additional_indicators:
            df = self.add_clamar(df,50,'vpin_50')
        if 'max_value_price_50' in self.additional_indicators:
            df = self.add_max_value(df,50,'max_value_price_50','close')
        if 'min_value_price_50' in self.additional_indicators:
            df = self.add_min_value(df,50,'min_value_price_50','close')
        if 'max_value_volume_50' in self.additional_indicators:
            df = self.add_max_value(df,50,'max_value_volume_50','volume')
        if 'min_value_volume_50' in self.additional_indicators:
            df = self.add_min_value(df,50,'min_value_volume_50','volume')
        
        if 'vix_fix_100' in self.additional_indicators:
            df = self.add_vix_fix(df,100,'vix_fix_100')
        if 'sharpe_100' in self.additional_indicators:
            df = self.add_sharpe(df,100,'sharpe_100')
        if 'sortino_100' in self.additional_indicators:
            df = self.add_sortino(df,100,'sortino_100')
        if 'calamar_100' in self.additional_indicators:
            df = self.add_clamar(df,100,'calamar_100')
        if 'vpin_100' in self.additional_indicators:
            df = self.add_clamar(df,100,'vpin_100')
        if 'max_value_price_100' in self.additional_indicators:
            df = self.add_max_value(df,100,'max_value_price_100','close')
        if 'min_value_price_100' in self.additional_indicators:
            df = self.add_min_value(df,100,'min_value_price_100','close')
        if 'max_value_volume_100' in self.additional_indicators:
            df = self.add_max_value(df,100,'max_value_volume_100','volume')
        if 'min_value_volume_100' in self.additional_indicators:
            df = self.add_min_value(df,100,'min_value_volume_100','volume')
        
        if 'vix_fix_500' in self.additional_indicators:
            df = self.add_vix_fix(df,500,'vix_fix_500')
        if 'sharpe_500' in self.additional_indicators:
            df = self.add_sharpe(df,500,'sharpe_500')
        if 'sortino_500' in self.additional_indicators:
            df = self.add_sortino(df,500,'sortino_500')
        if 'calamar_500' in self.additional_indicators:
            df = self.add_clamar(df,500,'calamar_500')
        if 'vpin_500' in self.additional_indicators:
            df = self.add_clamar(df,500,'vpin_500')
        if 'max_value_price_500' in self.additional_indicators:
            df = self.add_max_value(df,500,'max_value_price_500','close')
        if 'min_value_price_500' in self.additional_indicators:
            df = self.add_min_value(df,500,'min_value_price_500','close')
        if 'max_value_volume_500' in self.additional_indicators:
            df = self.add_max_value(df,500,'max_value_volume_500','volume')
        if 'min_value_volume_500' in self.additional_indicators:
            df = self.add_min_value(df,500,'min_value_volume_500','volume')
        
        if 'vix_fix_1000' in self.additional_indicators:
            df = self.add_vix_fix(df,1000,'vix_fix_1000')
        if 'sharpe_1000' in self.additional_indicators:
            df = self.add_sharpe(df,1000,'sharpe_1000')
        if 'sortino_1000' in self.additional_indicators:
            df = self.add_sortino(df,1000,'sortino_1000')
        if 'calamar_1000' in self.additional_indicators:
            df = self.add_clamar(df,1000,'calamar_1000')
        if 'vpin_1000' in self.additional_indicators:
            df = self.add_clamar(df,1000,'vpin_1000')
        if 'max_value_price_1000' in self.additional_indicators:
            df = self.add_max_value(df,1000,'max_value_price_1000','close')
        if 'min_value_price_1000' in self.additional_indicators:
            df = self.add_min_value(df,1000,'min_value_price_1000','close')
        if 'max_value_volume_1000' in self.additional_indicators:
            df = self.add_max_value(df,1000,'max_value_volume_1000','volume')
        if 'min_value_volume_1000' in self.additional_indicators:
            df = self.add_min_value(df,1000,'min_value_volume_1000','volume')
        
        if len(self.stock_indicator_list)>0:
            df = self.add_stock_indicators(df)

        df.loc[:,self.indicators] = df[self.indicators].replace([np.inf, -np.inf], np.nan)
        df = df.fillna(method="ffill").fillna(method="bfill")
        df = df.sort_values(["date", "tic"], ignore_index=True)
        df['closing time'] = df.groupby('date')['date'].transform(lambda x:(x==x.max))
        df.index = df["date"].factorize()[0]
        return df
    
    def time_series_split(self,df, start, end, target_date_col="date"):
        df = df.copy()
        data = df[(df[target_date_col] >= start) & (df[target_date_col] < end)]
        data.index = data[target_date_col].factorize()[0]
        return data
        
    def train_test_split(self,df,train_period,test_period):
        df = self.create_data(df)
        train = self.time_series_split(df, start = train_period[0], end = train_period[1])
        test = self.time_series_split(df, start = test_period[0], end = test_period[1])
        return train,test
    
    def clean_data(self,data):
        df = data.copy()
        df = df.replace([np.inf, -np.inf], np.nan)
        df = df.fillna(method="ffill").fillna(method="bfill")
        df = df.drop_duplicates(subset=['date','tic'])
        df = df.reset_index(drop=True)
        df = self.skip_missing_dates(df)
        df = self.remove_corrupt_ticker(df)
        df.index = df.date.factorize()[0]
        df=df.sort_values(['date','tic'],ignore_index=True)
        return df
    
    def remove_corrupt_ticker(self,df:pd.DataFrame):
        a = df.groupby('tic')['close'].apply(lambda x:sum(x==0))
        invalid_ticker = a[a>0].index.tolist()
        df = df[~df['tic'].isin(invalid_ticker)]
        df = df.reset_index(drop=True)
        print("Tickers with corrupt Data",invalid_ticker)
        print("Remaining ticker",df.tic.unique().tolist())
        return df
    
    def skip_missing_dates(self,df:pd.DataFrame):
        n_ticker = df['tic'].nunique()
        a = df.groupby('date')['tic'].count()
        invalid_dates = a[a<n_ticker].index.tolist()
        df = df[~df['date'].isin(invalid_dates)]
        df = df.reset_index(drop=True)
        return df
    
    def add_half_hour_time(self,df:pd.DataFrame):
        df['half_hour_time'] = [x.hour*2 +2 if int((x.minute//30)*30) else x.hour*2+1 for x in df['date']]
        return df
    
    def add_cov_matrix(self,df,lookback=252):
        df.index = df.date.factorize()[0]

        cov_list = []
        for i in range(lookback,len(df.index.unique())):
            data_lookback = df.loc[i-lookback:i,:]
            price_lookback=data_lookback.pivot_table(index = 'date',columns = 'tic', values = 'close')
            return_lookback = price_lookback.pct_change().dropna()
            return_lookback = return_lookback.replace([np.inf, -np.inf], np.nan)
            return_lookback = return_lookback.fillna(method="ffill").fillna(method="bfill")
            covs = return_lookback.cov().values 
            cov_list.append(covs)
        
        df_cov = pd.DataFrame({'date':df.date.unique()[lookback:],'cov_list':cov_list})
        df = df.merge(df_cov, on='date',how='left')
        return df
    
    def add_hurst_exponent(self,data,max_lag=20):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['hurst_exp'] = temp['close'].rolling(max_lag*2).apply(lambda x:self.get_hurst_exponent(x.values))
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date", f'hurst_exp']], on=["tic", "date"], how="left")
        return df

    def get_hurst_exponent(self,time_series, max_lag=20):
        lags = range(2, max_lag)
        tau = [np.std(np.subtract(time_series[lag:], time_series[:-lag])) for lag in lags]
        reg = np.polyfit(np.log(lags), np.log(tau), 1)
        return reg[0]
    
    def add_max_value(self,data,days,name,column):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp[name] = temp[column].rolling(days,min_periods=1).max()
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df
    
    def add_min_value(self,data,days,name,column):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp[name] = temp[column].rolling(days,min_periods=1).min()
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df

    def add_sharpe(self,data,n,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['daily_return'] = temp['close'].pct_change(1)
            temp['daily_return'].fillna(0,inplace=True)
            temp[name] = temp['daily_return'].rolling(n,min_periods=1).mean() / temp['daily_return'].rolling(n,min_periods=1).std()
            indicator_df = indicator_df.append(temp, ignore_index=True )
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df
    
    def add_sortino(self,data,n,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['daily_return'] = temp['close'].pct_change(1)
            temp['daily_return'].fillna(0,inplace=True) 
            temp['daily_negative_return'] = temp['daily_return'] 
            temp.loc[(temp['daily_negative_return']>0),'daily_negative_return'] = 0
            temp[name] = temp['daily_negative_return'].rolling(n,min_periods=1).mean() / temp['daily_negative_return'].rolling(n,min_periods=1).std()
            indicator_df = indicator_df.append(temp, ignore_index=True)
        df = df.merge(indicator_df[["tic", "date",name]], on=["tic", "date"], how="left")
        return df
    
    def add_clamar(self,data,n,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp['daily_return'] = temp['close'].pct_change(1)
            temp['daily_drawndown'] = temp['daily_return'].diff(1)
            temp['daily_return'].fillna(0,inplace=True)
            temp[name] = temp['daily_return'].rolling(n,min_periods=1).mean()/temp['daily_drawndown'].rolling(n,min_periods=1).min()
            indicator_df = indicator_df.append(temp, ignore_index=True)
        df = df.merge(indicator_df[["tic", "date",name]], on=["tic", "date"], how="left")
        return df
    
    def add_vix_fix(self,data,n,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic'] == ticker)].copy()
            temp[name] = ((temp['close'].rolling(n,min_periods=1).max() \
                                         - temp['low'])/temp['close'].rolling(n,min_periods=1).max()) * 100
            indicator_df = indicator_df.append(temp, ignore_index=True)
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df
    
    def get_buckets(self,df,bucketSize):
        volumeBuckets = pd.DataFrame(columns=['buy','sell','date'])
        count = 0
        BV = 0
        SV = 0
        for index,row in df.iterrows():
            newVolume = row['volume']
            z = row['z']
            if bucketSize < count + newVolume:
                BV = BV + (bucketSize-count)*z
                SV = SV + (bucketSize-count)*(1-z)
                volumeBuckets = volumeBuckets.append({'buy':BV,'sell':SV,'date':index},ignore_index=True)
                count = newVolume - (bucketSize-count)
                if int(count/bucketSize) > 0:
                    for i in range(0,int(count/bucketSize)):
                        BV = (bucketSize)*z
                        SV = (bucketSize)*(1-z)
                        volumeBuckets = volumeBuckets.append({'buy':BV,'sell':SV,'date':index},ignore_index=True)
                count = count%bucketSize
                BV = (count)*z
                SV = (count)*(1-z)
            else:
                BV = BV + (newVolume)*z
                SV = SV + (newVolume)*(1-z)
                count = count + newVolume
        
        volumeBuckets = volumeBuckets.drop_duplicates('date',keep='last')
        return volumeBuckets

    def calc_vpin(self,data,bucketSize,window):
        volume = (data['volume'])
        trades = (data['close'])
        trades = trades.diff(1).dropna()
        volume = volume.dropna()
        sigma = trades.std()
        z = trades.apply(lambda x: norm.cdf(x/sigma))
        df = pd.DataFrame({'z':z,'volume':volume}).dropna()
        volumeBuckets = self.get_buckets(df,bucketSize)
        volumeBuckets['vpin'] = abs(volumeBuckets['buy']-volumeBuckets['sell']).rolling(window,min_periods=1).mean()/bucketSize
        return volumeBuckets[['date','vpin']]
    
    def add_vpin(self,data,n,name):
        df = data.copy()
        unique_ticker = df.tic.unique()
        indicator_df = pd.DataFrame()
        for ticker in unique_ticker:
            temp = df[(df['tic']==ticker)].copy()
            bucketsize = temp['volume'].mean()
            temp = self.calc_vpin(temp,bucketsize,n)
            temp[name] = temp['vpin']
            temp['tic'] = ticker
            indicator_df = indicator_df.append(temp,ignore_index=True)
        df = df.merge(indicator_df[["tic", "date", name]], on=["tic", "date"], how="left")
        return df


    def add_stock_indicators(self,data):
        df = data.copy()
        stock = Sdf.retype(df.copy())
        unique_ticker = stock.tic.unique()
        for indicator in self.stock_indicator_list:
            indicator_df = pd.DataFrame()
            for i in range(len(unique_ticker)):
                try:
                    temp_indicator = stock[stock.tic == unique_ticker[i]][indicator]
                    temp_indicator = pd.DataFrame(temp_indicator)
                    temp_indicator["tic"] = unique_ticker[i]
                    temp_indicator["date"] = df[df.tic == unique_ticker[i]][
                        "date"
                    ].to_list()
                    indicator_df = indicator_df.append(temp_indicator, ignore_index=True)
                except Exception as e:
                    print(e)
            df = df.merge(indicator_df[["tic", "date", indicator]], on=["tic", "date"], how="left")
        return df